#80 implement Ragel based lexer

php7/parser.go

@@ -1,7 +1,6 @@
 package php7
 
 import (
-	"io"
 	"strings"
 
 	"github.com/z7zmey/php-parser/errors"
@@ -18,27 +17,24 @@ func (lval *yySymType) Token(t *scanner.Token) {
 
 // Parser structure
 type Parser struct {
-	*scanner.Lexer
-	path            string
+	Lexer           scanner.Scanner
 	currentToken    *scanner.Token
 	positionBuilder *parser.PositionBuilder
 	rootNode        node.Node
 }
 
 // NewParser creates and returns new Parser
-func NewParser(src io.Reader, path string) *Parser {
-	lexer := scanner.NewLexer(src, path)
+func NewParser(src []byte) *Parser {
+	lexer := scanner.NewLexer(src)
 
 	return &Parser{
 		lexer,
-		path,
 		nil,
 		nil,
 		nil,
 	}
 }
 
-// Lex proxy to lexer Lex
 func (l *Parser) Lex(lval *yySymType) int {
 	t := l.Lexer.Lex(lval)
 	l.currentToken = lval.token
@@ -53,17 +49,17 @@ func (l *Parser) Error(msg string) {
 		EndPos:    l.currentToken.EndPos,
 	}
 
-	l.Lexer.Errors = append(l.Lexer.Errors, errors.NewError(msg, pos))
+	l.Lexer.AddError(errors.NewError(msg, pos))
 }
 
 func (l *Parser) WithFreeFloating() {
-	l.Lexer.WithFreeFloating = true
+	l.Lexer.SetWithFreeFloating(true)
 }
 
 // Parse the php7 Parser entrypoint
 func (l *Parser) Parse() int {
 	// init
-	l.Lexer.Errors = nil
+	l.Lexer.SetErrors(nil)
 	l.rootNode = nil
 	l.positionBuilder = &parser.PositionBuilder{}
 
@@ -72,11 +68,6 @@ func (l *Parser) Parse() int {
 	return yyParse(l)
 }
 
-// GetPath return path to file
-func (l *Parser) GetPath() string {
-	return l.path
-}
-
 // GetRootNode returns root node
 func (l *Parser) GetRootNode() node.Node {
 	return l.rootNode
@@ -84,7 +75,7 @@ func (l *Parser) GetRootNode() node.Node {
 
 // GetErrors returns errors list
 func (l *Parser) GetErrors() []*errors.Error {
-	return l.Lexer.Errors
+	return l.Lexer.GetErrors()
 }
 
 // helpers
@@ -105,7 +96,7 @@ func isDollar(r rune) bool {
 }
 
 func (l *Parser) MoveFreeFloating(src node.Node, dst node.Node) {
-	if l.Lexer.WithFreeFloating == false {
+	if l.Lexer.GetWithFreeFloating() == false {
 		return
 	}
 
@@ -118,7 +109,7 @@ func (l *Parser) MoveFreeFloating(src node.Node, dst node.Node) {
 }
 
 func (l *Parser) setFreeFloating(dst node.Node, p freefloating.Position, strings []freefloating.String) {
-	if l.Lexer.WithFreeFloating == false {
+	if l.Lexer.GetWithFreeFloating() == false {
 		return
 	}
 
@@ -135,7 +126,7 @@ func (l *Parser) setFreeFloating(dst node.Node, p freefloating.Position, strings
 }
 
 func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []freefloating.String {
-	if l.Lexer.WithFreeFloating == false {
+	if l.Lexer.GetWithFreeFloating() == false {
 		return []freefloating.String{}
 	}
 
@@ -143,7 +134,7 @@ func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []freefloating.String {
 }
 
 func (l *Parser) addDollarToken(v node.Node) {
-	if l.Lexer.WithFreeFloating == false {
+	if l.Lexer.GetWithFreeFloating() == false {
 		return
 	}
 
@@ -162,7 +153,7 @@ func (l *Parser) addDollarToken(v node.Node) {
 }
 
 func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode node.Node, prevNode node.Node) {
-	if l.Lexer.WithFreeFloating == false {
+	if l.Lexer.GetWithFreeFloating() == false {
 		return
 	}
 
@@ -225,7 +216,7 @@ func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode node.Node, prevNode node.
 func (p *Parser) returnTokenToPool(yyDollar []yySymType, yyVAL *yySymType) {
 	for i := 1; i < len(yyDollar); i++ {
 		if yyDollar[i].token != nil {
-			p.TokenPool.Put(yyDollar[i].token)
+			p.Lexer.ReturnTokenToPool(yyDollar[i].token)
 		}
 		yyDollar[i].token = nil
 	}

php7/php7.y

@@ -303,11 +303,9 @@ start:
                 // save position
                 yylex.(*Parser).rootNode.SetPosition(yylex.(*Parser).positionBuilder.NewNodeListPosition($1))
 
+                yylex.(*Parser).setFreeFloating(yylex.(*Parser).rootNode, freefloating.End, yylex.(*Parser).currentToken.FreeFloating)
+
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
-                
-                if yylex.(*Parser).currentToken.Value == "\xff" {
-                    yylex.(*Parser).setFreeFloating(yylex.(*Parser).rootNode, freefloating.End, yylex.(*Parser).currentToken.FreeFloating)
-                }
             }
 ;
 
@@ -485,8 +483,6 @@ top_statement:
                 yylex.(*Parser).setFreeFloating($$, freefloating.SemiColon, yylex.(*Parser).GetFreeFloatingToken($4))
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
-
-                yylex.(*Parser).Begin(scanner.HALT_COMPILER)
             }
     |   T_NAMESPACE namespace_name ';'
             {
@@ -4194,8 +4190,8 @@ expr_without_variable:
 backup_doc_comment:
         /* empty */
             {
-                $$ = yylex.(*Parser).PhpDocComment
-                yylex.(*Parser).PhpDocComment = ""
+                $$ = yylex.(*Parser).Lexer.GetPhpDocComment()
+                yylex.(*Parser).Lexer.SetPhpDocComment("")
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
             }

php7/php7_bench_test.go

@@ -1,7 +1,6 @@
 package php7_test
 
 import (
-	"bytes"
 	"testing"
 
 	"github.com/z7zmey/php-parser/php7"
@@ -382,7 +381,7 @@ CAD;
 	`
 
 	for n := 0; n < b.N; n++ {
-		php7parser := php7.NewParser(bytes.NewBufferString(src), "test.php")
+		php7parser := php7.NewParser([]byte(src))
 		php7parser.Parse()
 	}
 }