#80 implement Ragel based lexer

php5/parser.go

@@ -1,12 +1,10 @@
 package php5
 
 import (
-	"io"
 	"strings"
 
-	"github.com/z7zmey/php-parser/freefloating"
-
 	"github.com/z7zmey/php-parser/errors"
+	"github.com/z7zmey/php-parser/freefloating"
 	"github.com/z7zmey/php-parser/node"
 	"github.com/z7zmey/php-parser/parser"
 	"github.com/z7zmey/php-parser/position"
@@ -19,20 +17,18 @@ func (lval *yySymType) Token(t *scanner.Token) {
 
 // Parser structure
 type Parser struct {
-	*scanner.Lexer
-	path            string
+	Lexer           scanner.Scanner
 	currentToken    *scanner.Token
 	positionBuilder *parser.PositionBuilder
 	rootNode        node.Node
 }
 
 // NewParser creates and returns new Parser
-func NewParser(src io.Reader, path string) *Parser {
-	lexer := scanner.NewLexer(src, path)
+func NewParser(src []byte) *Parser {
+	lexer := scanner.NewLexer(src)
 
 	return &Parser{
 		lexer,
-		path,
 		nil,
 		nil,
 		nil,
@@ -54,17 +50,17 @@ func (l *Parser) Error(msg string) {
 		EndPos:    l.currentToken.EndPos,
 	}
 
-	l.Lexer.Errors = append(l.Lexer.Errors, errors.NewError(msg, pos))
+	l.Lexer.AddError(errors.NewError(msg, pos))
 }
 
 func (l *Parser) WithFreeFloating() {
-	l.Lexer.WithFreeFloating = true
+	l.Lexer.SetWithFreeFloating(true)
 }
 
 // Parse the php7 Parser entrypoint
 func (l *Parser) Parse() int {
 	// init
-	l.Lexer.Errors = nil
+	l.Lexer.SetErrors(nil)
 	l.rootNode = nil
 	l.positionBuilder = &parser.PositionBuilder{}
 
@@ -73,11 +69,6 @@ func (l *Parser) Parse() int {
 	return yyParse(l)
 }
 
-// GetPath return path to file
-func (l *Parser) GetPath() string {
-	return l.path
-}
-
 // GetRootNode returns root node
 func (l *Parser) GetRootNode() node.Node {
 	return l.rootNode
@@ -85,7 +76,7 @@ func (l *Parser) GetRootNode() node.Node {
 
 // GetErrors returns errors list
 func (l *Parser) GetErrors() []*errors.Error {
-	return l.Lexer.Errors
+	return l.Lexer.GetErrors()
 }
 
 // helpers
@@ -106,7 +97,7 @@ func isDollar(r rune) bool {
 }
 
 func (l *Parser) MoveFreeFloating(src node.Node, dst node.Node) {
-	if l.Lexer.WithFreeFloating == false {
+	if l.Lexer.GetWithFreeFloating() == false {
 		return
 	}
 
@@ -119,7 +110,7 @@ func (l *Parser) MoveFreeFloating(src node.Node, dst node.Node) {
 }
 
 func (l *Parser) setFreeFloating(dst node.Node, p freefloating.Position, strings []freefloating.String) {
-	if l.Lexer.WithFreeFloating == false {
+	if l.Lexer.GetWithFreeFloating() == false {
 		return
 	}
 
@@ -136,7 +127,7 @@ func (l *Parser) setFreeFloating(dst node.Node, p freefloating.Position, strings
 }
 
 func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []freefloating.String {
-	if l.Lexer.WithFreeFloating == false {
+	if l.Lexer.GetWithFreeFloating() == false {
 		return []freefloating.String{}
 	}
 
@@ -144,7 +135,7 @@ func (l *Parser) GetFreeFloatingToken(t *scanner.Token) []freefloating.String {
 }
 
 func (l *Parser) addDollarToken(v node.Node) {
-	if l.Lexer.WithFreeFloating == false {
+	if l.Lexer.GetWithFreeFloating() == false {
 		return
 	}
 
@@ -163,7 +154,7 @@ func (l *Parser) addDollarToken(v node.Node) {
 }
 
 func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode node.Node, prevNode node.Node) {
-	if l.Lexer.WithFreeFloating == false {
+	if l.Lexer.GetWithFreeFloating() == false {
 		return
 	}
 
@@ -226,7 +217,7 @@ func (l *Parser) splitSemiColonAndPhpCloseTag(htmlNode node.Node, prevNode node.
 func (p *Parser) returnTokenToPool(yyDollar []yySymType, yyVAL *yySymType) {
 	for i := 1; i < len(yyDollar); i++ {
 		if yyDollar[i].token != nil {
-			p.TokenPool.Put(yyDollar[i].token)
+			p.Lexer.ReturnTokenToPool(yyDollar[i].token)
 		}
 		yyDollar[i].token = nil
 	}

php5/php5.y

@@ -282,11 +282,9 @@ start:
                 yylex.(*Parser).rootNode = node.NewRoot($1)
                 yylex.(*Parser).rootNode.SetPosition(yylex.(*Parser).positionBuilder.NewNodeListPosition($1))
 
+                yylex.(*Parser).setFreeFloating(yylex.(*Parser).rootNode, freefloating.End, yylex.(*Parser).currentToken.FreeFloating)
+
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
-                
-                if yylex.(*Parser).currentToken.Value == "\xff" {
-                    yylex.(*Parser).setFreeFloating(yylex.(*Parser).rootNode, freefloating.End, yylex.(*Parser).currentToken.FreeFloating)
-                }
             }
 ;
 
@@ -383,8 +381,6 @@ top_statement:
                 yylex.(*Parser).setFreeFloating($$, freefloating.SemiColon, yylex.(*Parser).GetFreeFloatingToken($4))
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
-
-                yylex.(*Parser).Begin(scanner.HALT_COMPILER)
             }
     |   T_NAMESPACE namespace_name ';'
             {
@@ -871,8 +867,6 @@ inner_statement:
                 yylex.(*Parser).setFreeFloating($$, freefloating.SemiColon, yylex.(*Parser).GetFreeFloatingToken($4))
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
-
-                yylex.(*Parser).Begin(scanner.HALT_COMPILER)
             }
 ;
 

php5/php5_bench_test.go

@@ -1,7 +1,6 @@
 package php5_test
 
 import (
-	"bytes"
 	"testing"
 
 	"github.com/z7zmey/php-parser/php5"
@@ -414,7 +413,7 @@ CAD;
 	`
 
 	for n := 0; n < b.N; n++ {
-		php5parser := php5.NewParser(bytes.NewBufferString(src), "test.php")
+		php5parser := php5.NewParser([]byte(src))
 		php5parser.Parse()
 	}
 }