From 83bb7610628e996e9e44741ea42c5aadabdc2060 Mon Sep 17 00:00:00 2001
From: z7zmey <z7zmey@gmail.com>
Date: Thu, 29 Mar 2018 16:46:21 +0300
Subject: [PATCH] issue #2 - fix template string scanning

`l.Prev.Rune` is actually current rune and `c` is next rune.
---
 node/scalar/t_encapsed_test.go | 24 +++++++++++++++++++
 scanner/scanner.go             | 34 +++++++++++++--------------
 scanner/scanner.l              | 39 ++++++++++++++++---------------
 scanner/scanner_test.go        | 42 ++++++++++++++++++++++++++++++++--
 4 files changed, 101 insertions(+), 38 deletions(-)

diff --git a/node/scalar/t_encapsed_test.go b/node/scalar/t_encapsed_test.go
index e4ffae4..5fb3745 100644
--- a/node/scalar/t_encapsed_test.go
+++ b/node/scalar/t_encapsed_test.go
@@ -36,6 +36,30 @@ func TestSimpleVar(t *testing.T) {
 	assertEqual(t, expected, actual)
 }
 
+func TestSimpleVarEndsEcapsed(t *testing.T) {
+	src := `<? "test $var\"";`
+
+	expected := &stmt.StmtList{
+		Stmts: []node.Node{
+			&stmt.Expression{
+				Expr: &scalar.Encapsed{
+					Parts: []node.Node{
+						&scalar.EncapsedStringPart{Value: "test "},
+						&expr.Variable{VarName: &node.Identifier{Value: "var"}},
+						&scalar.EncapsedStringPart{Value: "\\\""},
+					},
+				},
+			},
+		},
+	}
+
+	actual, _, _ := php7.Parse(bytes.NewBufferString(src), "test.php")
+	assertEqual(t, expected, actual)
+
+	actual, _, _ = php5.Parse(bytes.NewBufferString(src), "test.php")
+	assertEqual(t, expected, actual)
+}
+
 func TestSimpleVarPropertyFetch(t *testing.T) {
 	src := `<? "test $foo->bar()";`
 
diff --git a/scanner/scanner.go b/scanner/scanner.go
index d847c0b..7e8ed25 100644
--- a/scanner/scanner.go
+++ b/scanner/scanner.go
@@ -8624,40 +8624,40 @@ yyrule149: // .|[ \t\n\r]
 
 	F1:
 		for {
-			if c == -1 {
-				break
-			}
-			switch c {
-			case '"':
-				lval.Token(l.newToken(l.Token()))
-				return T_ENCAPSED_AND_WHITESPACE
-				break F1
-
+			switch l.Prev.Rune {
 			case '$':
 				c = l.Next()
-				if rune(c) == '{' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z' || c >= '\u007f' && c <= 'ÿ' {
-					l.ungetChars(1)
+				if l.Prev.Rune == '{' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z' || c >= '\u007f' && c <= 'ÿ' {
+					l.ungetChars(2)
 					tb := l.Token()
-					lval.Token(l.newToken(tb[:len(tb)-1]))
+					lval.Token(l.newToken(tb[:len(tb)-2]))
 					return T_ENCAPSED_AND_WHITESPACE
 					break F1
 				}
-				l.ungetChars(0)
+				l.ungetChars(1)
 
 			case '{':
 				c = l.Next()
-				if rune(c) == '$' {
-					l.ungetChars(1)
+				if l.Prev.Rune == '$' {
+					l.ungetChars(2)
 					tb := l.Token()
-					lval.Token(l.newToken(tb[:len(tb)-1]))
+					lval.Token(l.newToken(tb[:len(tb)-2]))
 					return T_ENCAPSED_AND_WHITESPACE
 					break F1
 				}
-				l.ungetChars(0)
+				l.ungetChars(1)
 			case '\\':
 				c = l.Next()
 			}
+			if rune(c) == '"' {
+				lval.Token(l.newToken(l.Token()))
+				return T_ENCAPSED_AND_WHITESPACE
+				break F1
+			}
 			c = l.Next()
+			if c == -1 {
+				break
+			}
 		}
 		goto yystate0
 	}
diff --git a/scanner/scanner.l b/scanner/scanner.l
index 3d2daf0..855b85a 100644
--- a/scanner/scanner.l
+++ b/scanner/scanner.l
@@ -10,7 +10,7 @@ package scanner
 import (
     "fmt"
     "bytes"
-	"github.com/cznic/golex/lex"
+    "github.com/cznic/golex/lex"
     "github.com/z7zmey/php-parser/comment"
 )
 
@@ -448,43 +448,44 @@ NEW_LINE        (\r|\n|\r\n)
 <STRING,HEREDOC,BACKQUOTE>\$                l.ungetChars(1);l.pushState(STRING_VAR)
 <STRING>.|[ \t\n\r]
     F1:for {
-        if c == -1 {
-            break;
-        }
-
-        switch c {
-            case '"' : 
-                lval.Token(l.newToken(l.Token()));
-                return T_ENCAPSED_AND_WHITESPACE
-                break F1;
-            
+        switch l.Prev.Rune {
             case '$':
                 c = l.Next();
-                if rune(c) == '{' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z' || c >= '\u007f' && c <= 'ÿ' {
-                    l.ungetChars(1)
+                if l.Prev.Rune == '{' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z' || c >= '\u007f' && c <= 'ÿ' {
+                    l.ungetChars(2)
                     tb := l.Token()
-                    lval.Token(l.newToken(tb[:len(tb)-1]));
+                    lval.Token(l.newToken(tb[:len(tb)-2]));
                     return T_ENCAPSED_AND_WHITESPACE
                     break F1;
                 } 
-                l.ungetChars(0)
+                l.ungetChars(1)
             
             case '{':
                 c = l.Next();
-                if rune(c) == '$' {
-                    l.ungetChars(1)
+                if l.Prev.Rune == '$' {
+                    l.ungetChars(2)
                     tb := l.Token()
-                    lval.Token(l.newToken(tb[:len(tb)-1])); 
+                    lval.Token(l.newToken(tb[:len(tb)-2])); 
                     return T_ENCAPSED_AND_WHITESPACE
                     break F1;
                 } 
-                l.ungetChars(0)
+                l.ungetChars(1)
 
             case '\\':
                 c = l.Next();
         }
 
+        if rune(c) == '"' {
+            lval.Token(l.newToken(l.Token()));
+            return T_ENCAPSED_AND_WHITESPACE
+            break F1;
+        }
+
         c = l.Next()
+
+        if c == -1 {
+            break;
+        }
     }
 
 <BACKQUOTE>.|[ \t\n\r]
diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go
index b9768bc..c72ed43 100644
--- a/scanner/scanner_test.go
+++ b/scanner/scanner_test.go
@@ -25,11 +25,11 @@ func assertEqual(t *testing.T, expected interface{}, actual interface{}) {
 }
 
 type lval struct {
-	token token.Token
+	Tkn token.Token
 }
 
 func (lv *lval) Token(t token.Token) {
-	lv.token = t
+	lv.Tkn = t
 }
 
 func TestIdentifier(t *testing.T) {
@@ -538,3 +538,41 @@ CAT;
 
 	assertEqual(t, expected, actual)
 }
+
+func TestStringTokensAfterVariable(t *testing.T) {
+	src := `<?php "test \"$var\""`
+
+	expected := []int{
+		scanner.Rune2Class('"'),
+		scanner.T_ENCAPSED_AND_WHITESPACE,
+		scanner.T_VARIABLE,
+		scanner.T_ENCAPSED_AND_WHITESPACE,
+		scanner.Rune2Class('"'),
+	}
+
+	expectedTokens := []string{
+		"\"",
+		"test \\\"",
+		"$var",
+		"\\\"",
+		"\"",
+	}
+
+	lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
+	lv := &lval{}
+	actual := []int{}
+	actualTokens := []string{}
+
+	for {
+		token := lexer.Lex(lv)
+		if token < 0 {
+			break
+		}
+
+		actualTokens = append(actualTokens, lv.Tkn.Value)
+		actual = append(actual, token)
+	}
+
+	assertEqual(t, expected, actual)
+	assertEqual(t, expectedTokens, actualTokens)
+}