scanner: fix heredoc

scanner/scanner.go

@@ -8720,57 +8720,65 @@ yyrule152: // .|[ \t\n\r]
 		searchLabel := []byte{}
 		tb := []lex.Char{}
 
-	HEREDOCFOR:
 		for {
 			if c == -1 {
 				break
 			}
+			nls := 0
 			switch c {
-			case '\n':
-				fallthrough
 			case '\r':
+				nls = 1
+				c := l.Next()
+				if c != '\n' {
+					nls = 0
+					l.ungetChars(0)
+				}
+				fallthrough
+			case '\n':
 				if bytes.Equal(append(l.charsToBytes(heredocLabel), ';'), searchLabel) {
 					l.begin(HEREDOC_END)
-					tb = l.ungetChars(len(heredocLabel) + 1)
+					tb = l.ungetChars(len(heredocLabel) + 1 + nls)
-					break HEREDOCFOR
+					lval.Token(l.newToken(tb))
+					return T_ENCAPSED_AND_WHITESPACE
 				}
 				if bytes.Equal(l.charsToBytes(heredocLabel), searchLabel) {
 					l.begin(HEREDOC_END)
-					tb = l.ungetChars(len(heredocLabel))
+					tb = l.ungetChars(len(heredocLabel) + nls)
-					break HEREDOCFOR
+					lval.Token(l.newToken(tb))
+					return T_ENCAPSED_AND_WHITESPACE
 				}
 
 				searchLabel = []byte{}
 
 			case '$':
 				c = l.Next()
-				if rune(c) == '{' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z' || c >= '\u007f' && c <= 'ÿ' {
+				if rune(c) == '{' || isValidFirstVarNameRune(rune(c)) {
 					tb = l.ungetChars(1)
-					break HEREDOCFOR
+					lval.Token(l.newToken(tb))
+					return T_ENCAPSED_AND_WHITESPACE
 				}
 				l.ungetChars(0)
-				searchLabel = []byte{}
 
 			case '{':
 				c = l.Next()
 				if rune(c) == '$' {
 					tb = l.ungetChars(1)
-					break HEREDOCFOR
+					lval.Token(l.newToken(tb))
+					return T_ENCAPSED_AND_WHITESPACE
 				}
 				l.ungetChars(0)
-				searchLabel = []byte{}
 			case '\\':
 				c = l.Next()
-				searchLabel = []byte{}
+				if c == '\n' || c == '\r' {
+					l.ungetChars(0)
+				}
 
 			default:
 				searchLabel = append(searchLabel, byte(rune(c)))
 			}
 			c = l.Next()
 		}
-
+		goto yystate0
-		lval.Token(l.newToken(tb))
-		return T_ENCAPSED_AND_WHITESPACE
 	}
 yyrule153: // \${VAR_NAME}
 	{

scanner/scanner.l

@@ -578,49 +578,65 @@ NEW_LINE        (\r|\n|\r\n)
     searchLabel := []byte{}
     tb := []lex.Char{}
     
-    HEREDOCFOR:for {
+    for {
         if c == -1 {
             break;
         }
 
+        nls := 0
+
         switch c {
-            case '\n': fallthrough
             case '\r':
+                nls = 1
+                c := l.Next()
+
+                if c != '\n' {
+                    nls = 0
+                    l.ungetChars(0)
+                }
+
+                fallthrough
+
+            case '\n':
                 if bytes.Equal(append(l.charsToBytes(heredocLabel), ';'), searchLabel) { 
                     l.begin(HEREDOC_END)
-                    tb = l.ungetChars(len(heredocLabel)+1)
+                    tb = l.ungetChars(len(heredocLabel)+1+nls)
-                    break HEREDOCFOR;
+                    lval.Token(l.newToken(tb)); 
+                    return T_ENCAPSED_AND_WHITESPACE
                 }
 
                 if bytes.Equal(l.charsToBytes(heredocLabel), searchLabel) { 
                     l.begin(HEREDOC_END)
-                    tb = l.ungetChars(len(heredocLabel))
+                    tb = l.ungetChars(len(heredocLabel)+nls)
-                    break HEREDOCFOR;
+                    lval.Token(l.newToken(tb)); 
+                    return T_ENCAPSED_AND_WHITESPACE
                 }
                 
                 searchLabel = []byte{}
             
             case '$':
                 c = l.Next();
-                if rune(c) == '{' || c >= 'A' && c <= 'Z' || c == '_' || c >= 'a' && c <= 'z' || c >= '\u007f' && c <= 'ÿ' {
+                if rune(c) == '{' || isValidFirstVarNameRune(rune(c)) {
                     tb = l.ungetChars(1)
-                    break HEREDOCFOR;
+                    lval.Token(l.newToken(tb)); 
+                    return T_ENCAPSED_AND_WHITESPACE
                 } 
                 l.ungetChars(0)
-                searchLabel = []byte{}
             
             case '{':
                 c = l.Next();
                 if rune(c) == '$' {
                     tb = l.ungetChars(1)
-                    break HEREDOCFOR;
+                    lval.Token(l.newToken(tb)); 
+                    return T_ENCAPSED_AND_WHITESPACE
                 } 
                 l.ungetChars(0)
-                searchLabel = []byte{}
 
             case '\\':
                 c = l.Next();
-                searchLabel = []byte{}
+                if c == '\n' || c == '\r' {
+                    l.ungetChars(0)
+                }
             
             default:
                 searchLabel = append(searchLabel, byte(rune(c)))
@@ -629,9 +645,6 @@ NEW_LINE        (\r|\n|\r\n)
         c = l.Next()
     }
 
-    lval.Token(l.newToken(tb)); 
-    return T_ENCAPSED_AND_WHITESPACE
-
 <STRING_VAR>\${VAR_NAME}                    lval.Token(l.newToken(l.Token())); return T_VARIABLE
 <STRING_VAR>->{VAR_NAME}                    lval.Token(l.newToken(l.ungetChars(len(l.Token())-2))); return T_OBJECT_OPERATOR
 <STRING_VAR>{VAR_NAME}                      l.popState();lval.Token(l.newToken(l.Token())); return T_STRING

scanner/scanner_test.go

@@ -693,6 +693,79 @@ CAT;
 	assertEqual(t, expected, actual)
 }
 
+func TestHereDocTokens2(t *testing.T) {
+	src := `<?php
+	<<<CAT
+$foo/
+CAT;
+
+	<<<CAT
+$foo/100
+CAT;
+
+	<<<CAT
+$/$foo
+CAT;
+
+	<<<CAT
+$0$foo
+CAT;
+
+	<<<CAT
+$foo$bar\
+CAT
+`
+
+	expected := []int{
+		scanner.T_START_HEREDOC,
+		scanner.T_VARIABLE,
+		scanner.T_ENCAPSED_AND_WHITESPACE,
+		scanner.T_END_HEREDOC,
+		scanner.Rune2Class(';'),
+
+		scanner.T_START_HEREDOC,
+		scanner.T_VARIABLE,
+		scanner.T_ENCAPSED_AND_WHITESPACE,
+		scanner.T_END_HEREDOC,
+		scanner.Rune2Class(';'),
+
+		scanner.T_START_HEREDOC,
+		scanner.T_ENCAPSED_AND_WHITESPACE,
+		scanner.T_VARIABLE,
+		scanner.T_ENCAPSED_AND_WHITESPACE,
+		scanner.T_END_HEREDOC,
+		scanner.Rune2Class(';'),
+
+		scanner.T_START_HEREDOC,
+		scanner.T_ENCAPSED_AND_WHITESPACE,
+		scanner.T_VARIABLE,
+		scanner.T_ENCAPSED_AND_WHITESPACE,
+		scanner.T_END_HEREDOC,
+		scanner.Rune2Class(';'),
+
+		scanner.T_START_HEREDOC,
+		scanner.T_VARIABLE,
+		scanner.T_VARIABLE,
+		scanner.T_ENCAPSED_AND_WHITESPACE,
+		scanner.T_END_HEREDOC,
+	}
+
+	lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
+	lv := &lval{}
+	actual := []int{}
+
+	for {
+		token := lexer.Lex(lv)
+		if token < 0 {
+			break
+		}
+
+		actual = append(actual, token)
+	}
+
+	assertEqual(t, expected, actual)
+}
+
 func TestInlineHtmlNopTokens(t *testing.T) {
 	src := `<?php
 		$a; ?> test <?php