Merge branch 'master' into dev

php5/php5.y

@@ -377,6 +377,8 @@ top_statement:
                 addMeta($$, $4.Meta, meta.SemiColonToken)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
+
+                yylex.(*Parser).Begin(scanner.HALT_COMPILER)
             }
     |   T_NAMESPACE namespace_name ';'
             {

php7/php7.y

@@ -476,6 +476,8 @@ top_statement:
                 addMeta($$, $4.Meta, meta.SemiColonToken)
 
                 yylex.(*Parser).returnTokenToPool(yyDollar, &yyVAL)
+
+                yylex.(*Parser).Begin(scanner.HALT_COMPILER)
             }
     |   T_NAMESPACE namespace_name ';'
             {

scanner/lexer.go

@@ -108,7 +108,7 @@ func (l *Lexer) popState() {
 	l.StateStack = l.StateStack[:len-1]
 }
 
-func (l *Lexer) begin(state int) {
+func (l *Lexer) Begin(state int) {
 	len := len(l.StateStack)
 	l.StateStack = l.StateStack[:len-1]
 	l.StateStack = append(l.StateStack, state)

scanner/scanner.l

@@ -24,6 +24,7 @@ const (
     NOWDOC
     HEREDOC
     BACKQUOTE
+    HALT_COMPILER
 )
 
 func isValidFirstVarNameRune(r rune) bool {
@@ -36,7 +37,7 @@ func (l *Lexer) Lex(lval Lval) int {
 
 %}
 
-%s PHP STRING STRING_VAR STRING_VAR_INDEX STRING_VAR_NAME PROPERTY HEREDOC_END NOWDOC HEREDOC BACKQUOTE
+%s PHP STRING STRING_VAR STRING_VAR_INDEX STRING_VAR_NAME PROPERTY HEREDOC_END NOWDOC HEREDOC BACKQUOTE HALT_COMPILER
 
 %yyb last == '\n' || last = '\0'
 %yyt l.getCurrentState()
@@ -84,13 +85,13 @@ NEW_LINE        (\r|\n|\r\n)
     lval.Token(l.createToken(tb))
     return int(T_INLINE_HTML)
 
-<INITIAL>\<\?php([ \t]|{NEW_LINE})              l.begin(PHP);l.ungetChars(len(l.Token())-5)
+<INITIAL>\<\?php([ \t]|{NEW_LINE})              l.Begin(PHP);l.ungetChars(len(l.Token())-5)
-<INITIAL>\<\?                                   l.begin(PHP);
+<INITIAL>\<\?                                   l.Begin(PHP);
-<INITIAL>\<\?=                                  l.begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_ECHO);
+<INITIAL>\<\?=                                  l.Begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_ECHO);
 
 <PHP>[ \t\n\r]+                                 l.addWhiteSpace(l.Token())
-<PHP>[;][ \t\n\r]*\?\>{NEW_LINE}?               l.begin(INITIAL);lval.Token(l.createToken(l.Token())); return Rune2Class(';');
+<PHP>[;][ \t\n\r]*\?\>{NEW_LINE}?               l.Begin(INITIAL);lval.Token(l.createToken(l.Token())); return Rune2Class(';');
-<PHP>\?\>{NEW_LINE}?                            l.begin(INITIAL);lval.Token(l.createToken(l.Token())); return Rune2Class(';');
+<PHP>\?\>{NEW_LINE}?                            l.Begin(INITIAL);lval.Token(l.createToken(l.Token())); return Rune2Class(';');
 
 <PHP>{DNUM}|{EXPONENT_DNUM}                     lval.Token(l.createToken(l.Token())); return int(T_DNUMBER)
 <PHP>{BNUM}
@@ -287,6 +288,8 @@ NEW_LINE        (\r|\n|\r\n)
 
     l.addComments(tb)
 
+<PHP>[/][*][*][/]
+    l.addComments(l.Token())
 <PHP>([/][*])|([/][*][*])
     tb := l.Token()
     is_doc_comment := false
@@ -295,6 +298,7 @@ NEW_LINE        (\r|\n|\r\n)
         l.PhpDocComment = ""
     }
 
+    c = l.Next()
     for {
         if c == -1 {
             break; // TODO: Unterminated comment starting line %d
@@ -322,16 +326,16 @@ NEW_LINE        (\r|\n|\r\n)
 <PHP>\${VAR_NAME}                               lval.Token(l.createToken(l.Token())); return int(T_VARIABLE)
 <PHP>{VAR_NAME}                                 lval.Token(l.createToken(l.Token())); return int(T_STRING)
 
-<PHP>->                                         l.begin(PROPERTY);lval.Token(l.createToken(l.Token())); return int(T_OBJECT_OPERATOR);
+<PHP>->                                         l.Begin(PROPERTY);lval.Token(l.createToken(l.Token())); return int(T_OBJECT_OPERATOR);
 <PROPERTY>[ \t\n\r]+                            l.addWhiteSpace(l.Token())
 <PROPERTY>->                                    lval.Token(l.createToken(l.Token())); return int(T_OBJECT_OPERATOR);
-<PROPERTY>{VAR_NAME}                            l.begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_STRING);
+<PROPERTY>{VAR_NAME}                            l.Begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_STRING);
-<PROPERTY>.                                     l.ungetChars(1);l.begin(PHP)
+<PROPERTY>.                                     l.ungetChars(1);l.Begin(PHP)
 
-<PHP>[\']([^\\\']*([\\].)*)*[\']                lval.Token(l.createToken(l.Token())); return int(T_CONSTANT_ENCAPSED_STRING);
+<PHP>[\']([^\\\']*(\\(.|\n))*)*[\']             lval.Token(l.createToken(l.Token())); return int(T_CONSTANT_ENCAPSED_STRING);
 
-<PHP>`                                          l.begin(BACKQUOTE); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
+<PHP>`                                          l.Begin(BACKQUOTE); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
-<BACKQUOTE>`                                    l.begin(PHP); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
+<BACKQUOTE>`                                    l.Begin(PHP); lval.Token(l.createToken(l.Token())); return Rune2Class(rune(l.TokenBytes(nil)[0]))
 
 <PHP>[b]?\<\<\<[ \t]*({VAR_NAME}|([']{VAR_NAME}['])|(["]{VAR_NAME}["])){NEW_LINE}
     tb := l.Token()
@@ -362,13 +366,13 @@ NEW_LINE        (\r|\n|\r\n)
         case '\'' : 
             lblFirst++
             lblLast--
-            l.begin(NOWDOC)
+            l.Begin(NOWDOC)
         case '"' : 
             lblFirst++
             lblLast--
-            l.begin(HEREDOC)
+            l.Begin(HEREDOC)
         default:
-            l.begin(HEREDOC)
+            l.Begin(HEREDOC)
     }
 
     l.heredocLabel = l.tokenString(tb[lblFirst:lblLast+1])
@@ -387,7 +391,7 @@ NEW_LINE        (\r|\n|\r\n)
         ungetCnt++
         c = l.Next()
         if '\n' == rune(c) || '\r' == rune(c) {
-            l.begin(HEREDOC_END)
+            l.Begin(HEREDOC_END)
         }
     }
 
@@ -407,14 +411,14 @@ NEW_LINE        (\r|\n|\r\n)
 
         if '\n' == rune(c) || '\r' == rune(c) {
             if l.heredocLabel + ";" == string(searchLabel) {
-                l.begin(HEREDOC_END)
+                l.Begin(HEREDOC_END)
                 tb = l.ungetChars(len(l.heredocLabel)+1)
                 tb = tb[:len(tb)-1]
                 break;
             }
 
             if l.heredocLabel == string(searchLabel) {
-                l.begin(HEREDOC_END)
+                l.Begin(HEREDOC_END)
                 tb = l.ungetChars(len(l.heredocLabel))
                 tb = tb[:len(tb)-1]
                 break;
@@ -431,8 +435,8 @@ NEW_LINE        (\r|\n|\r\n)
     lval.Token(l.createToken(tb) )
     return int(T_ENCAPSED_AND_WHITESPACE)
 
-<HEREDOC_END>{VAR_NAME}\;                       l.begin(PHP);lval.Token(l.createToken(l.ungetChars(1))); return int(T_END_HEREDOC)
+<HEREDOC_END>{VAR_NAME}\;                       l.Begin(PHP);lval.Token(l.createToken(l.ungetChars(1))); return int(T_END_HEREDOC)
-<HEREDOC_END>{VAR_NAME}                         l.begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_END_HEREDOC)
+<HEREDOC_END>{VAR_NAME}                         l.Begin(PHP);lval.Token(l.createToken(l.Token())); return int(T_END_HEREDOC)
 
 <PHP>[b]?[\"]
     binPrefix := l.Token()[0].Rune == 'b'
@@ -586,7 +590,7 @@ NEW_LINE        (\r|\n|\r\n)
 
             case '\n':
                 if l.heredocLabel + ";" == string(searchLabel) {
-                    l.begin(HEREDOC_END)
+                    l.Begin(HEREDOC_END)
                     l.ungetChars(len(l.heredocLabel)+1+nls)
 
                     i := len(tb) - len(l.heredocLabel) - 3 - nls
@@ -600,7 +604,7 @@ NEW_LINE        (\r|\n|\r\n)
                 }
 
                 if l.heredocLabel == string(searchLabel) { 
-                    l.begin(HEREDOC_END)
+                    l.Begin(HEREDOC_END)
                     l.ungetChars(len(l.heredocLabel)+nls)
 
                     i := len(tb) - len(l.heredocLabel) - 2 - nls
@@ -667,6 +671,8 @@ NEW_LINE        (\r|\n|\r\n)
 <STRING_VAR_NAME>{VAR_NAME}[\[\}]           l.popState();l.pushState(PHP);lval.Token(l.createToken(l.ungetChars(1))); return int(T_STRING_VARNAME)
 <STRING_VAR_NAME>.                          l.ungetChars(1);l.popState();l.pushState(PHP)
 
+<HALT_COMPILER>.|[ \t\n\r]                  // do nothing
+
 %%
     if _, ok := l.Abort(); ok {
         // always return same $end token

scanner/scanner_test.go

@@ -431,6 +431,53 @@ func TestTokens(t *testing.T) {
 	assertEqual(t, expected, actual)
 }
 
+func TestSingleQuoteStringTokens(t *testing.T) {
+	src := `<?php
+		'str $var str'
+		
+		'\''
+		
+		'\'
+		'
+		
+		'\
+		\''
+		
+		'\\'
+		
+		'\\
+		'
+
+		'\	
+		\''
+	`
+
+	expected := []string{
+		scanner.T_CONSTANT_ENCAPSED_STRING.String(),
+		scanner.T_CONSTANT_ENCAPSED_STRING.String(),
+		scanner.T_CONSTANT_ENCAPSED_STRING.String(),
+		scanner.T_CONSTANT_ENCAPSED_STRING.String(),
+		scanner.T_CONSTANT_ENCAPSED_STRING.String(),
+		scanner.T_CONSTANT_ENCAPSED_STRING.String(),
+		scanner.T_CONSTANT_ENCAPSED_STRING.String(),
+	}
+
+	lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
+	lv := &lval{}
+	actual := []string{}
+
+	for {
+		token := lexer.Lex(lv)
+		if token < 0 {
+			break
+		}
+
+		actual = append(actual, scanner.LexerToken(token).String())
+	}
+
+	assertEqual(t, expected, actual)
+}
+
 func TestTeplateStringTokens(t *testing.T) {
 	src := `<?php
 		"foo $a"
@@ -1033,6 +1080,26 @@ func TestInlineComment(t *testing.T) {
 	assertEqual(t, expected, actual)
 }
 
+func TestInlineComment2(t *testing.T) {
+	src := `<?php
+	/*/*/`
+
+	expected := []meta.Meta{
+		meta.NewWhiteSpace("\n\t", position.NewPosition(1, 2, 6, 7)),
+		meta.NewComment("/*/*/", position.NewPosition(2, 2, 8, 12)),
+	}
+
+	lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
+	lexer.WithMeta = true
+	lv := &lval{}
+
+	lexer.Lex(lv)
+
+	actual := lexer.Meta
+
+	assertEqual(t, expected, actual)
+}
+
 func TestEmptyInlineComment(t *testing.T) {
 	src := `<?php
 	/**/ `
@@ -1040,6 +1107,7 @@ func TestEmptyInlineComment(t *testing.T) {
 	expected := []meta.Meta{
 		meta.NewWhiteSpace("\n\t", position.NewPosition(1, 2, 6, 7)),
 		meta.NewComment("/**/", position.NewPosition(2, 2, 8, 11)),
+		meta.NewWhiteSpace(" ", position.NewPosition(2, 2, 12, 12)),
 	}
 
 	lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")