[#82] Lexer: handle PHP 7.3 heredoc

scanner/lexer.go

@@ -41,6 +41,7 @@ type Lexer struct {
 	lastToken        *Token
 	Errors           []*errors.Error
 	NewLines         NewLines
+	PHPVersion       string
 }
 
 func (l *Lexer) ReturnTokenToPool(t *Token) {
@@ -132,6 +133,18 @@ func (lex *Lexer) isNotStringEnd(s byte) bool {
 }
 
 func (lex *Lexer) isHeredocEnd(p int) bool {
+	if lex.PHPVersion == "" {
+		return lex.isHeredocEndSince73(p)
+	}
+
+	if comparePHPVersion(lex.PHPVersion, "7.3") == -1 {
+		return lex.isHeredocEndBefore73(p)
+	}
+
+	return lex.isHeredocEndSince73(p)
+}
+
+func (lex *Lexer) isHeredocEndBefore73(p int) bool {
 	if lex.data[p-1] != '\r' && lex.data[p-1] != '\n' {
 		return false
 	}
@@ -152,6 +165,37 @@ func (lex *Lexer) isHeredocEnd(p int) bool {
 	return bytes.Equal(lex.heredocLabel, lex.data[p:p+l])
 }
 
+func (lex *Lexer) isHeredocEndSince73(p int) bool {
+	if lex.data[p-1] != '\r' && lex.data[p-1] != '\n' {
+		return false
+	}
+
+	for lex.data[p] == ' ' || lex.data[p] == '\t' {
+		p++
+	}
+
+	l := len(lex.heredocLabel)
+	if len(lex.data) < p+l {
+		return false
+	}
+
+	if len(lex.data) > p+l && isValidVarName(lex.data[p+l]) {
+		return false
+	}
+
+	a := string(lex.heredocLabel)
+	b := string(lex.data[p : p+l])
+
+	_, _ = a, b
+
+	if bytes.Equal(lex.heredocLabel, lex.data[p:p+l]) {
+		lex.p = p
+		return true
+	}
+
+	return false
+}
+
 func (lex *Lexer) isNotHeredocEnd(p int) bool {
 	return !lex.isHeredocEnd(p)
 }
@@ -221,5 +265,32 @@ func (lex *Lexer) Error(msg string) {
 }
 
 func isValidVarNameStart(r byte) bool {
-	return r >= 'A' && r <= 'Z' || r == '_' || r >= 'a' && r <= 'z' || r >= '\u007f' && r <= 'ÿ'
+	return (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || r == '_' || (r >= 0x80 && r <= 0xff)
+}
+
+func isValidVarName(r byte) bool {
+	return (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' || (r >= 0x80 && r <= 0xff)
+}
+
+func comparePHPVersion(a string, b string) int {
+	first := strings.Split(a, ".")
+	second := strings.Split(b, ".")
+
+	if first[0] < second[0] {
+		return -1
+	}
+
+	if first[0] > second[0] {
+		return 1
+	}
+
+	if first[1] < second[1] {
+		return -1
+	}
+
+	if first[1] > second[1] {
+		return 1
+	}
+
+	return 0
 }

scanner/scanner.go

@@ -17157,7 +17157,7 @@ func (lex *Lexer) Lex(lval Lval) int {
 				lex.setTokenPosition(token)
 				tok = T_ENCAPSED_AND_WHITESPACE
 
-				if lex.data[lex.p+1] != '$' && lex.data[lex.p+1] != '{' {
+				if len(lex.data) > lex.p+1 && lex.data[lex.p+1] != '$' && lex.data[lex.p+1] != '{' {
 					lex.cs = 486
 				}
 				{
@@ -17179,7 +17179,7 @@ func (lex *Lexer) Lex(lval Lval) int {
 			lex.setTokenPosition(token)
 			tok = T_ENCAPSED_AND_WHITESPACE
 
-			if lex.data[lex.p+1] != '$' && lex.data[lex.p+1] != '{' {
+			if len(lex.data) > lex.p+1 && lex.data[lex.p+1] != '$' && lex.data[lex.p+1] != '{' {
 				lex.cs = 486
 			}
 			{
@@ -17197,7 +17197,7 @@ func (lex *Lexer) Lex(lval Lval) int {
 			lex.setTokenPosition(token)
 			tok = T_ENCAPSED_AND_WHITESPACE
 
-			if lex.data[lex.p+1] != '$' && lex.data[lex.p+1] != '{' {
+			if len(lex.data) > lex.p+1 && lex.data[lex.p+1] != '$' && lex.data[lex.p+1] != '{' {
 				lex.cs = 486
 			}
 			{

scanner/scanner.rl

@@ -394,7 +394,7 @@ func (lex *Lexer) Lex(lval Lval) int {
                 lex.setTokenPosition(token);
                 tok = T_ENCAPSED_AND_WHITESPACE;
 
-                if lex.data[lex.p+1] != '$' && lex.data[lex.p+1] != '{' {
+                if len(lex.data) > lex.p+1 && lex.data[lex.p+1] != '$' && lex.data[lex.p+1] != '{' {
                     fnext heredoc_end;
                 }
                 fbreak;

scanner/scanner_test.go

@@ -892,6 +892,70 @@ CAT;
 	assert.DeepEqual(t, expected, actual)
 }
 
+func TestHereDocTokens73(t *testing.T) {
+	src := `<?php
+	<<<"CAT"
+		text
+	CAT, $b`
+
+	expected := []string{
+
+		T_START_HEREDOC.String(),
+		T_ENCAPSED_AND_WHITESPACE.String(),
+		T_END_HEREDOC.String(),
+		TokenID(int(',')).String(),
+		T_VARIABLE.String(),
+	}
+
+	lexer := NewLexer([]byte(src))
+	lexer.WithFreeFloating = true
+	lv := &lval{}
+	actual := []string{}
+
+	for {
+		token := lexer.Lex(lv)
+		if token == 0 {
+			break
+		}
+
+		actual = append(actual, TokenID(token).String())
+	}
+
+	assert.DeepEqual(t, expected, actual)
+}
+
+func TestHereDocTokensBefore73(t *testing.T) {
+	src := `<?php
+	<<<"CAT"
+	CAT
+CAT;`
+
+	expected := []string{
+
+		T_START_HEREDOC.String(),
+		T_ENCAPSED_AND_WHITESPACE.String(),
+		T_END_HEREDOC.String(),
+		TokenID(int(';')).String(),
+	}
+
+	lexer := NewLexer([]byte(src))
+	lexer.PHPVersion = "7.2"
+	lexer.WithFreeFloating = true
+	lv := &lval{}
+	actual := []string{}
+
+	for {
+		token := lexer.Lex(lv)
+		if token == 0 {
+			break
+		}
+
+		actual = append(actual, TokenID(token).String())
+	}
+
+	assert.DeepEqual(t, expected, actual)
+}
+
 func TestInlineHtmlNopTokens(t *testing.T) {
 	src := `<?php
 		$a; ?> test <?php