php8: fixed a bug with # comments (#22)

Example from issue:
```php
<?php
#
# Comment
#

$a = 100;
```

The problem with the example from the issue is that `#` is immediately followed by a line break.
And since the rule in the lexer for such comments was changed, this case was handled incorrectly.

```
(('#' ^'[') | '//') any_line* when is_not_comment_end => {
   lex.ungetStr("?>")
   lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
};
```

This rule has one problem, it checks two characters at once, first for the match `#`, and 
then for the mismatch `[`, which leads to the fact that in the case of an empty comment, the first 
matcher will capture `#`, and the second line break (`\n`), which will lead to the fact that `any_line` 
matcher will not work and will not increase the line number.

The next rule added is specifically for this case.

```
'#' newline when is_not_comment_end => {
    lex.ungetStr("?>")
    lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
};
```
This commit is contained in:
Makhnev Petr
2021-08-02 12:37:08 +03:00
committed by GitHub
parent d846011a9b
commit d85f5a4816
4 changed files with 12654 additions and 12360 deletions

View File

@@ -18,6 +18,9 @@ type LexerTokenStructTestSuite struct {
Expected []*token.Token
Version version.Version
withPosition bool
withFreeFloating bool
}
func NewLexerTokenStructTestSuite(t *testing.T) *LexerTokenStructTestSuite {
@@ -34,6 +37,14 @@ func (l *LexerTokenStructTestSuite) UsePHP8() {
l.Version = version.Version{Major: 8, Minor: 0}
}
func (l *LexerTokenStructTestSuite) WithPosition() {
l.withPosition = true
}
func (l *LexerTokenStructTestSuite) WithFreeFloating() {
l.withFreeFloating = true
}
func (l *LexerTokenStructTestSuite) Run() {
config := conf.Config{
Version: &l.Version,
@@ -49,8 +60,12 @@ func (l *LexerTokenStructTestSuite) Run() {
for _, expected := range l.Expected {
actual := lexer.Lex()
actual.Position = nil
actual.FreeFloating = nil
if !l.withPosition {
actual.Position = nil
}
if !l.withFreeFloating {
actual.FreeFloating = nil
}
assert.DeepEqual(l.t, expected, actual)
}
}