d85f5a4816
Example from issue:
```php
<?php
#
# Comment
#
$a = 100;
```
The problem with the example from the issue is that `#` is immediately followed by a line break.
And since the rule in the lexer for such comments was changed, this case was handled incorrectly.
```
(('#' ^'[') | '//') any_line* when is_not_comment_end => {
lex.ungetStr("?>")
lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
};
```
This rule has one problem, it checks two characters at once, first for the match `#`, and
then for the mismatch `[`, which leads to the fact that in the case of an empty comment, the first
matcher will capture `#`, and the second line break (`\n`), which will lead to the fact that `any_line`
matcher will not work and will not increase the line number.
The next rule added is specifically for this case.
```
'#' newline when is_not_comment_end => {
lex.ungetStr("?>")
lex.addFreeFloatingToken(tkn, token.T_COMMENT, lex.ts, lex.te)
};
```
72 lines
1.4 KiB
Go
72 lines
1.4 KiB
Go
package tester
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/VKCOM/php-parser/internal/php8"
|
|
"github.com/VKCOM/php-parser/internal/scanner"
|
|
"github.com/VKCOM/php-parser/pkg/conf"
|
|
"github.com/VKCOM/php-parser/pkg/token"
|
|
"github.com/VKCOM/php-parser/pkg/version"
|
|
"gotest.tools/assert"
|
|
)
|
|
|
|
type LexerTokenStructTestSuite struct {
|
|
t *testing.T
|
|
|
|
Code string
|
|
Expected []*token.Token
|
|
|
|
Version version.Version
|
|
|
|
withPosition bool
|
|
withFreeFloating bool
|
|
}
|
|
|
|
func NewLexerTokenStructTestSuite(t *testing.T) *LexerTokenStructTestSuite {
|
|
return &LexerTokenStructTestSuite{
|
|
t: t,
|
|
Version: version.Version{
|
|
Major: 7,
|
|
Minor: 4,
|
|
},
|
|
}
|
|
}
|
|
|
|
func (l *LexerTokenStructTestSuite) UsePHP8() {
|
|
l.Version = version.Version{Major: 8, Minor: 0}
|
|
}
|
|
|
|
func (l *LexerTokenStructTestSuite) WithPosition() {
|
|
l.withPosition = true
|
|
}
|
|
|
|
func (l *LexerTokenStructTestSuite) WithFreeFloating() {
|
|
l.withFreeFloating = true
|
|
}
|
|
|
|
func (l *LexerTokenStructTestSuite) Run() {
|
|
config := conf.Config{
|
|
Version: &l.Version,
|
|
}
|
|
|
|
var lexer Lexer
|
|
|
|
if l.Version.Less(&version.Version{Major: 8, Minor: 0}) {
|
|
lexer = scanner.NewLexer([]byte(l.Code), config)
|
|
} else {
|
|
lexer = php8.NewLexer([]byte(l.Code), config)
|
|
}
|
|
|
|
for _, expected := range l.Expected {
|
|
actual := lexer.Lex()
|
|
if !l.withPosition {
|
|
actual.Position = nil
|
|
}
|
|
if !l.withFreeFloating {
|
|
actual.FreeFloating = nil
|
|
}
|
|
assert.DeepEqual(l.t, expected, actual)
|
|
}
|
|
}
|