maride/php-parser
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: move php7 scanner into php7 package

Browse Source
This commit is contained in:
Laytan Laats 2023-03-25 15:02:34 +01:00
parent a97686125d
commit 74a8771740
17 changed files with 260 additions and 266 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/scanner/lexer.go → internal/php7/lexer.go
View File

@ -1,4 +1,4 @@
package scanner package php7
import ( import (
"bytes" "bytes"

2
internal/scanner/newline.go → internal/php7/newline.go
View File

@ -1,4 +1,4 @@
package scanner package php7
type NewLines struct { type NewLines struct {
data []int data []int

5
internal/php7/parser.go
View File

@ -2,7 +2,6 @@ package php7
import ( import (
"github.com/VKCOM/php-parser/internal/position" "github.com/VKCOM/php-parser/internal/position"
"github.com/VKCOM/php-parser/internal/scanner"
"github.com/VKCOM/php-parser/pkg/ast" "github.com/VKCOM/php-parser/pkg/ast"
"github.com/VKCOM/php-parser/pkg/conf" "github.com/VKCOM/php-parser/pkg/conf"
"github.com/VKCOM/php-parser/pkg/errors" "github.com/VKCOM/php-parser/pkg/errors"
@ -11,7 +10,7 @@ import (
// Parser structure // Parser structure
type Parser struct { type Parser struct {
Lexer *scanner.Lexer Lexer *Lexer
currentToken *token.Token currentToken *token.Token
rootNode ast.Vertex rootNode ast.Vertex
errHandlerFunc func(*errors.Error) errHandlerFunc func(*errors.Error)
@ -19,7 +18,7 @@ type Parser struct {
} }
// NewParser creates and returns new Parser // NewParser creates and returns new Parser
func NewParser(lexer *scanner.Lexer, config conf.Config) *Parser { func NewParser(lexer *Lexer, config conf.Config) *Parser {
return &Parser{ return &Parser{
Lexer: lexer, Lexer: lexer,
errHandlerFunc: config.ErrorHandlerFunc, errHandlerFunc: config.ErrorHandlerFunc,

465
internal/php7/parser_test.go
View File

File diff suppressed because it is too large Load Diff

3
internal/php7/php7_bench_test.go
View File

@ -6,7 +6,6 @@ import (
"testing" "testing"
"github.com/VKCOM/php-parser/internal/php7" "github.com/VKCOM/php-parser/internal/php7"
"github.com/VKCOM/php-parser/internal/scanner"
"github.com/VKCOM/php-parser/pkg/conf" "github.com/VKCOM/php-parser/pkg/conf"
"github.com/VKCOM/php-parser/pkg/version" "github.com/VKCOM/php-parser/pkg/version"
) )
@ -24,7 +23,7 @@ func BenchmarkPhp7(b *testing.B) {
Minor: 4, Minor: 4,
}, },
} }
lexer := scanner.NewLexer(src, config) lexer := php7.NewLexer(src, config)
php7parser := php7.NewParser(lexer, config) php7parser := php7.NewParser(lexer, config)
php7parser.Parse() php7parser.Parse()
} }

BIN
internal/scanner/scanner.go → internal/php7/scanner.go
View File

Binary file not shown.

4
internal/scanner/scanner.rl → internal/php7/scanner.rl
View File

@ -1,4 +1,4 @@
package scanner package php7
import ( import (
"fmt" "fmt"
@ -500,4 +500,4 @@ func (lex *Lexer) Lex() *token.Token {
tkn.ID = token.ID(tok) tkn.ID = token.ID(tok)
return tkn return tkn
} }

2
internal/scanner/scanner_test.go → internal/php7/scanner_test.go
View File

@ -1,4 +1,4 @@
package scanner package php7
import ( import (
"gotest.tools/assert" "gotest.tools/assert"

9
internal/php8/lexer.go
View File

@ -124,11 +124,13 @@ func (lex *Lexer) isHeredocEndBefore73(p int) bool {
return false return false
} }
if len(lex.data) > p+l && lex.data[p+l] != ';' && lex.data[p+l] != '\r' && lex.data[p+l] != '\n' { if len(lex.data) > p+l && lex.data[p+l] != ';' && lex.data[p+l] != '\r' &&
lex.data[p+l] != '\n' {
return false return false
} }
if len(lex.data) > p+l+1 && lex.data[p+l] == ';' && lex.data[p+l+1] != '\r' && lex.data[p+l+1] != '\n' { if len(lex.data) > p+l+1 && lex.data[p+l] == ';' && lex.data[p+l+1] != '\r' &&
lex.data[p+l+1] != '\n' {
return false return false
} }
@ -262,5 +264,6 @@ func isValidVarNameStart(r byte) bool {
} }
func isValidVarName(r byte) bool { func isValidVarName(r byte) bool {
return (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' || r >= 0x80 return (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' ||
r >= 0x80
} }

4
internal/php8/parser_php8_test.go
View File

@ -4177,7 +4177,7 @@ class Point {
}, },
}, },
AmpersandTkn: &token.Token{ AmpersandTkn: &token.Token{
ID: token.ID(57492), ID: token.T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG,
Val: []byte("&"), Val: []byte("&"),
FreeFloating: []*token.Token{ FreeFloating: []*token.Token{
{ {
@ -4302,7 +4302,7 @@ class Point {
}, },
}, },
AmpersandTkn: &token.Token{ AmpersandTkn: &token.Token{
ID: token.ID(57492), ID: token.T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG,
Val: []byte("&"), Val: []byte("&"),
FreeFloating: []*token.Token{ FreeFloating: []*token.Token{
{ {

2
internal/php8/scanner.rl
View File

@ -539,4 +539,4 @@ func (lex *Lexer) Lex() *token.Token {
tkn.ID = token.ID(tok) tkn.ID = token.ID(tok)
return tkn return tkn
} }

2
internal/php8/scanner_test.go
View File

@ -317,7 +317,7 @@ func TestTokens(t *testing.T) {
token.ID(int('|')).String(), token.ID(int('|')).String(),
token.ID(int('/')).String(), token.ID(int('/')).String(),
token.ID(int('^')).String(), token.ID(int('^')).String(),
token.ID(T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG).String(), token.ID(token.T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG).String(),
token.ID(int('+')).String(), token.ID(int('+')).String(),
token.ID(int('-')).String(), token.ID(int('-')).String(),
token.ID(int('*')).String(), token.ID(int('*')).String(),

4
internal/tester/lexer_token_freefloating.go
View File

@ -3,8 +3,8 @@ package tester
import ( import (
"testing" "testing"
"github.com/VKCOM/php-parser/internal/php7"
"github.com/VKCOM/php-parser/internal/php8" "github.com/VKCOM/php-parser/internal/php8"
"github.com/VKCOM/php-parser/internal/scanner"
"github.com/VKCOM/php-parser/pkg/conf" "github.com/VKCOM/php-parser/pkg/conf"
"github.com/VKCOM/php-parser/pkg/token" "github.com/VKCOM/php-parser/pkg/token"
"github.com/VKCOM/php-parser/pkg/version" "github.com/VKCOM/php-parser/pkg/version"
@ -46,7 +46,7 @@ func (l *LexerTokenFreeFloatingTestSuite) Run() {
var lexer Lexer var lexer Lexer
if l.Version.Less(&version.Version{Major: 8, Minor: 0}) { if l.Version.Less(&version.Version{Major: 8, Minor: 0}) {
lexer = scanner.NewLexer([]byte(l.Code), config) lexer = php7.NewLexer([]byte(l.Code), config)
} else { } else {
lexer = php8.NewLexer([]byte(l.Code), config) lexer = php8.NewLexer([]byte(l.Code), config)
} }

4
internal/tester/lexer_token_string.go
View File

@ -4,7 +4,7 @@ import (
"testing" "testing"
"github.com/VKCOM/php-parser/internal/php8" "github.com/VKCOM/php-parser/internal/php8"
"github.com/VKCOM/php-parser/internal/scanner" "github.com/VKCOM/php-parser/internal/php7"
"github.com/VKCOM/php-parser/pkg/conf" "github.com/VKCOM/php-parser/pkg/conf"
"github.com/VKCOM/php-parser/pkg/version" "github.com/VKCOM/php-parser/pkg/version"
"gotest.tools/assert" "gotest.tools/assert"
@ -41,7 +41,7 @@ func (l *LexerTokenStringTestSuite) Run() {
var lexer Lexer var lexer Lexer
if l.Version.Less(&version.Version{Major: 8, Minor: 0}) { if l.Version.Less(&version.Version{Major: 8, Minor: 0}) {
lexer = scanner.NewLexer([]byte(l.Code), config) lexer = php7.NewLexer([]byte(l.Code), config)
} else { } else {
lexer = php8.NewLexer([]byte(l.Code), config) lexer = php8.NewLexer([]byte(l.Code), config)
} }

4
internal/tester/lexer_token_struct.go
View File

@ -4,7 +4,7 @@ import (
"testing" "testing"
"github.com/VKCOM/php-parser/internal/php8" "github.com/VKCOM/php-parser/internal/php8"
"github.com/VKCOM/php-parser/internal/scanner" "github.com/VKCOM/php-parser/internal/php7"
"github.com/VKCOM/php-parser/pkg/conf" "github.com/VKCOM/php-parser/pkg/conf"
"github.com/VKCOM/php-parser/pkg/token" "github.com/VKCOM/php-parser/pkg/token"
"github.com/VKCOM/php-parser/pkg/version" "github.com/VKCOM/php-parser/pkg/version"
@ -53,7 +53,7 @@ func (l *LexerTokenStructTestSuite) Run() {
var lexer Lexer var lexer Lexer
if l.Version.Less(&version.Version{Major: 8, Minor: 0}) { if l.Version.Less(&version.Version{Major: 8, Minor: 0}) {
lexer = scanner.NewLexer([]byte(l.Code), config) lexer = php7.NewLexer([]byte(l.Code), config)
} else { } else {
lexer = php8.NewLexer([]byte(l.Code), config) lexer = php8.NewLexer([]byte(l.Code), config)
} }

2
internal/tester/parser_dump.go
View File

@ -49,6 +49,8 @@ func (p *ParserDumpTestSuite) UsePHP8() {
} }
func (p *ParserDumpTestSuite) Run() { func (p *ParserDumpTestSuite) Run() {
p.t.Helper()
config := conf.Config{ config := conf.Config{
Version: &p.Version, Version: &p.Version,
} }

12
pkg/visitor/printer/printer_php7_test.go
View File

@ -6,7 +6,6 @@ import (
"testing" "testing"
"github.com/VKCOM/php-parser/internal/php7" "github.com/VKCOM/php-parser/internal/php7"
"github.com/VKCOM/php-parser/internal/scanner"
"github.com/VKCOM/php-parser/pkg/ast" "github.com/VKCOM/php-parser/pkg/ast"
"github.com/VKCOM/php-parser/pkg/conf" "github.com/VKCOM/php-parser/pkg/conf"
"github.com/VKCOM/php-parser/pkg/version" "github.com/VKCOM/php-parser/pkg/version"
@ -36,7 +35,7 @@ abstract class Bar extends Baz
Minor: 4, Minor: 4,
}, },
} }
lexer := scanner.NewLexer([]byte(src), config) lexer := php7.NewLexer([]byte(src), config)
php7parser := php7.NewParser(lexer, config) php7parser := php7.NewParser(lexer, config)
php7parser.Parse() php7parser.Parse()
@ -74,7 +73,7 @@ func parse(src string) ast.Vertex {
Minor: 4, Minor: 4,
}, },
} }
lexer := scanner.NewLexer([]byte(src), config) lexer := php7.NewLexer([]byte(src), config)
php7parser := php7.NewParser(lexer, config) php7parser := php7.NewParser(lexer, config)
php7parser.Parse() php7parser.Parse()
@ -93,7 +92,6 @@ func print(n ast.Vertex) string {
// test node // test node
func TestParseAndPrintRoot(t *testing.T) { func TestParseAndPrintRoot(t *testing.T) {
src := ` <div>Hello</div> src := ` <div>Hello</div>
<?php <?php
$a; $a;
@ -120,7 +118,6 @@ func TestParseAndPrintIdentifier(t *testing.T) {
} }
func TestParseAndPrintParameterTMP(t *testing.T) { func TestParseAndPrintParameterTMP(t *testing.T) {
src := `<?php src := `<?php
function foo ( foo & ... $foo = null ) {}` function foo ( foo & ... $foo = null ) {}`
@ -132,7 +129,6 @@ func TestParseAndPrintParameterTMP(t *testing.T) {
} }
func TestParseAndPrintParameter(t *testing.T) { func TestParseAndPrintParameter(t *testing.T) {
src := `<?php src := `<?php
function & foo ( function & foo (
? int $a , & $b = null ? int $a , & $b = null
@ -149,7 +145,6 @@ func TestParseAndPrintParameter(t *testing.T) {
} }
func TestParseAndPrintNullable(t *testing.T) { func TestParseAndPrintNullable(t *testing.T) {
src := `<?php src := `<?php
function & foo ( ? int $a ) { function & foo ( ? int $a ) {
/* do nothing */ /* do nothing */
@ -599,7 +594,6 @@ func TestParseAndPrintFunctionCall(t *testing.T) {
} }
func TestParseAndPrintInclude(t *testing.T) { func TestParseAndPrintInclude(t *testing.T) {
src := `<?php src := `<?php
include 'foo' ; include 'foo' ;
include_once 'bar' ;` include_once 'bar' ;`
@ -732,7 +726,6 @@ func TestParseAndPrintReference(t *testing.T) {
} }
func TestParseAndPrintRequire(t *testing.T) { func TestParseAndPrintRequire(t *testing.T) {
src := `<?php src := `<?php
require __DIR__ . '/folder' ; require __DIR__ . '/folder' ;
require_once $a ;` require_once $a ;`
@ -1134,7 +1127,6 @@ func TestParseAndPrintForeach(t *testing.T) {
} }
func TestParseAndPrintFunction(t *testing.T) { func TestParseAndPrintFunction(t *testing.T) {
src := `<?php src := `<?php
function & foo ( ) : void { function & foo ( ) : void {
; ;