scanner test

This commit is contained in:
z7zmey 2018-02-17 12:53:10 +02:00
parent 55316625e8
commit b2d35d353d
3 changed files with 494 additions and 18 deletions

View File

@ -8398,8 +8398,7 @@ yyrule132: // ->
} }
yyrule133: // [ \t\n\r]+ yyrule133: // [ \t\n\r]+
{ {
lval.Token(l.newToken(l.Token())) lval.Token(l.newToken(l.Token())) // return T_WHITESPACE;
return T_WHITESPACE
goto yystate0 goto yystate0
} }
yyrule134: // -> yyrule134: // ->

View File

@ -143,15 +143,15 @@ NEW_LINE (\r|\n|\r\n)
<PHP>catch lval.Token(l.newToken(l.Token())); return T_CATCH <PHP>catch lval.Token(l.newToken(l.Token())); return T_CATCH
<PHP>class lval.Token(l.newToken(l.Token())); return T_CLASS <PHP>class lval.Token(l.newToken(l.Token())); return T_CLASS
<PHP>clone lval.Token(l.newToken(l.Token())); return T_CLONE <PHP>clone lval.Token(l.newToken(l.Token())); return T_CLONE
<PHP>const lval.Token(l.newToken(l.Token())); return T_CONST; <PHP>const lval.Token(l.newToken(l.Token())); return T_CONST
<PHP>continue lval.Token(l.newToken(l.Token())); return T_CONTINUE; <PHP>continue lval.Token(l.newToken(l.Token())); return T_CONTINUE
<PHP>declare lval.Token(l.newToken(l.Token())); return T_DECLARE; <PHP>declare lval.Token(l.newToken(l.Token())); return T_DECLARE
<PHP>default lval.Token(l.newToken(l.Token())); return T_DEFAULT; <PHP>default lval.Token(l.newToken(l.Token())); return T_DEFAULT
<PHP>do lval.Token(l.newToken(l.Token())); return T_DO; <PHP>do lval.Token(l.newToken(l.Token())); return T_DO
<PHP>echo lval.Token(l.newToken(l.Token())); return T_ECHO; <PHP>echo lval.Token(l.newToken(l.Token())); return T_ECHO
<PHP>else lval.Token(l.newToken(l.Token())); return T_ELSE; <PHP>else lval.Token(l.newToken(l.Token())); return T_ELSE
<PHP>elseif lval.Token(l.newToken(l.Token())); return T_ELSEIF; <PHP>elseif lval.Token(l.newToken(l.Token())); return T_ELSEIF
<PHP>empty lval.Token(l.newToken(l.Token())); return T_EMPTY; <PHP>empty lval.Token(l.newToken(l.Token())); return T_EMPTY
<PHP>enddeclare lval.Token(l.newToken(l.Token())); return T_ENDDECLARE <PHP>enddeclare lval.Token(l.newToken(l.Token())); return T_ENDDECLARE
<PHP>endfor lval.Token(l.newToken(l.Token())); return T_ENDFOR <PHP>endfor lval.Token(l.newToken(l.Token())); return T_ENDFOR
<PHP>endforeach lval.Token(l.newToken(l.Token())); return T_ENDFOREACH <PHP>endforeach lval.Token(l.newToken(l.Token())); return T_ENDFOREACH
@ -217,23 +217,23 @@ NEW_LINE (\r|\n|\r\n)
<PHP>or lval.Token(l.newToken(l.Token())); return T_LOGICAL_OR <PHP>or lval.Token(l.newToken(l.Token())); return T_LOGICAL_OR
<PHP>xor lval.Token(l.newToken(l.Token())); return T_LOGICAL_XOR <PHP>xor lval.Token(l.newToken(l.Token())); return T_LOGICAL_XOR
<PHP>\\ lval.Token(l.newToken(l.Token())); return T_NS_SEPARATOR <PHP>\\ lval.Token(l.newToken(l.Token())); return T_NS_SEPARATOR
<PHP>\.\.\. lval.Token(l.newToken(l.Token())); return T_ELLIPSIS; <PHP>\.\.\. lval.Token(l.newToken(l.Token())); return T_ELLIPSIS
<PHP>:: lval.Token(l.newToken(l.Token())); return T_PAAMAYIM_NEKUDOTAYIM; // T_DOUBLE_COLON <PHP>:: lval.Token(l.newToken(l.Token())); return T_PAAMAYIM_NEKUDOTAYIM // T_DOUBLE_COLON
<PHP>&& lval.Token(l.newToken(l.Token())); return T_BOOLEAN_AND <PHP>&& lval.Token(l.newToken(l.Token())); return T_BOOLEAN_AND
<PHP>\|\| lval.Token(l.newToken(l.Token())); return T_BOOLEAN_OR <PHP>\|\| lval.Token(l.newToken(l.Token())); return T_BOOLEAN_OR
<PHP>&= lval.Token(l.newToken(l.Token())); return T_AND_EQUAL <PHP>&= lval.Token(l.newToken(l.Token())); return T_AND_EQUAL
<PHP>\|= lval.Token(l.newToken(l.Token())); return T_OR_EQUAL <PHP>\|= lval.Token(l.newToken(l.Token())); return T_OR_EQUAL
<PHP>\.= lval.Token(l.newToken(l.Token())); return T_CONCAT_EQUAL; <PHP>\.= lval.Token(l.newToken(l.Token())); return T_CONCAT_EQUAL
<PHP>\*= lval.Token(l.newToken(l.Token())); return T_MUL_EQUAL <PHP>\*= lval.Token(l.newToken(l.Token())); return T_MUL_EQUAL
<PHP>\*\*= lval.Token(l.newToken(l.Token())); return T_POW_EQUAL <PHP>\*\*= lval.Token(l.newToken(l.Token())); return T_POW_EQUAL
<PHP>[/]= lval.Token(l.newToken(l.Token())); return T_DIV_EQUAL; <PHP>[/]= lval.Token(l.newToken(l.Token())); return T_DIV_EQUAL
<PHP>\+= lval.Token(l.newToken(l.Token())); return T_PLUS_EQUAL <PHP>\+= lval.Token(l.newToken(l.Token())); return T_PLUS_EQUAL
<PHP>-= lval.Token(l.newToken(l.Token())); return T_MINUS_EQUAL <PHP>-= lval.Token(l.newToken(l.Token())); return T_MINUS_EQUAL
<PHP>\^= lval.Token(l.newToken(l.Token())); return T_XOR_EQUAL <PHP>\^= lval.Token(l.newToken(l.Token())); return T_XOR_EQUAL
<PHP>%= lval.Token(l.newToken(l.Token())); return T_MOD_EQUAL <PHP>%= lval.Token(l.newToken(l.Token())); return T_MOD_EQUAL
<PHP>-- lval.Token(l.newToken(l.Token())); return T_DEC; <PHP>-- lval.Token(l.newToken(l.Token())); return T_DEC
<PHP>\+\+ lval.Token(l.newToken(l.Token())); return T_INC <PHP>\+\+ lval.Token(l.newToken(l.Token())); return T_INC
<PHP>=> lval.Token(l.newToken(l.Token())); return T_DOUBLE_ARROW; <PHP>=> lval.Token(l.newToken(l.Token())); return T_DOUBLE_ARROW
<PHP>\<=\> lval.Token(l.newToken(l.Token())); return T_SPACESHIP <PHP>\<=\> lval.Token(l.newToken(l.Token())); return T_SPACESHIP
<PHP>\!=|\<\> lval.Token(l.newToken(l.Token())); return T_IS_NOT_EQUAL <PHP>\!=|\<\> lval.Token(l.newToken(l.Token())); return T_IS_NOT_EQUAL
<PHP>\!== lval.Token(l.newToken(l.Token())); return T_IS_NOT_IDENTICAL <PHP>\!== lval.Token(l.newToken(l.Token())); return T_IS_NOT_IDENTICAL
@ -288,7 +288,7 @@ NEW_LINE (\r|\n|\r\n)
<PHP>{VAR_NAME} lval.Token(l.newToken(l.Token())); return T_STRING <PHP>{VAR_NAME} lval.Token(l.newToken(l.Token())); return T_STRING
<PHP>-> l.begin(PROPERTY);lval.Token(l.newToken(l.Token())); return T_OBJECT_OPERATOR; <PHP>-> l.begin(PROPERTY);lval.Token(l.newToken(l.Token())); return T_OBJECT_OPERATOR;
<PROPERTY>[ \t\n\r]+ lval.Token(l.newToken(l.Token())); return T_WHITESPACE; <PROPERTY>[ \t\n\r]+ lval.Token(l.newToken(l.Token())); // return T_WHITESPACE;
<PROPERTY>-> lval.Token(l.newToken(l.Token())); return T_OBJECT_OPERATOR; <PROPERTY>-> lval.Token(l.newToken(l.Token())); return T_OBJECT_OPERATOR;
<PROPERTY>{VAR_NAME} l.begin(PHP);lval.Token(l.newToken(l.Token())); return T_STRING; <PROPERTY>{VAR_NAME} l.begin(PHP);lval.Token(l.newToken(l.Token())); return T_STRING;
<PROPERTY>. l.ungetChars(1);l.begin(PHP) <PROPERTY>. l.ungetChars(1);l.begin(PHP)

477
scanner/scanner_test.go Normal file
View File

@ -0,0 +1,477 @@
package scanner_test
import (
"bytes"
"reflect"
"testing"
"github.com/z7zmey/php-parser/scanner"
"github.com/z7zmey/php-parser/token"
"github.com/kylelemons/godebug/pretty"
)
func assertEqual(t *testing.T, expected interface{}, actual interface{}) {
if !reflect.DeepEqual(expected, actual) {
diff := pretty.Compare(expected, actual)
if diff != "" {
t.Errorf("diff: (-expected +actual)\n%s", diff)
} else {
t.Errorf("expected and actual are not equal\n")
}
}
}
type lval struct {
token token.Token
}
func (lv *lval) Token(t token.Token) {
lv.token = t
}
func TestIdentifier(t *testing.T) {
src := `inline html -
<? ?>
<?= ?>
<?php
0.1
.1
1e10
.1e10
0b1
0b1111111111111111111111111111111111111111111111111111111111111111
0x007111111111111111
0x8111111111111111
1234567890123456789
12345678901234567890
abstract
array
as
break
callable
case
catch
class
clone
const
continue
declare
default
do
echo
else
elseif
empty
enddeclare
endfor
endforeach
endif
endswitch
endwhile
eval
exit
extends
final
finally
for
foreach
function
cfunction
global
goto
if
isset
implements
instanceof
insteadof
interface
list
namespace
private
public
print
protected
return
static
switch
throw
trait
try
unset
use
var
while
yield ` + "\t\r\n" + ` from
yield
include
include_once
require
require_once
__CLASS__
__DIR__
__FILE__
__FUNCTION__
__LINE__
__NAMESPACE__
__METHOD__
__TRAIT__
__halt_compiler
new
and
or
xor
\
...
::
&&
||
&=
|=
.=
*=
**=
/=
+=
-=
^=
%=
--
++
=>
<=>
!=
<>
!==
==
===
<<=
>>=
>=
<=
**
<<
>>
??
# inline comment
// inline comment
/*
multiline comment
*/
/**
* PHP Doc comment
*/
;
:
,
.
[
]
(
)
|
/
^
&
+
-
*
=
%
!
~
$
<
>
?
@
{
}
$var
str
-> ` + "\t\r\n" + ` ->prop
'adsf\'adsf\''
` + "`test $var {$var} ${var_name}`" + `
<<<CAT
test
CAT;
<<<'CAT'
test
CAT;
<<<"CAT"
$var->prop
$var[1]
$var[0x1]
$var[0b1]
$var[var_name]
$var[$var]
CAT;
`
expected := []int{
scanner.T_INLINE_HTML,
scanner.Rune2Class(';'),
scanner.T_ECHO,
scanner.Rune2Class(';'),
scanner.T_DNUMBER,
scanner.T_DNUMBER,
scanner.T_DNUMBER,
scanner.T_DNUMBER,
scanner.T_LNUMBER,
scanner.T_DNUMBER,
scanner.T_LNUMBER,
scanner.T_DNUMBER,
scanner.T_LNUMBER,
scanner.T_DNUMBER,
scanner.T_ABSTRACT,
scanner.T_ARRAY,
scanner.T_AS,
scanner.T_BREAK,
scanner.T_CALLABLE,
scanner.T_CASE,
scanner.T_CATCH,
scanner.T_CLASS,
scanner.T_CLONE,
scanner.T_CONST,
scanner.T_CONTINUE,
scanner.T_DECLARE,
scanner.T_DEFAULT,
scanner.T_DO,
scanner.T_ECHO,
scanner.T_ELSE,
scanner.T_ELSEIF,
scanner.T_EMPTY,
scanner.T_ENDDECLARE,
scanner.T_ENDFOR,
scanner.T_ENDFOREACH,
scanner.T_ENDIF,
scanner.T_ENDSWITCH,
scanner.T_ENDWHILE,
scanner.T_EVAL,
scanner.T_EXIT,
scanner.T_EXTENDS,
scanner.T_FINAL,
scanner.T_FINALLY,
scanner.T_FOR,
scanner.T_FOREACH,
scanner.T_FUNCTION,
scanner.T_FUNCTION,
scanner.T_GLOBAL,
scanner.T_GOTO,
scanner.T_IF,
scanner.T_ISSET,
scanner.T_IMPLEMENTS,
scanner.T_INSTANCEOF,
scanner.T_INSTEADOF,
scanner.T_INTERFACE,
scanner.T_LIST,
scanner.T_NAMESPACE,
scanner.T_PRIVATE,
scanner.T_PUBLIC,
scanner.T_PRINT,
scanner.T_PROTECTED,
scanner.T_RETURN,
scanner.T_STATIC,
scanner.T_SWITCH,
scanner.T_THROW,
scanner.T_TRAIT,
scanner.T_TRY,
scanner.T_UNSET,
scanner.T_USE,
scanner.T_VAR,
scanner.T_WHILE,
scanner.T_YIELD_FROM,
scanner.T_YIELD,
scanner.T_INCLUDE,
scanner.T_INCLUDE_ONCE,
scanner.T_REQUIRE,
scanner.T_REQUIRE_ONCE,
scanner.T_CLASS_C,
scanner.T_DIR,
scanner.T_FILE,
scanner.T_FUNC_C,
scanner.T_LINE,
scanner.T_NS_C,
scanner.T_METHOD_C,
scanner.T_TRAIT_C,
scanner.T_HALT_COMPILER,
scanner.T_NEW,
scanner.T_LOGICAL_AND,
scanner.T_LOGICAL_OR,
scanner.T_LOGICAL_XOR,
scanner.T_NS_SEPARATOR,
scanner.T_ELLIPSIS,
scanner.T_PAAMAYIM_NEKUDOTAYIM,
scanner.T_BOOLEAN_AND,
scanner.T_BOOLEAN_OR,
scanner.T_AND_EQUAL,
scanner.T_OR_EQUAL,
scanner.T_CONCAT_EQUAL,
scanner.T_MUL_EQUAL,
scanner.T_POW_EQUAL,
scanner.T_DIV_EQUAL,
scanner.T_PLUS_EQUAL,
scanner.T_MINUS_EQUAL,
scanner.T_XOR_EQUAL,
scanner.T_MOD_EQUAL,
scanner.T_DEC,
scanner.T_INC,
scanner.T_DOUBLE_ARROW,
scanner.T_SPACESHIP,
scanner.T_IS_NOT_EQUAL,
scanner.T_IS_NOT_EQUAL,
scanner.T_IS_NOT_IDENTICAL,
scanner.T_IS_EQUAL,
scanner.T_IS_IDENTICAL,
scanner.T_SL_EQUAL,
scanner.T_SR_EQUAL,
scanner.T_IS_GREATER_OR_EQUAL,
scanner.T_IS_SMALLER_OR_EQUAL,
scanner.T_POW,
scanner.T_SL,
scanner.T_SR,
scanner.T_COALESCE,
scanner.Rune2Class(';'),
scanner.Rune2Class(':'),
scanner.Rune2Class(','),
scanner.Rune2Class('.'),
scanner.Rune2Class('['),
scanner.Rune2Class(']'),
scanner.Rune2Class('('),
scanner.Rune2Class(')'),
scanner.Rune2Class('|'),
scanner.Rune2Class('/'),
scanner.Rune2Class('^'),
scanner.Rune2Class('&'),
scanner.Rune2Class('+'),
scanner.Rune2Class('-'),
scanner.Rune2Class('*'),
scanner.Rune2Class('='),
scanner.Rune2Class('%'),
scanner.Rune2Class('!'),
scanner.Rune2Class('~'),
scanner.Rune2Class('$'),
scanner.Rune2Class('<'),
scanner.Rune2Class('>'),
scanner.Rune2Class('?'),
scanner.Rune2Class('@'),
scanner.Rune2Class('{'),
scanner.Rune2Class('}'),
scanner.T_VARIABLE,
scanner.T_STRING,
scanner.T_OBJECT_OPERATOR,
scanner.T_OBJECT_OPERATOR,
scanner.T_STRING,
scanner.T_CONSTANT_ENCAPSED_STRING,
scanner.Rune2Class('`'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_CURLY_OPEN,
scanner.T_VARIABLE,
scanner.Rune2Class('}'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_DOLLAR_OPEN_CURLY_BRACES,
scanner.T_STRING_VARNAME,
scanner.Rune2Class('}'),
scanner.Rune2Class('`'),
scanner.T_START_HEREDOC,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_END_HEREDOC,
scanner.Rune2Class(';'),
scanner.T_START_HEREDOC,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_END_HEREDOC,
scanner.Rune2Class(';'),
scanner.T_START_HEREDOC,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.T_OBJECT_OPERATOR,
scanner.T_STRING,
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.Rune2Class('['),
scanner.T_NUM_STRING,
scanner.Rune2Class(']'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.Rune2Class('['),
scanner.T_NUM_STRING,
scanner.Rune2Class(']'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.Rune2Class('['),
scanner.T_NUM_STRING,
scanner.Rune2Class(']'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.Rune2Class('['),
scanner.T_STRING,
scanner.Rune2Class(']'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_VARIABLE,
scanner.Rune2Class('['),
scanner.T_VARIABLE,
scanner.Rune2Class(']'),
scanner.T_ENCAPSED_AND_WHITESPACE,
scanner.T_END_HEREDOC,
scanner.Rune2Class(';'),
}
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
lv := &lval{}
actual := []int{}
for {
token := lexer.Lex(lv)
if token < 0 {
break
}
actual = append(actual, token)
}
assertEqual(t, expected, actual)
}