scanner test
This commit is contained in:
z7zmey
parent
55316625e8
commit
b2d35d353d
|
|
@ -8398,8 +8398,7 @@ yyrule132: // ->
|
|||
}
|
||||
yyrule133: // [ \t\n\r]+
|
||||
{
|
||||
lval.Token(l.newToken(l.Token()))
|
||||
return T_WHITESPACE
|
||||
lval.Token(l.newToken(l.Token())) // return T_WHITESPACE;
|
||||
goto yystate0
|
||||
}
|
||||
yyrule134: // ->
|
||||
|
|
|
|||
|
|
@ -143,15 +143,15 @@ NEW_LINE (\r|\n|\r\n)
|
|||
<PHP>catch lval.Token(l.newToken(l.Token())); return T_CATCH
|
||||
<PHP>class lval.Token(l.newToken(l.Token())); return T_CLASS
|
||||
<PHP>clone lval.Token(l.newToken(l.Token())); return T_CLONE
|
||||
<PHP>const lval.Token(l.newToken(l.Token())); return T_CONST;
|
||||
<PHP>continue lval.Token(l.newToken(l.Token())); return T_CONTINUE;
|
||||
<PHP>declare lval.Token(l.newToken(l.Token())); return T_DECLARE;
|
||||
<PHP>default lval.Token(l.newToken(l.Token())); return T_DEFAULT;
|
||||
<PHP>do lval.Token(l.newToken(l.Token())); return T_DO;
|
||||
<PHP>echo lval.Token(l.newToken(l.Token())); return T_ECHO;
|
||||
<PHP>else lval.Token(l.newToken(l.Token())); return T_ELSE;
|
||||
<PHP>elseif lval.Token(l.newToken(l.Token())); return T_ELSEIF;
|
||||
<PHP>empty lval.Token(l.newToken(l.Token())); return T_EMPTY;
|
||||
<PHP>const lval.Token(l.newToken(l.Token())); return T_CONST
|
||||
<PHP>continue lval.Token(l.newToken(l.Token())); return T_CONTINUE
|
||||
<PHP>declare lval.Token(l.newToken(l.Token())); return T_DECLARE
|
||||
<PHP>default lval.Token(l.newToken(l.Token())); return T_DEFAULT
|
||||
<PHP>do lval.Token(l.newToken(l.Token())); return T_DO
|
||||
<PHP>echo lval.Token(l.newToken(l.Token())); return T_ECHO
|
||||
<PHP>else lval.Token(l.newToken(l.Token())); return T_ELSE
|
||||
<PHP>elseif lval.Token(l.newToken(l.Token())); return T_ELSEIF
|
||||
<PHP>empty lval.Token(l.newToken(l.Token())); return T_EMPTY
|
||||
<PHP>enddeclare lval.Token(l.newToken(l.Token())); return T_ENDDECLARE
|
||||
<PHP>endfor lval.Token(l.newToken(l.Token())); return T_ENDFOR
|
||||
<PHP>endforeach lval.Token(l.newToken(l.Token())); return T_ENDFOREACH
|
||||
|
|
@ -217,23 +217,23 @@ NEW_LINE (\r|\n|\r\n)
|
|||
<PHP>or lval.Token(l.newToken(l.Token())); return T_LOGICAL_OR
|
||||
<PHP>xor lval.Token(l.newToken(l.Token())); return T_LOGICAL_XOR
|
||||
<PHP>\\ lval.Token(l.newToken(l.Token())); return T_NS_SEPARATOR
|
||||
<PHP>\.\.\. lval.Token(l.newToken(l.Token())); return T_ELLIPSIS;
|
||||
<PHP>:: lval.Token(l.newToken(l.Token())); return T_PAAMAYIM_NEKUDOTAYIM; // T_DOUBLE_COLON
|
||||
<PHP>\.\.\. lval.Token(l.newToken(l.Token())); return T_ELLIPSIS
|
||||
<PHP>:: lval.Token(l.newToken(l.Token())); return T_PAAMAYIM_NEKUDOTAYIM // T_DOUBLE_COLON
|
||||
<PHP>&& lval.Token(l.newToken(l.Token())); return T_BOOLEAN_AND
|
||||
<PHP>\|\| lval.Token(l.newToken(l.Token())); return T_BOOLEAN_OR
|
||||
<PHP>&= lval.Token(l.newToken(l.Token())); return T_AND_EQUAL
|
||||
<PHP>\|= lval.Token(l.newToken(l.Token())); return T_OR_EQUAL
|
||||
<PHP>\.= lval.Token(l.newToken(l.Token())); return T_CONCAT_EQUAL;
|
||||
<PHP>\.= lval.Token(l.newToken(l.Token())); return T_CONCAT_EQUAL
|
||||
<PHP>\*= lval.Token(l.newToken(l.Token())); return T_MUL_EQUAL
|
||||
<PHP>\*\*= lval.Token(l.newToken(l.Token())); return T_POW_EQUAL
|
||||
<PHP>[/]= lval.Token(l.newToken(l.Token())); return T_DIV_EQUAL;
|
||||
<PHP>[/]= lval.Token(l.newToken(l.Token())); return T_DIV_EQUAL
|
||||
<PHP>\+= lval.Token(l.newToken(l.Token())); return T_PLUS_EQUAL
|
||||
<PHP>-= lval.Token(l.newToken(l.Token())); return T_MINUS_EQUAL
|
||||
<PHP>\^= lval.Token(l.newToken(l.Token())); return T_XOR_EQUAL
|
||||
<PHP>%= lval.Token(l.newToken(l.Token())); return T_MOD_EQUAL
|
||||
<PHP>-- lval.Token(l.newToken(l.Token())); return T_DEC;
|
||||
<PHP>-- lval.Token(l.newToken(l.Token())); return T_DEC
|
||||
<PHP>\+\+ lval.Token(l.newToken(l.Token())); return T_INC
|
||||
<PHP>=> lval.Token(l.newToken(l.Token())); return T_DOUBLE_ARROW;
|
||||
<PHP>=> lval.Token(l.newToken(l.Token())); return T_DOUBLE_ARROW
|
||||
<PHP>\<=\> lval.Token(l.newToken(l.Token())); return T_SPACESHIP
|
||||
<PHP>\!=|\<\> lval.Token(l.newToken(l.Token())); return T_IS_NOT_EQUAL
|
||||
<PHP>\!== lval.Token(l.newToken(l.Token())); return T_IS_NOT_IDENTICAL
|
||||
|
|
@ -288,7 +288,7 @@ NEW_LINE (\r|\n|\r\n)
|
|||
<PHP>{VAR_NAME} lval.Token(l.newToken(l.Token())); return T_STRING
|
||||
|
||||
<PHP>-> l.begin(PROPERTY);lval.Token(l.newToken(l.Token())); return T_OBJECT_OPERATOR;
|
||||
<PROPERTY>[ \t\n\r]+ lval.Token(l.newToken(l.Token())); return T_WHITESPACE;
|
||||
<PROPERTY>[ \t\n\r]+ lval.Token(l.newToken(l.Token())); // return T_WHITESPACE;
|
||||
<PROPERTY>-> lval.Token(l.newToken(l.Token())); return T_OBJECT_OPERATOR;
|
||||
<PROPERTY>{VAR_NAME} l.begin(PHP);lval.Token(l.newToken(l.Token())); return T_STRING;
|
||||
<PROPERTY>. l.ungetChars(1);l.begin(PHP)
|
||||
|
|
|
|||
477
scanner/scanner_test.go
Normal file
477
scanner/scanner_test.go
Normal file
|
|
@ -0,0 +1,477 @@
|
|||
package scanner_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/z7zmey/php-parser/scanner"
|
||||
"github.com/z7zmey/php-parser/token"
|
||||
|
||||
"github.com/kylelemons/godebug/pretty"
|
||||
)
|
||||
|
||||
func assertEqual(t *testing.T, expected interface{}, actual interface{}) {
|
||||
if !reflect.DeepEqual(expected, actual) {
|
||||
diff := pretty.Compare(expected, actual)
|
||||
|
||||
if diff != "" {
|
||||
t.Errorf("diff: (-expected +actual)\n%s", diff)
|
||||
} else {
|
||||
t.Errorf("expected and actual are not equal\n")
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
type lval struct {
|
||||
token token.Token
|
||||
}
|
||||
|
||||
func (lv *lval) Token(t token.Token) {
|
||||
lv.token = t
|
||||
}
|
||||
|
||||
func TestIdentifier(t *testing.T) {
|
||||
src := `inline html -
|
||||
<? ?>
|
||||
<?= ?>
|
||||
<?php
|
||||
|
||||
0.1
|
||||
.1
|
||||
1e10
|
||||
.1e10
|
||||
|
||||
0b1
|
||||
0b1111111111111111111111111111111111111111111111111111111111111111
|
||||
|
||||
0x007111111111111111
|
||||
0x8111111111111111
|
||||
|
||||
1234567890123456789
|
||||
12345678901234567890
|
||||
|
||||
abstract
|
||||
array
|
||||
as
|
||||
break
|
||||
callable
|
||||
case
|
||||
catch
|
||||
class
|
||||
clone
|
||||
const
|
||||
continue
|
||||
declare
|
||||
default
|
||||
do
|
||||
echo
|
||||
else
|
||||
elseif
|
||||
empty
|
||||
enddeclare
|
||||
endfor
|
||||
endforeach
|
||||
endif
|
||||
endswitch
|
||||
endwhile
|
||||
eval
|
||||
exit
|
||||
extends
|
||||
final
|
||||
finally
|
||||
for
|
||||
foreach
|
||||
function
|
||||
cfunction
|
||||
global
|
||||
goto
|
||||
if
|
||||
isset
|
||||
implements
|
||||
instanceof
|
||||
insteadof
|
||||
interface
|
||||
list
|
||||
namespace
|
||||
private
|
||||
public
|
||||
print
|
||||
protected
|
||||
return
|
||||
static
|
||||
switch
|
||||
throw
|
||||
trait
|
||||
try
|
||||
unset
|
||||
use
|
||||
var
|
||||
while
|
||||
yield ` + "\t\r\n" + ` from
|
||||
yield
|
||||
include
|
||||
include_once
|
||||
require
|
||||
require_once
|
||||
|
||||
__CLASS__
|
||||
__DIR__
|
||||
__FILE__
|
||||
__FUNCTION__
|
||||
__LINE__
|
||||
__NAMESPACE__
|
||||
__METHOD__
|
||||
__TRAIT__
|
||||
__halt_compiler
|
||||
|
||||
new
|
||||
and
|
||||
or
|
||||
xor
|
||||
|
||||
\
|
||||
...
|
||||
::
|
||||
&&
|
||||
||
|
||||
&=
|
||||
|=
|
||||
.=
|
||||
*=
|
||||
**=
|
||||
/=
|
||||
+=
|
||||
-=
|
||||
^=
|
||||
%=
|
||||
--
|
||||
++
|
||||
=>
|
||||
<=>
|
||||
!=
|
||||
<>
|
||||
!==
|
||||
==
|
||||
===
|
||||
<<=
|
||||
>>=
|
||||
>=
|
||||
<=
|
||||
**
|
||||
<<
|
||||
>>
|
||||
??
|
||||
|
||||
# inline comment
|
||||
// inline comment
|
||||
|
||||
/*
|
||||
multiline comment
|
||||
*/
|
||||
|
||||
/**
|
||||
* PHP Doc comment
|
||||
*/
|
||||
|
||||
;
|
||||
:
|
||||
,
|
||||
.
|
||||
[
|
||||
]
|
||||
(
|
||||
)
|
||||
|
|
||||
/
|
||||
^
|
||||
&
|
||||
+
|
||||
-
|
||||
*
|
||||
=
|
||||
%
|
||||
!
|
||||
~
|
||||
$
|
||||
<
|
||||
>
|
||||
?
|
||||
@
|
||||
{
|
||||
}
|
||||
|
||||
$var
|
||||
str
|
||||
|
||||
-> ` + "\t\r\n" + ` ->prop
|
||||
|
||||
'adsf\'adsf\''
|
||||
|
||||
` + "`test $var {$var} ${var_name}`" + `
|
||||
|
||||
<<<CAT
|
||||
test
|
||||
CAT;
|
||||
<<<'CAT'
|
||||
test
|
||||
CAT;
|
||||
|
||||
<<<"CAT"
|
||||
$var->prop
|
||||
$var[1]
|
||||
$var[0x1]
|
||||
$var[0b1]
|
||||
$var[var_name]
|
||||
$var[$var]
|
||||
CAT;
|
||||
|
||||
`
|
||||
|
||||
expected := []int{
|
||||
scanner.T_INLINE_HTML,
|
||||
scanner.Rune2Class(';'),
|
||||
scanner.T_ECHO,
|
||||
scanner.Rune2Class(';'),
|
||||
|
||||
scanner.T_DNUMBER,
|
||||
scanner.T_DNUMBER,
|
||||
scanner.T_DNUMBER,
|
||||
scanner.T_DNUMBER,
|
||||
|
||||
scanner.T_LNUMBER,
|
||||
scanner.T_DNUMBER,
|
||||
|
||||
scanner.T_LNUMBER,
|
||||
scanner.T_DNUMBER,
|
||||
|
||||
scanner.T_LNUMBER,
|
||||
scanner.T_DNUMBER,
|
||||
|
||||
scanner.T_ABSTRACT,
|
||||
scanner.T_ARRAY,
|
||||
scanner.T_AS,
|
||||
scanner.T_BREAK,
|
||||
scanner.T_CALLABLE,
|
||||
scanner.T_CASE,
|
||||
scanner.T_CATCH,
|
||||
scanner.T_CLASS,
|
||||
scanner.T_CLONE,
|
||||
scanner.T_CONST,
|
||||
scanner.T_CONTINUE,
|
||||
scanner.T_DECLARE,
|
||||
scanner.T_DEFAULT,
|
||||
scanner.T_DO,
|
||||
scanner.T_ECHO,
|
||||
scanner.T_ELSE,
|
||||
scanner.T_ELSEIF,
|
||||
scanner.T_EMPTY,
|
||||
scanner.T_ENDDECLARE,
|
||||
scanner.T_ENDFOR,
|
||||
scanner.T_ENDFOREACH,
|
||||
scanner.T_ENDIF,
|
||||
scanner.T_ENDSWITCH,
|
||||
scanner.T_ENDWHILE,
|
||||
scanner.T_EVAL,
|
||||
scanner.T_EXIT,
|
||||
scanner.T_EXTENDS,
|
||||
scanner.T_FINAL,
|
||||
scanner.T_FINALLY,
|
||||
scanner.T_FOR,
|
||||
scanner.T_FOREACH,
|
||||
scanner.T_FUNCTION,
|
||||
scanner.T_FUNCTION,
|
||||
scanner.T_GLOBAL,
|
||||
scanner.T_GOTO,
|
||||
scanner.T_IF,
|
||||
scanner.T_ISSET,
|
||||
scanner.T_IMPLEMENTS,
|
||||
scanner.T_INSTANCEOF,
|
||||
scanner.T_INSTEADOF,
|
||||
scanner.T_INTERFACE,
|
||||
scanner.T_LIST,
|
||||
scanner.T_NAMESPACE,
|
||||
scanner.T_PRIVATE,
|
||||
scanner.T_PUBLIC,
|
||||
scanner.T_PRINT,
|
||||
scanner.T_PROTECTED,
|
||||
scanner.T_RETURN,
|
||||
scanner.T_STATIC,
|
||||
scanner.T_SWITCH,
|
||||
scanner.T_THROW,
|
||||
scanner.T_TRAIT,
|
||||
scanner.T_TRY,
|
||||
scanner.T_UNSET,
|
||||
scanner.T_USE,
|
||||
scanner.T_VAR,
|
||||
scanner.T_WHILE,
|
||||
scanner.T_YIELD_FROM,
|
||||
scanner.T_YIELD,
|
||||
scanner.T_INCLUDE,
|
||||
scanner.T_INCLUDE_ONCE,
|
||||
scanner.T_REQUIRE,
|
||||
scanner.T_REQUIRE_ONCE,
|
||||
|
||||
scanner.T_CLASS_C,
|
||||
scanner.T_DIR,
|
||||
scanner.T_FILE,
|
||||
scanner.T_FUNC_C,
|
||||
scanner.T_LINE,
|
||||
scanner.T_NS_C,
|
||||
scanner.T_METHOD_C,
|
||||
scanner.T_TRAIT_C,
|
||||
scanner.T_HALT_COMPILER,
|
||||
|
||||
scanner.T_NEW,
|
||||
scanner.T_LOGICAL_AND,
|
||||
scanner.T_LOGICAL_OR,
|
||||
scanner.T_LOGICAL_XOR,
|
||||
|
||||
scanner.T_NS_SEPARATOR,
|
||||
scanner.T_ELLIPSIS,
|
||||
scanner.T_PAAMAYIM_NEKUDOTAYIM,
|
||||
scanner.T_BOOLEAN_AND,
|
||||
scanner.T_BOOLEAN_OR,
|
||||
scanner.T_AND_EQUAL,
|
||||
scanner.T_OR_EQUAL,
|
||||
scanner.T_CONCAT_EQUAL,
|
||||
scanner.T_MUL_EQUAL,
|
||||
scanner.T_POW_EQUAL,
|
||||
scanner.T_DIV_EQUAL,
|
||||
scanner.T_PLUS_EQUAL,
|
||||
scanner.T_MINUS_EQUAL,
|
||||
scanner.T_XOR_EQUAL,
|
||||
scanner.T_MOD_EQUAL,
|
||||
scanner.T_DEC,
|
||||
scanner.T_INC,
|
||||
scanner.T_DOUBLE_ARROW,
|
||||
scanner.T_SPACESHIP,
|
||||
scanner.T_IS_NOT_EQUAL,
|
||||
scanner.T_IS_NOT_EQUAL,
|
||||
scanner.T_IS_NOT_IDENTICAL,
|
||||
scanner.T_IS_EQUAL,
|
||||
scanner.T_IS_IDENTICAL,
|
||||
scanner.T_SL_EQUAL,
|
||||
scanner.T_SR_EQUAL,
|
||||
scanner.T_IS_GREATER_OR_EQUAL,
|
||||
scanner.T_IS_SMALLER_OR_EQUAL,
|
||||
scanner.T_POW,
|
||||
scanner.T_SL,
|
||||
scanner.T_SR,
|
||||
scanner.T_COALESCE,
|
||||
|
||||
scanner.Rune2Class(';'),
|
||||
scanner.Rune2Class(':'),
|
||||
scanner.Rune2Class(','),
|
||||
scanner.Rune2Class('.'),
|
||||
scanner.Rune2Class('['),
|
||||
scanner.Rune2Class(']'),
|
||||
scanner.Rune2Class('('),
|
||||
scanner.Rune2Class(')'),
|
||||
scanner.Rune2Class('|'),
|
||||
scanner.Rune2Class('/'),
|
||||
scanner.Rune2Class('^'),
|
||||
scanner.Rune2Class('&'),
|
||||
scanner.Rune2Class('+'),
|
||||
scanner.Rune2Class('-'),
|
||||
scanner.Rune2Class('*'),
|
||||
scanner.Rune2Class('='),
|
||||
scanner.Rune2Class('%'),
|
||||
scanner.Rune2Class('!'),
|
||||
scanner.Rune2Class('~'),
|
||||
scanner.Rune2Class('$'),
|
||||
scanner.Rune2Class('<'),
|
||||
scanner.Rune2Class('>'),
|
||||
scanner.Rune2Class('?'),
|
||||
scanner.Rune2Class('@'),
|
||||
scanner.Rune2Class('{'),
|
||||
scanner.Rune2Class('}'),
|
||||
|
||||
scanner.T_VARIABLE,
|
||||
scanner.T_STRING,
|
||||
|
||||
scanner.T_OBJECT_OPERATOR,
|
||||
scanner.T_OBJECT_OPERATOR,
|
||||
scanner.T_STRING,
|
||||
|
||||
scanner.T_CONSTANT_ENCAPSED_STRING,
|
||||
|
||||
scanner.Rune2Class('`'),
|
||||
scanner.T_ENCAPSED_AND_WHITESPACE,
|
||||
scanner.T_VARIABLE,
|
||||
scanner.T_ENCAPSED_AND_WHITESPACE,
|
||||
scanner.T_CURLY_OPEN,
|
||||
scanner.T_VARIABLE,
|
||||
scanner.Rune2Class('}'),
|
||||
scanner.T_ENCAPSED_AND_WHITESPACE,
|
||||
scanner.T_DOLLAR_OPEN_CURLY_BRACES,
|
||||
scanner.T_STRING_VARNAME,
|
||||
scanner.Rune2Class('}'),
|
||||
scanner.Rune2Class('`'),
|
||||
|
||||
scanner.T_START_HEREDOC,
|
||||
scanner.T_ENCAPSED_AND_WHITESPACE,
|
||||
scanner.T_END_HEREDOC,
|
||||
scanner.Rune2Class(';'),
|
||||
scanner.T_START_HEREDOC,
|
||||
scanner.T_ENCAPSED_AND_WHITESPACE,
|
||||
scanner.T_END_HEREDOC,
|
||||
scanner.Rune2Class(';'),
|
||||
|
||||
scanner.T_START_HEREDOC,
|
||||
scanner.T_ENCAPSED_AND_WHITESPACE,
|
||||
scanner.T_VARIABLE,
|
||||
scanner.T_OBJECT_OPERATOR,
|
||||
scanner.T_STRING,
|
||||
scanner.T_ENCAPSED_AND_WHITESPACE,
|
||||
|
||||
scanner.T_VARIABLE,
|
||||
scanner.Rune2Class('['),
|
||||
scanner.T_NUM_STRING,
|
||||
scanner.Rune2Class(']'),
|
||||
scanner.T_ENCAPSED_AND_WHITESPACE,
|
||||
|
||||
scanner.T_VARIABLE,
|
||||
scanner.Rune2Class('['),
|
||||
scanner.T_NUM_STRING,
|
||||
scanner.Rune2Class(']'),
|
||||
scanner.T_ENCAPSED_AND_WHITESPACE,
|
||||
|
||||
scanner.T_VARIABLE,
|
||||
scanner.Rune2Class('['),
|
||||
scanner.T_NUM_STRING,
|
||||
scanner.Rune2Class(']'),
|
||||
scanner.T_ENCAPSED_AND_WHITESPACE,
|
||||
|
||||
scanner.T_VARIABLE,
|
||||
scanner.Rune2Class('['),
|
||||
scanner.T_STRING,
|
||||
scanner.Rune2Class(']'),
|
||||
scanner.T_ENCAPSED_AND_WHITESPACE,
|
||||
|
||||
scanner.T_VARIABLE,
|
||||
scanner.Rune2Class('['),
|
||||
scanner.T_VARIABLE,
|
||||
scanner.Rune2Class(']'),
|
||||
scanner.T_ENCAPSED_AND_WHITESPACE,
|
||||
|
||||
scanner.T_END_HEREDOC,
|
||||
scanner.Rune2Class(';'),
|
||||
}
|
||||
|
||||
lexer := scanner.NewLexer(bytes.NewBufferString(src), "test.php")
|
||||
lv := &lval{}
|
||||
actual := []int{}
|
||||
|
||||
for {
|
||||
token := lexer.Lex(lv)
|
||||
if token < 0 {
|
||||
break
|
||||
}
|
||||
|
||||
actual = append(actual, token)
|
||||
}
|
||||
|
||||
assertEqual(t, expected, actual)
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user