diff options
| author | Lorry Tar Creator <lorry-tar-importer@baserock.org> | 2013-03-14 05:42:27 +0000 |
|---|---|---|
| committer | <> | 2013-04-03 16:25:08 +0000 |
| commit | c4dd7a1a684490673e25aaf4fabec5df138854c4 (patch) | |
| tree | 4d57c44caae4480efff02b90b9be86f44bf25409 /ext/tokenizer | |
| download | php2-master.tar.gz | |
Imported from /home/lorry/working-area/delta_php2/php-5.4.13.tar.bz2.HEADphp-5.4.13master
Diffstat (limited to 'ext/tokenizer')
37 files changed, 14092 insertions, 0 deletions
diff --git a/ext/tokenizer/CREDITS b/ext/tokenizer/CREDITS new file mode 100644 index 0000000..6189688 --- /dev/null +++ b/ext/tokenizer/CREDITS @@ -0,0 +1,2 @@ +tokenizer +Andrei Zmievski, Johannes Schlueter diff --git a/ext/tokenizer/Makefile.frag b/ext/tokenizer/Makefile.frag new file mode 100644 index 0000000..eb122f4 --- /dev/null +++ b/ext/tokenizer/Makefile.frag @@ -0,0 +1,3 @@ +$(top_srcdir)/Zend/zend_language_parser.c: +$(top_srcdir)/Zend/zend_language_scanner.c: +$(builddir)/tokenizer.lo: $(top_srcdir)/Zend/zend_language_parser.c $(top_srcdir)/Zend/zend_language_scanner.c diff --git a/ext/tokenizer/config.m4 b/ext/tokenizer/config.m4 new file mode 100644 index 0000000..007dddf --- /dev/null +++ b/ext/tokenizer/config.m4 @@ -0,0 +1,12 @@ +dnl $Id$ +dnl config.m4 for extension tokenizer + +dnl Otherwise use enable: + +PHP_ARG_ENABLE(tokenizer, whether to enable tokenizer support, +[ --disable-tokenizer Disable tokenizer support], yes) + +if test "$PHP_TOKENIZER" != "no"; then + PHP_NEW_EXTENSION(tokenizer, tokenizer.c tokenizer_data.c, $ext_shared) + PHP_ADD_MAKEFILE_FRAGMENT +fi diff --git a/ext/tokenizer/config.w32 b/ext/tokenizer/config.w32 new file mode 100644 index 0000000..dd3b89e --- /dev/null +++ b/ext/tokenizer/config.w32 @@ -0,0 +1,11 @@ +// $Id$ +// vim:ft=javascript + +ARG_ENABLE("tokenizer", "tokenizer support", "yes"); + +if (PHP_TOKENIZER == "yes") { + EXTENSION("tokenizer", "tokenizer.c tokenizer_data.c"); + AC_DEFINE("HAVE_TOKENIZER", 1, "Tokenizer support"); +} + + diff --git a/ext/tokenizer/package.xml b/ext/tokenizer/package.xml new file mode 100644 index 0000000..f6f754c --- /dev/null +++ b/ext/tokenizer/package.xml @@ -0,0 +1,46 @@ +<?xml version="1.0" encoding="ISO-8859-1" ?> +<!DOCTYPE package SYSTEM "../pear/package.dtd"> +<package> + <name>tokenizer</name> + <summary>PHP Source code tokenizer</summary> + <maintainers> + <maintainer> + <user>andrei</user> + <name>Andrei Zmievski</name> + <email>andrei@php.net</email> + <role>lead</role> + </maintainer> + </maintainers> + <description> +The tokenizer functions provide an interface to the PHP tokenizer +embedded in the Zend Engine. Using these functions you may write +your own PHP source analyzing or modification tools without having +to deal with the language specification at the lexical level. + </description> + <license>PHP</license> + <release> + <state>beta</state> + <version>5.0.0rc1</version> + <date>2004-03-19</date> + <notes> +package.xml added to support installation using pear installer + </notes> + <filelist> + <file role="doc" name="CREDITS"/> + <file role="src" name="config.m4"/> + <file role="src" name="Makefile.frag"/> + <file role="src" name="config.w32"/> + <file role="src" name="tokenizer.dsp"/> + <file role="src" name="tokenizer.c"/> + <file role="src" name="php_tokenizer.h"/> + <file role="doc" name="tokenizer.php"/> + <file role="test" name="tests/bug26463.phpt"/> + </filelist> + <deps> + <dep type="php" rel="ge" version="5" /> + </deps> + </release> +</package> +<!-- +vim:et:ts=1:sw=1 +--> diff --git a/ext/tokenizer/php_tokenizer.h b/ext/tokenizer/php_tokenizer.h new file mode 100644 index 0000000..761556a --- /dev/null +++ b/ext/tokenizer/php_tokenizer.h @@ -0,0 +1,56 @@ +/* + +----------------------------------------------------------------------+ + | PHP Version 5 | + +----------------------------------------------------------------------+ + | Copyright (c) 1997-2013 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Author: Andrei Zmievski <andrei@php.net> | + +----------------------------------------------------------------------+ + */ + +/* $Id$ */ + +#ifndef PHP_TOKENIZER_H +#define PHP_TOKENIZER_H + +extern zend_module_entry tokenizer_module_entry; +#define phpext_tokenizer_ptr &tokenizer_module_entry + +#ifdef ZTS +#include "TSRM.h" +#endif + +void tokenizer_register_constants(INIT_FUNC_ARGS); +char *get_token_type_name(int token_type); + + +PHP_MINIT_FUNCTION(tokenizer); +PHP_MINFO_FUNCTION(tokenizer); + +PHP_FUNCTION(token_get_all); +PHP_FUNCTION(token_name); + +#ifdef ZTS +#define TOKENIZER_G(v) TSRMG(tokenizer_globals_id, zend_tokenizer_globals *, v) +#else +#define TOKENIZER_G(v) (tokenizer_globals.v) +#endif + +#endif /* PHP_TOKENIZER_H */ + + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * indent-tabs-mode: t + * End: + */ diff --git a/ext/tokenizer/tests/001.phpt b/ext/tokenizer/tests/001.phpt new file mode 100644 index 0000000..203e3c7 --- /dev/null +++ b/ext/tokenizer/tests/001.phpt @@ -0,0 +1,259 @@ +--TEST-- +token_name() +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php + +echo token_name(T_INCLUDE), "\n"; +echo token_name(T_INCLUDE_ONCE), "\n"; +echo token_name(T_EVAL), "\n"; +echo token_name(T_REQUIRE), "\n"; +echo token_name(T_REQUIRE_ONCE), "\n"; +echo token_name(T_LOGICAL_OR), "\n"; +echo token_name(T_LOGICAL_XOR), "\n"; +echo token_name(T_LOGICAL_AND), "\n"; +echo token_name(T_PRINT), "\n"; +echo token_name(T_PLUS_EQUAL), "\n"; +echo token_name(T_MINUS_EQUAL), "\n"; +echo token_name(T_MUL_EQUAL), "\n"; +echo token_name(T_DIV_EQUAL), "\n"; +echo token_name(T_CONCAT_EQUAL), "\n"; +echo token_name(T_MOD_EQUAL), "\n"; +echo token_name(T_AND_EQUAL), "\n"; +echo token_name(T_OR_EQUAL), "\n"; +echo token_name(T_XOR_EQUAL), "\n"; +echo token_name(T_SL_EQUAL), "\n"; +echo token_name(T_SR_EQUAL), "\n"; +echo token_name(T_BOOLEAN_OR), "\n"; +echo token_name(T_BOOLEAN_AND), "\n"; +echo token_name(T_IS_EQUAL), "\n"; +echo token_name(T_IS_NOT_EQUAL), "\n"; +echo token_name(T_IS_IDENTICAL), "\n"; +echo token_name(T_IS_NOT_IDENTICAL), "\n"; +echo token_name(T_IS_SMALLER_OR_EQUAL), "\n"; +echo token_name(T_IS_GREATER_OR_EQUAL), "\n"; +echo token_name(T_SL), "\n"; +echo token_name(T_SR), "\n"; +echo token_name(T_INC), "\n"; +echo token_name(T_DEC), "\n"; +echo token_name(T_INT_CAST), "\n"; +echo token_name(T_DOUBLE_CAST), "\n"; +echo token_name(T_STRING_CAST), "\n"; +echo token_name(T_ARRAY_CAST), "\n"; +echo token_name(T_OBJECT_CAST), "\n"; +echo token_name(T_BOOL_CAST), "\n"; +echo token_name(T_UNSET_CAST), "\n"; +echo token_name(T_NEW), "\n"; +echo token_name(T_EXIT), "\n"; +echo token_name(T_IF), "\n"; +echo token_name(T_ELSEIF), "\n"; +echo token_name(T_ELSE), "\n"; +echo token_name(T_ENDIF), "\n"; +echo token_name(T_LNUMBER), "\n"; +echo token_name(T_DNUMBER), "\n"; +echo token_name(T_STRING), "\n"; +echo token_name(T_STRING_VARNAME), "\n"; +echo token_name(T_VARIABLE), "\n"; +echo token_name(T_NUM_STRING), "\n"; +echo token_name(T_INLINE_HTML), "\n"; +echo token_name(T_ENCAPSED_AND_WHITESPACE), "\n"; +echo token_name(T_CONSTANT_ENCAPSED_STRING), "\n"; +echo token_name(T_ECHO), "\n"; +echo token_name(T_DO), "\n"; +echo token_name(T_WHILE), "\n"; +echo token_name(T_ENDWHILE), "\n"; +echo token_name(T_FOR), "\n"; +echo token_name(T_ENDFOR), "\n"; +echo token_name(T_FOREACH), "\n"; +echo token_name(T_ENDFOREACH), "\n"; +echo token_name(T_DECLARE), "\n"; +echo token_name(T_ENDDECLARE), "\n"; +echo token_name(T_AS), "\n"; +echo token_name(T_SWITCH), "\n"; +echo token_name(T_ENDSWITCH), "\n"; +echo token_name(T_CASE), "\n"; +echo token_name(T_DEFAULT), "\n"; +echo token_name(T_BREAK), "\n"; +echo token_name(T_CONTINUE), "\n"; +echo token_name(T_FUNCTION), "\n"; +echo token_name(T_CONST), "\n"; +echo token_name(T_RETURN), "\n"; +echo token_name(T_USE), "\n"; +echo token_name(T_GLOBAL), "\n"; +echo token_name(T_STATIC), "\n"; +echo token_name(T_VAR), "\n"; +echo token_name(T_UNSET), "\n"; +echo token_name(T_ISSET), "\n"; +echo token_name(T_EMPTY), "\n"; +echo token_name(T_CLASS), "\n"; +echo token_name(T_EXTENDS), "\n"; +echo token_name(T_INTERFACE), "\n"; +echo token_name(T_IMPLEMENTS), "\n"; +echo token_name(T_OBJECT_OPERATOR), "\n"; +echo token_name(T_DOUBLE_ARROW), "\n"; +echo token_name(T_LIST), "\n"; +echo token_name(T_ARRAY), "\n"; +echo token_name(T_CLASS_C), "\n"; +echo token_name(T_FUNC_C), "\n"; +echo token_name(T_METHOD_C), "\n"; +echo token_name(T_LINE), "\n"; +echo token_name(T_FILE), "\n"; +echo token_name(T_COMMENT), "\n"; +echo token_name(T_DOC_COMMENT), "\n"; +echo token_name(T_OPEN_TAG), "\n"; +echo token_name(T_OPEN_TAG_WITH_ECHO), "\n"; +echo token_name(T_CLOSE_TAG), "\n"; +echo token_name(T_WHITESPACE), "\n"; +echo token_name(T_START_HEREDOC), "\n"; +echo token_name(T_END_HEREDOC), "\n"; +echo token_name(T_DOLLAR_OPEN_CURLY_BRACES), "\n"; +echo token_name(T_CURLY_OPEN), "\n"; +echo token_name(T_PAAMAYIM_NEKUDOTAYIM), "\n"; +echo token_name(T_PAAMAYIM_NEKUDOTAYIM), "\n"; +echo token_name(T_ABSTRACT), "\n"; +echo token_name(T_CATCH), "\n"; +echo token_name(T_FINAL), "\n"; +echo token_name(T_INSTANCEOF), "\n"; +echo token_name(T_PRIVATE), "\n"; +echo token_name(T_PROTECTED), "\n"; +echo token_name(T_PUBLIC), "\n"; +echo token_name(T_THROW), "\n"; +echo token_name(T_TRY), "\n"; +echo token_name(T_CLONE), "\n"; +echo token_name(T_HALT_COMPILER), "\n"; + +echo token_name(-1), "\n"; +echo token_name(0x8000000F), "\n"; +echo token_name("string"), "\n"; +echo token_name(array()), "\n"; + +echo "Done\n"; +?> +--EXPECTF-- +T_INCLUDE +T_INCLUDE_ONCE +T_EVAL +T_REQUIRE +T_REQUIRE_ONCE +T_LOGICAL_OR +T_LOGICAL_XOR +T_LOGICAL_AND +T_PRINT +T_PLUS_EQUAL +T_MINUS_EQUAL +T_MUL_EQUAL +T_DIV_EQUAL +T_CONCAT_EQUAL +T_MOD_EQUAL +T_AND_EQUAL +T_OR_EQUAL +T_XOR_EQUAL +T_SL_EQUAL +T_SR_EQUAL +T_BOOLEAN_OR +T_BOOLEAN_AND +T_IS_EQUAL +T_IS_NOT_EQUAL +T_IS_IDENTICAL +T_IS_NOT_IDENTICAL +T_IS_SMALLER_OR_EQUAL +T_IS_GREATER_OR_EQUAL +T_SL +T_SR +T_INC +T_DEC +T_INT_CAST +T_DOUBLE_CAST +T_STRING_CAST +T_ARRAY_CAST +T_OBJECT_CAST +T_BOOL_CAST +T_UNSET_CAST +T_NEW +T_EXIT +T_IF +T_ELSEIF +T_ELSE +T_ENDIF +T_LNUMBER +T_DNUMBER +T_STRING +T_STRING_VARNAME +T_VARIABLE +T_NUM_STRING +T_INLINE_HTML +T_ENCAPSED_AND_WHITESPACE +T_CONSTANT_ENCAPSED_STRING +T_ECHO +T_DO +T_WHILE +T_ENDWHILE +T_FOR +T_ENDFOR +T_FOREACH +T_ENDFOREACH +T_DECLARE +T_ENDDECLARE +T_AS +T_SWITCH +T_ENDSWITCH +T_CASE +T_DEFAULT +T_BREAK +T_CONTINUE +T_FUNCTION +T_CONST +T_RETURN +T_USE +T_GLOBAL +T_STATIC +T_VAR +T_UNSET +T_ISSET +T_EMPTY +T_CLASS +T_EXTENDS +T_INTERFACE +T_IMPLEMENTS +T_OBJECT_OPERATOR +T_DOUBLE_ARROW +T_LIST +T_ARRAY +T_CLASS_C +T_FUNC_C +T_METHOD_C +T_LINE +T_FILE +T_COMMENT +T_DOC_COMMENT +T_OPEN_TAG +T_OPEN_TAG_WITH_ECHO +T_CLOSE_TAG +T_WHITESPACE +T_START_HEREDOC +T_END_HEREDOC +T_DOLLAR_OPEN_CURLY_BRACES +T_CURLY_OPEN +T_DOUBLE_COLON +T_DOUBLE_COLON +T_ABSTRACT +T_CATCH +T_FINAL +T_INSTANCEOF +T_PRIVATE +T_PROTECTED +T_PUBLIC +T_THROW +T_TRY +T_CLONE +T_HALT_COMPILER +UNKNOWN +UNKNOWN + +Warning: token_name() expects parameter 1 to be long, string given in %s on line %d + + +Warning: token_name() expects parameter 1 to be long, array given in %s on line %d + +Done diff --git a/ext/tokenizer/tests/002.phpt b/ext/tokenizer/tests/002.phpt new file mode 100644 index 0000000..2a40ffe --- /dev/null +++ b/ext/tokenizer/tests/002.phpt @@ -0,0 +1,982 @@ +--TEST-- +token_get_all() +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--INI-- +short_open_tag=1 +--FILE-- +<?php + +$strings = array( + '<? echo 1; if (isset($a)) print $a+1; $a++; $a--; $a == 2; $a === 2; endif; ?>', + '<?php switch($a) { case 1: break; default: break; } while($a) { exit; } ?>', + '<? /* comment */ if (1 || 2) { } $a = 2 | 1; $b = 3^2; $c = 4&2; ?>', + /* feel free to add more yourself */ + 'wrong syntax here' +); + +foreach ($strings as $s) { + var_dump(token_get_all($s)); +} + +echo "Done\n"; +?> +--EXPECTF-- +array(49) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "<?" + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(1) + } + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(1) + } + [5]=> + string(1) ";" + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "if" + [2]=> + int(1) + } + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [9]=> + string(1) "(" + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "isset" + [2]=> + int(1) + } + [11]=> + string(1) "(" + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [13]=> + string(1) ")" + [14]=> + string(1) ")" + [15]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "print" + [2]=> + int(1) + } + [17]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [18]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [19]=> + string(1) "+" + [20]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(1) + } + [21]=> + string(1) ";" + [22]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [23]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [24]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "++" + [2]=> + int(1) + } + [25]=> + string(1) ";" + [26]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [27]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [28]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "--" + [2]=> + int(1) + } + [29]=> + string(1) ";" + [30]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [31]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [32]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [33]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "==" + [2]=> + int(1) + } + [34]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [35]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(1) + } + [36]=> + string(1) ";" + [37]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [38]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [39]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [40]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "===" + [2]=> + int(1) + } + [41]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [42]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(1) + } + [43]=> + string(1) ";" + [44]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [45]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "endif" + [2]=> + int(1) + } + [46]=> + string(1) ";" + [47]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [48]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(1) + } +} +array(37) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "switch" + [2]=> + int(1) + } + [2]=> + string(1) "(" + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [4]=> + string(1) ")" + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [6]=> + string(1) "{" + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "case" + [2]=> + int(1) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(1) + } + [11]=> + string(1) ":" + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "break" + [2]=> + int(1) + } + [14]=> + string(1) ";" + [15]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "default" + [2]=> + int(1) + } + [17]=> + string(1) ":" + [18]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [19]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "break" + [2]=> + int(1) + } + [20]=> + string(1) ";" + [21]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [22]=> + string(1) "}" + [23]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [24]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "while" + [2]=> + int(1) + } + [25]=> + string(1) "(" + [26]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [27]=> + string(1) ")" + [28]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [29]=> + string(1) "{" + [30]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [31]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "exit" + [2]=> + int(1) + } + [32]=> + string(1) ";" + [33]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [34]=> + string(1) "}" + [35]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [36]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(1) + } +} +array(48) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "<?" + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(13) "/* comment */" + [2]=> + int(1) + } + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "if" + [2]=> + int(1) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [6]=> + string(1) "(" + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(1) + } + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "||" + [2]=> + int(1) + } + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(1) + } + [12]=> + string(1) ")" + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [14]=> + string(1) "{" + [15]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [16]=> + string(1) "}" + [17]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [18]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [19]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [20]=> + string(1) "=" + [21]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [22]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(1) + } + [23]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [24]=> + string(1) "|" + [25]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [26]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(1) + } + [27]=> + string(1) ";" + [28]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [29]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(1) + } + [30]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [31]=> + string(1) "=" + [32]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [33]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "3" + [2]=> + int(1) + } + [34]=> + string(1) "^" + [35]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(1) + } + [36]=> + string(1) ";" + [37]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [38]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$c" + [2]=> + int(1) + } + [39]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [40]=> + string(1) "=" + [41]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [42]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "4" + [2]=> + int(1) + } + [43]=> + string(1) "&" + [44]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(1) + } + [45]=> + string(1) ";" + [46]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [47]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(1) + } +} +array(1) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(17) "wrong syntax here" + [2]=> + int(1) + } +} +Done diff --git a/ext/tokenizer/tests/003.phpt b/ext/tokenizer/tests/003.phpt new file mode 100644 index 0000000..fdcf774 --- /dev/null +++ b/ext/tokenizer/tests/003.phpt @@ -0,0 +1,46 @@ +--TEST-- +token_get_all() and wrong parameters +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php + +var_dump(token_get_all(array())); +var_dump(token_get_all(new stdClass)); +var_dump(token_get_all("")); +var_dump(token_get_all(0)); +var_dump(token_get_all(-1)); + +echo "Done\n"; +?> +--EXPECTF-- +Warning: token_get_all() expects parameter 1 to be string, array given in %s on line %d +NULL + +Warning: token_get_all() expects parameter 1 to be string, object given in %s on line %d +NULL +array(0) { +} +array(1) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "0" + [2]=> + int(1) + } +} +array(1) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "-1" + [2]=> + int(1) + } +} +Done diff --git a/ext/tokenizer/tests/bug26463.phpt b/ext/tokenizer/tests/bug26463.phpt new file mode 100644 index 0000000..d07476b --- /dev/null +++ b/ext/tokenizer/tests/bug26463.phpt @@ -0,0 +1,164 @@ +--TEST-- +Bug #26463 (token_get_all() does not correctly handle semicolons after T_END_HEREDOC) +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +$str = '<?php +$x=<<<DD +jhdsjkfhjdsh +DD +.""; +$a=<<<DDDD +jhdsjkfhjdsh +DDDD; +?>'; +var_dump(token_get_all($str)); +?> +--EXPECTF-- +array(19) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php +" + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$x" + [2]=> + int(2) + } + [2]=> + string(1) "=" + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<<<DD +" + [2]=> + int(2) + } + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(13) "jhdsjkfhjdsh +" + [2]=> + int(3) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "DD" + [2]=> + int(4) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(4) + } + [7]=> + string(1) "." + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) """" + [2]=> + int(5) + } + [9]=> + string(1) ";" + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(5) + } + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(6) + } + [12]=> + string(1) "=" + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "<<<DDDD +" + [2]=> + int(6) + } + [14]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(13) "jhdsjkfhjdsh +" + [2]=> + int(7) + } + [15]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "DDDD" + [2]=> + int(8) + } + [16]=> + string(1) ";" + [17]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(8) + } + [18]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(9) + } +} diff --git a/ext/tokenizer/tests/bug54089.phpt b/ext/tokenizer/tests/bug54089.phpt new file mode 100644 index 0000000..77a4c1f --- /dev/null +++ b/ext/tokenizer/tests/bug54089.phpt @@ -0,0 +1,294 @@ +--TEST-- +Bug #54089 (token_get_all() does not stop after __halt_compiler) +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +$codes = array( + "<?php __halt_compiler", + "<?php __halt_compiler(", + "<?php __halt_compiler();", + "<?php __halt_compiler();ABC", + "<?php __halt_compiler\n(\n)\n;ABC", + "<?php __halt_compiler\nabc\ndef\nghi ABC", +); +foreach ($codes as $code) { + $tokens = token_get_all($code); + var_dump($tokens); + + $code = ''; + foreach ($tokens as $t) + { + $code .= isset($t[1]) ? $t[1] : $t; + } + var_dump($code); +} + +?> +--EXPECTF-- +array(2) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(15) "__halt_compiler" + [2]=> + int(1) + } +} +string(21) "<?php __halt_compiler" +array(3) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(15) "__halt_compiler" + [2]=> + int(1) + } + [2]=> + string(1) "(" +} +string(22) "<?php __halt_compiler(" +array(5) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(15) "__halt_compiler" + [2]=> + int(1) + } + [2]=> + string(1) "(" + [3]=> + string(1) ")" + [4]=> + string(1) ";" +} +string(24) "<?php __halt_compiler();" +array(6) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(15) "__halt_compiler" + [2]=> + int(1) + } + [2]=> + string(1) "(" + [3]=> + string(1) ")" + [4]=> + string(1) ";" + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "ABC" + [2]=> + int(1) + } +} +string(27) "<?php __halt_compiler();ABC" +array(9) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(15) "__halt_compiler" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(1) + } + [3]=> + string(1) "(" + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(2) + } + [5]=> + string(1) ")" + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(3) + } + [7]=> + string(1) ";" + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "ABC" + [2]=> + int(4) + } +} +string(30) "<?php __halt_compiler +( +) +;ABC" +array(9) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(15) "__halt_compiler" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(1) + } + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "abc" + [2]=> + int(2) + } + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(2) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "def" + [2]=> + int(3) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(3) + } + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "ghi" + [2]=> + int(4) + } + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) " ABC" + [2]=> + int(4) + } +} +string(37) "<?php __halt_compiler +abc +def +ghi ABC" diff --git a/ext/tokenizer/tests/token_get_all_basic.phpt b/ext/tokenizer/tests/token_get_all_basic.phpt new file mode 100644 index 0000000..7af8109 --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_basic.phpt @@ -0,0 +1,99 @@ +--TEST-- +Test token_get_all() function : basic functionality +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description : splits the given source into an array of PHP languange tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +echo "*** Testing token_get_all() : basic functionality ***\n"; + +// with php open/close tags +$source = '<?php echo "Hello World"; ?>'; +echo "-- source string with PHP open and close tags --\n"; +var_dump( token_get_all($source) ); + +// without php open/close tags testing for T_INLINE_HTML +$source = "echo 'Hello World';"; +echo "-- source string without PHP open and close tags --\n"; +var_dump( token_get_all($source) ); + +echo "Done" +?> +--EXPECTF-- +*** Testing token_get_all() : basic functionality *** +-- source string with PHP open and close tags -- +array(7) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(13) ""Hello World"" + [2]=> + int(1) + } + [4]=> + string(1) ";" + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(1) + } +} +-- source string without PHP open and close tags -- +array(1) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(19) "echo 'Hello World';" + [2]=> + int(1) + } +} +Done
\ No newline at end of file diff --git a/ext/tokenizer/tests/token_get_all_error.phpt b/ext/tokenizer/tests/token_get_all_error.phpt new file mode 100644 index 0000000..29e97c3 --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_error.phpt @@ -0,0 +1,37 @@ +--TEST-- +Test token_get_all() function : error conditions +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description: splits the given source into an array of PHP languange tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +echo "*** Testing token_get_all() : error conditions ***\n"; + +// with zero arguments +echo "\n-- Testing token_get_all() function with zero arguments --\n"; +var_dump( token_get_all()); + +// with one more than the expected number of arguments +echo "-- Testing token_get_all() function with more than expected no. of arguments --\n"; +$source = '<?php ?>'; +$extra_arg = 10; +var_dump( token_get_all($source, $extra_arg)); + +echo "Done" +?> +--EXPECTF-- +*** Testing token_get_all() : error conditions *** + +-- Testing token_get_all() function with zero arguments -- + +Warning: token_get_all() expects exactly 1 parameter, 0 given in %s on line %d +NULL +-- Testing token_get_all() function with more than expected no. of arguments -- + +Warning: token_get_all() expects exactly 1 parameter, 2 given in %s on line %d +NULL +Done diff --git a/ext/tokenizer/tests/token_get_all_variation1.phpt b/ext/tokenizer/tests/token_get_all_variation1.phpt new file mode 100644 index 0000000..276453f --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_variation1.phpt @@ -0,0 +1,287 @@ +--TEST-- +Test token_get_all() function : usage variations - unexpected values for 'source' argument +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description: splits the given source into an array of PHP languange tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +/* + * Passing different scalar/non-scalar values in place of 'source' argument + * It returns either T_INLINE_HTML by converting values into string or gives warning +*/ + +echo "*** Testing token_get_all() : unexpected values for 'source' argument ***\n"; + +// get an unset variable +$unset_var = 10; +unset ($unset_var); + +// class definition +class MyClass +{ + public function __toString() + { + return "object"; + } +} + +// get resource +$fp = fopen(__FILE__, 'r'); + +// different scalar/nonscalar values for 'source' +$source_values = array( + + // int data +/*1*/ 0, + 1, + 12345, + -2345, + + // float data +/*5*/ 10.5, + -10.5, + 10.1234567e8, + 10.7654321E-8, + .5, + + // array data +/*10*/ array(), + array(0), + array(1), + array(1, 2), + array('color' => 'red', 'item' => 'pen'), + + // null data +/*15*/ NULL, + null, + + // boolean data +/*17*/ true, + false, + TRUE, + FALSE, + + // empty string +/*21*/ "", + '', + + // object data +/*23*/ new MyClass(), + + // resource data + $fp, + + // undefined data + @$undefined_var, + + // unset data +/*26*/ @$unset_var, +); + +for($count = 0; $count < count($source_values); $count++) { + echo "--Iteration ".($count + 1)." --\n"; + var_dump( token_get_all($source_values[$count])); +}; + +fclose($fp); +echo "Done" +?> +--EXPECTF-- +*** Testing token_get_all() : unexpected values for 'source' argument *** +--Iteration 1 -- +array(1) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "0" + [2]=> + int(1) + } +} +--Iteration 2 -- +array(1) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(1) + } +} +--Iteration 3 -- +array(1) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "12345" + [2]=> + int(1) + } +} +--Iteration 4 -- +array(1) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "-2345" + [2]=> + int(1) + } +} +--Iteration 5 -- +array(1) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "10.5" + [2]=> + int(1) + } +} +--Iteration 6 -- +array(1) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "-10.5" + [2]=> + int(1) + } +} +--Iteration 7 -- +array(1) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(10) "1012345670" + [2]=> + int(1) + } +} +--Iteration 8 -- +array(1) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(13) "1.07654321E-7" + [2]=> + int(1) + } +} +--Iteration 9 -- +array(1) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "0.5" + [2]=> + int(1) + } +} +--Iteration 10 -- + +Warning: token_get_all() expects parameter 1 to be string, array given in %s on line %d +NULL +--Iteration 11 -- + +Warning: token_get_all() expects parameter 1 to be string, array given in %s on line %d +NULL +--Iteration 12 -- + +Warning: token_get_all() expects parameter 1 to be string, array given in %s on line %d +NULL +--Iteration 13 -- + +Warning: token_get_all() expects parameter 1 to be string, array given in %s on line %d +NULL +--Iteration 14 -- + +Warning: token_get_all() expects parameter 1 to be string, array given in %s on line %d +NULL +--Iteration 15 -- +array(0) { +} +--Iteration 16 -- +array(0) { +} +--Iteration 17 -- +array(1) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(1) + } +} +--Iteration 18 -- +array(0) { +} +--Iteration 19 -- +array(1) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(1) + } +} +--Iteration 20 -- +array(0) { +} +--Iteration 21 -- +array(0) { +} +--Iteration 22 -- +array(0) { +} +--Iteration 23 -- +array(1) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "object" + [2]=> + int(1) + } +} +--Iteration 24 -- + +Warning: token_get_all() expects parameter 1 to be string, resource given in %s on line %d +NULL +--Iteration 25 -- +array(0) { +} +--Iteration 26 -- +array(0) { +} +Done diff --git a/ext/tokenizer/tests/token_get_all_variation10.phpt b/ext/tokenizer/tests/token_get_all_variation10.phpt new file mode 100644 index 0000000..55e9d7e --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_variation10.phpt @@ -0,0 +1,783 @@ +--TEST-- +Test token_get_all() function : usage variations - with constant tokens +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description: splits the given source into an array of PHP languange tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +/* + * Using different types of constants in 'source' string to check them for token + * integer const - T_LNUMBER(305) + * float/double/real const - T_DNUMBER(306) + * string cosnt - T_CONSTANT_ESCAPED_STRING(315) + * bool const (no tokens specified) - T_UNKNOWN(307) + * null const (no tokens specified) - T_UNKNOWN(307) +*/ + +echo "*** Testing token_get_all() : 'source' string with different constants ***\n"; + +$a = 1; +$b = 0; + +$source = array ( + // int const + '<?php $a = 1 + 034; $b = $a + 0x3F; ?>', + + // float const + '<?php $a = 0.23E-2 + 0.43e2 + 0.5; ?>', + + // string const + '<?php $a = "hello ".\'world\'; ?>', + + // bool const + "<?php \$a = (\$b)? true : false; ?>", + "<?php \$b = (\$a)? FALSE : TRUE; ?>", + + // null const + '<?php $b = null | NULL; ?>' +); +for($count = 0; $count < count($source); $count++) { + echo "-- Iteration ".($count + 1)." --\n"; + var_dump( token_get_all($source[$count])); +} + +echo "Done" +?> +--EXPECTF-- +*** Testing token_get_all() : 'source' string with different constants *** +-- Iteration 1 -- +array(24) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [3]=> + string(1) "=" + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(1) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [7]=> + string(1) "+" + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "034" + [2]=> + int(1) + } + [10]=> + string(1) ";" + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(1) + } + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [14]=> + string(1) "=" + [15]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [17]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [18]=> + string(1) "+" + [19]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [20]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "0x3F" + [2]=> + int(1) + } + [21]=> + string(1) ";" + [22]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [23]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(1) + } +} +-- Iteration 2 -- +array(17) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [3]=> + string(1) "=" + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "0.23E-2" + [2]=> + int(1) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [7]=> + string(1) "+" + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "0.43e2" + [2]=> + int(1) + } + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [11]=> + string(1) "+" + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "0.5" + [2]=> + int(1) + } + [14]=> + string(1) ";" + [15]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(1) + } +} +-- Iteration 3 -- +array(11) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [3]=> + string(1) "=" + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) ""hello "" + [2]=> + int(1) + } + [6]=> + string(1) "." + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "'world'" + [2]=> + int(1) + } + [8]=> + string(1) ";" + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(1) + } +} +-- Iteration 4 -- +array(18) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [3]=> + string(1) "=" + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [5]=> + string(1) "(" + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(1) + } + [7]=> + string(1) ")" + [8]=> + string(1) "?" + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "true" + [2]=> + int(1) + } + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [12]=> + string(1) ":" + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [14]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "false" + [2]=> + int(1) + } + [15]=> + string(1) ";" + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [17]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(1) + } +} +-- Iteration 5 -- +array(18) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [3]=> + string(1) "=" + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [5]=> + string(1) "(" + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [7]=> + string(1) ")" + [8]=> + string(1) "?" + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "FALSE" + [2]=> + int(1) + } + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [12]=> + string(1) ":" + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [14]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "TRUE" + [2]=> + int(1) + } + [15]=> + string(1) ";" + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [17]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(1) + } +} +-- Iteration 6 -- +array(13) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [3]=> + string(1) "=" + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "null" + [2]=> + int(1) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [7]=> + string(1) "|" + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "NULL" + [2]=> + int(1) + } + [10]=> + string(1) ";" + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(1) + } +} +Done diff --git a/ext/tokenizer/tests/token_get_all_variation11.phpt b/ext/tokenizer/tests/token_get_all_variation11.phpt new file mode 100644 index 0000000..ecc8617 --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_variation11.phpt @@ -0,0 +1,1149 @@ +--TEST-- +Test token_get_all() function : usage variations - with control structure tokens +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description: splits the given source into an array of PHP languange tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +/* + * Using different control structure keywords + * if..else, elseif - T_IF(301), T_ELSEIF(302), T_ELSE(303) + * while - T_WHILE(318) + * do...while - T_DO(317) + * for - T_ENDFOR(320) + * foreach - T_ENDFOREACH(322) + * switch...case - T_ENDSWITCH(327), T_CASE(329) + * break - T_BREAK(331) + * continue - T_CONTINUE(332) +*/ + +echo "*** Testing token_get_all() : for control structure tokens ***\n"; + +// if..elseif....else +echo "-- with if..elseif..else..tokens --\n"; + +$source = '<?php +if($a == true) { + echo "$a = true"; +} +elseif($a == false) { + echo false; +} +else + echo 1; +?>'; + +var_dump( token_get_all($source)); + +// while..., do..while, break, continue +echo "-- with while..., do..while, switch & continue tokens --\n"; + +$source = "<?php while(true) { +echo 'True'; +break; +} +do { +continue; +}while(false); ?>"; + +var_dump( token_get_all($source)); + +// for..., foreach( as ) +echo "-- with for..foreach( as ) tokens --\n"; + +$source = '<?php for($count=0; $count < 5; $count++) { +echo $count; +} +foreach($values as $index) { +continue; +} ?>'; + +var_dump( token_get_all($source)); + +// switch..case, default +echo "-- with switch...case tokens --\n"; + +$source = '<?php switch($var) +case 1: break; +default: echo "default"; ?>'; + +var_dump( token_get_all($source)); + +echo "Done" +?> +--EXPECTF-- +*** Testing token_get_all() : for control structure tokens *** +-- with if..elseif..else..tokens -- +array(49) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "if" + [2]=> + int(2) + } + [3]=> + string(1) "(" + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(2) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [6]=> + array(3) { + [0]=> + int(283) + [1]=> + string(2) "==" + [2]=> + int(2) + } + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "true" + [2]=> + int(2) + } + [9]=> + string(1) ")" + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [11]=> + string(1) "{" + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) " + " + [2]=> + int(2) + } + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(%d) + } + [14]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [15]=> + string(1) """ + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(%d) + } + [17]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) " = true" + [2]=> + int(%d) + } + [18]=> + string(1) """ + [19]=> + string(1) ";" + [20]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(%d) + } + [21]=> + string(1) "}" + [22]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(4) + } + [23]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "elseif" + [2]=> + int(5) + } + [24]=> + string(1) "(" + [25]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(5) + } + [26]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(5) + } + [27]=> + array(3) { + [0]=> + int(283) + [1]=> + string(2) "==" + [2]=> + int(5) + } + [28]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(5) + } + [29]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "false" + [2]=> + int(5) + } + [30]=> + string(1) ")" + [31]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(5) + } + [32]=> + string(1) "{" + [33]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(5) + } + [34]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(6) + } + [35]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(6) + } + [36]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "false" + [2]=> + int(6) + } + [37]=> + string(1) ";" + [38]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(6) + } + [39]=> + string(1) "}" + [40]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(7) + } + [41]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "else" + [2]=> + int(8) + } + [42]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(8) + } + [43]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(9) + } + [44]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(9) + } + [45]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(9) + } + [46]=> + string(1) ";" + [47]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(9) + } + [48]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(10) + } +} +-- with while..., do..while, switch & continue tokens -- +array(33) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "while" + [2]=> + int(1) + } + [2]=> + string(1) "(" + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "true" + [2]=> + int(1) + } + [4]=> + string(1) ")" + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [6]=> + string(1) "{" + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(1) + } + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(2) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "'True'" + [2]=> + int(2) + } + [11]=> + string(1) ";" + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(2) + } + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "break" + [2]=> + int(%d) + } + [14]=> + string(1) ";" + [15]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(%d) + } + [16]=> + string(1) "}" + [17]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(4) + } + [18]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "do" + [2]=> + int(5) + } + [19]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(5) + } + [20]=> + string(1) "{" + [21]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(5) + } + [22]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "continue" + [2]=> + int(6) + } + [23]=> + string(1) ";" + [24]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(6) + } + [25]=> + string(1) "}" + [26]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "while" + [2]=> + int(7) + } + [27]=> + string(1) "(" + [28]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "false" + [2]=> + int(7) + } + [29]=> + string(1) ")" + [30]=> + string(1) ";" + [31]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(7) + } + [32]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(7) + } +} +-- with for..foreach( as ) tokens -- +array(45) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "for" + [2]=> + int(1) + } + [2]=> + string(1) "(" + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "$count" + [2]=> + int(1) + } + [4]=> + string(1) "=" + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "0" + [2]=> + int(1) + } + [6]=> + string(1) ";" + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "$count" + [2]=> + int(1) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [10]=> + string(1) "<" + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "5" + [2]=> + int(1) + } + [13]=> + string(1) ";" + [14]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [15]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "$count" + [2]=> + int(1) + } + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "++" + [2]=> + int(1) + } + [17]=> + string(1) ")" + [18]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [19]=> + string(1) "{" + [20]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(1) + } + [21]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(2) + } + [22]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [23]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "$count" + [2]=> + int(2) + } + [24]=> + string(1) ";" + [25]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(2) + } + [26]=> + string(1) "}" + [27]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(%d) + } + [28]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "foreach" + [2]=> + int(4) + } + [29]=> + string(1) "(" + [30]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "$values" + [2]=> + int(4) + } + [31]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [32]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "as" + [2]=> + int(4) + } + [33]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [34]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "$index" + [2]=> + int(4) + } + [35]=> + string(1) ")" + [36]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [37]=> + string(1) "{" + [38]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(4) + } + [39]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "continue" + [2]=> + int(5) + } + [40]=> + string(1) ";" + [41]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(5) + } + [42]=> + string(1) "}" + [43]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(6) + } + [44]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(6) + } +} +-- with switch...case tokens -- +array(23) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "switch" + [2]=> + int(1) + } + [2]=> + string(1) "(" + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "$var" + [2]=> + int(1) + } + [4]=> + string(1) ")" + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(1) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "case" + [2]=> + int(2) + } + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(2) + } + [9]=> + string(1) ":" + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "break" + [2]=> + int(2) + } + [12]=> + string(1) ";" + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(2) + } + [14]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "default" + [2]=> + int(%d) + } + [15]=> + string(1) ":" + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [17]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(%d) + } + [18]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [19]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(9) ""default"" + [2]=> + int(%d) + } + [20]=> + string(1) ";" + [21]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [22]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(%d) + } +} +Done diff --git a/ext/tokenizer/tests/token_get_all_variation12.phpt b/ext/tokenizer/tests/token_get_all_variation12.phpt new file mode 100644 index 0000000..368fb08 --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_variation12.phpt @@ -0,0 +1,575 @@ +--TEST-- +Test token_get_all() function : usage variations - with predefined language constants +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description: splits the given source into an array of PHP language tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +/* + * Testing token_get_all() with following predefined language constants: + * __FILE__ - T_FILE + * __CLASS__ - T_CLASS_C + * __TRAIT__ - T_TRAIT_C + * __FUNCTION__ - T_FUNC_C + * __LINE__ - T_LINE + * __METHOD__ - T_METHOD_C +*/ + +echo "*** Testing token_get_all() : with language constants ***\n"; + +// parsing __FILE__ token +echo "-- with FILE --\n"; +$source = "<?php +\$fp = fopen(__FILE__, 'r'); +?>"; +var_dump( token_get_all($source)); + +// parsing __CLASS__, __TRAIT__ and __FUNCTION__ tokens +echo "-- with CLASS, TRAIT and FUNCTION --\n"; +$source = '<?php +class MyClass +{ + echo __CLASS__; + echo __TRAIT__; + function myFunction() + { echo __FUNCTION__; } +} +?>'; +var_dump( token_get_all($source)); + +// parsing __LINE__ and __METHOD__ tokens +echo "-- with LINE and METHOD --\n"; +$source = '<?php + $a = __LINE__; + $b = $b.__METHOD__; +?>'; +var_dump( token_get_all($source)); + +echo "Done" +?> +--EXPECTF-- +*** Testing token_get_all() : with language constants *** +-- with FILE -- +array(16) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "$fp" + [2]=> + int(2) + } + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [4]=> + string(1) "=" + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) " " + [2]=> + int(2) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "fopen" + [2]=> + int(2) + } + [7]=> + string(1) "(" + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "__FILE__" + [2]=> + int(2) + } + [9]=> + string(1) "," + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "'r'" + [2]=> + int(2) + } + [12]=> + string(1) ")" + [13]=> + string(1) ";" + [14]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(2) + } + [15]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(%d) + } +} +-- with CLASS, TRAIT and FUNCTION -- +array(35) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php +" + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "class" + [2]=> + int(2) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "MyClass" + [2]=> + int(2) + } + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(2) + } + [5]=> + string(1) "{" + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(%d) + } + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(4) + } + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) " " + [2]=> + int(4) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(9) "__CLASS__" + [2]=> + int(4) + } + [10]=> + string(1) ";" + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(%d) + } + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(5) + } + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) " " + [2]=> + int(5) + } + [14]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(9) "__TRAIT__" + [2]=> + int(5) + } + [15]=> + string(1) ";" + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(5) + } + [17]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "function" + [2]=> + int(6) + } + [18]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(6) + } + [19]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(10) "myFunction" + [2]=> + int(6) + } + [20]=> + string(1) "(" + [21]=> + string(1) ")" + [22]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(6) + } + [23]=> + string(1) "{" + [24]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) " " + [2]=> + int(7) + } + [25]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(7) + } + [26]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) " " + [2]=> + int(7) + } + [27]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(12) "__FUNCTION__" + [2]=> + int(7) + } + [28]=> + string(1) ";" + [29]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(7) + } + [30]=> + string(1) "}" + [31]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(7) + } + [32]=> + string(1) "}" + [33]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(8) + } + [34]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(9) + } +} +-- with LINE and METHOD -- +array(19) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php +" + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) " " + [2]=> + int(2) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(2) + } + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [4]=> + string(1) "=" + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "__LINE__" + [2]=> + int(2) + } + [7]=> + string(1) ";" + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(2) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(%d) + } + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [11]=> + string(1) "=" + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(%d) + } + [14]=> + string(1) "." + [15]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(10) "__METHOD__" + [2]=> + int(%d) + } + [16]=> + string(1) ";" + [17]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(%d) + } + [18]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(4) + } +} +Done diff --git a/ext/tokenizer/tests/token_get_all_variation13.phpt b/ext/tokenizer/tests/token_get_all_variation13.phpt new file mode 100644 index 0000000..9b2f3bc --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_variation13.phpt @@ -0,0 +1,1169 @@ +--TEST-- +Test token_get_all() function : usage variations - with class/object constructs +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description: splits the given source into an array of PHP languange tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +/* + * Testing token_get_all() with different class/object keywords + * scope related : + * static - T_STATIC(346), global - T_GLOBAL(340), + * private - T_PRIVATE(343), public - T_PUBLIC(341), + * protected - T_PROTECTED(342) + * class/object related : + * var - T_VAR(347), abstract - T_ABSTRACT(345), + * interface - T_INTERFACE(353), class - T_CLASS(352), + * extends - T_EXTENDS(354), implements - T_IMPLEMENTS(355), new - T_NEW(299) +*/ + +echo "*** Testing token_get_all() : with class/object constructs ***\n"; + +$source = '<?php +interface MyInterface +{ + public const var $var = 10; +} +abstract class MyClass +{ + private var $a; + public var $b; + protected var $c; + static $d; + final $e = 10; + + abstract public function myFunction($a); +} +class ChildClass extends MyClass implements MyInterface +{ + global $value; + function myFunction($a) + { + $a = new ChildClass(); + if($a instanceof MyClass) + echo "object created"; + } +} +ChildClass::myFunction(10); +?>'; +$tokens = token_get_all($source); +var_dump($tokens); + +echo "Done" +?> +--EXPECTF-- +*** Testing token_get_all() : with class/object constructs *** +array(145) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php +" + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(9) "interface" + [2]=> + int(2) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(11) "MyInterface" + [2]=> + int(2) + } + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(2) + } + [5]=> + string(1) "{" + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(%d) + } + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "public" + [2]=> + int(4) + } + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "const" + [2]=> + int(4) + } + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "var" + [2]=> + int(4) + } + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "$var" + [2]=> + int(4) + } + [14]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [15]=> + string(1) "=" + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [17]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "10" + [2]=> + int(4) + } + [18]=> + string(1) ";" + [19]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(4) + } + [20]=> + string(1) "}" + [21]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(5) + } + [22]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "abstract" + [2]=> + int(6) + } + [23]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(6) + } + [24]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "class" + [2]=> + int(6) + } + [25]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(6) + } + [26]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "MyClass" + [2]=> + int(6) + } + [27]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(6) + } + [28]=> + string(1) "{" + [29]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(7) + } + [30]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "private" + [2]=> + int(8) + } + [31]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [32]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "var" + [2]=> + int(8) + } + [33]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [34]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(8) + } + [35]=> + string(1) ";" + [36]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(8) + } + [37]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "public" + [2]=> + int(9) + } + [38]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(9) + } + [39]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "var" + [2]=> + int(9) + } + [40]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(9) + } + [41]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(9) + } + [42]=> + string(1) ";" + [43]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(9) + } + [44]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(9) "protected" + [2]=> + int(10) + } + [45]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(10) + } + [46]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "var" + [2]=> + int(10) + } + [47]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(10) + } + [48]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$c" + [2]=> + int(10) + } + [49]=> + string(1) ";" + [50]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(10) + } + [51]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "static" + [2]=> + int(11) + } + [52]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(11) + } + [53]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$d" + [2]=> + int(11) + } + [54]=> + string(1) ";" + [55]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(11) + } + [56]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "final" + [2]=> + int(12) + } + [57]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(12) + } + [58]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$e" + [2]=> + int(12) + } + [59]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(12) + } + [60]=> + string(1) "=" + [61]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(12) + } + [62]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "10" + [2]=> + int(12) + } + [63]=> + string(1) ";" + [64]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) " + + " + [2]=> + int(12) + } + [65]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "abstract" + [2]=> + int(14) + } + [66]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(14) + } + [67]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "public" + [2]=> + int(14) + } + [68]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(14) + } + [69]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "function" + [2]=> + int(14) + } + [70]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(14) + } + [71]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(10) "myFunction" + [2]=> + int(14) + } + [72]=> + string(1) "(" + [73]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(14) + } + [74]=> + string(1) ")" + [75]=> + string(1) ";" + [76]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(14) + } + [77]=> + string(1) "}" + [78]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(15) + } + [79]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "class" + [2]=> + int(16) + } + [80]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(16) + } + [81]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(10) "ChildClass" + [2]=> + int(16) + } + [82]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(16) + } + [83]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "extends" + [2]=> + int(16) + } + [84]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(16) + } + [85]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "MyClass" + [2]=> + int(16) + } + [86]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(16) + } + [87]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(10) "implements" + [2]=> + int(16) + } + [88]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(16) + } + [89]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(11) "MyInterface" + [2]=> + int(16) + } + [90]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(16) + } + [91]=> + string(1) "{" + [92]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(17) + } + [93]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "global" + [2]=> + int(18) + } + [94]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(18) + } + [95]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "$value" + [2]=> + int(18) + } + [96]=> + string(1) ";" + [97]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(18) + } + [98]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "function" + [2]=> + int(19) + } + [99]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(19) + } + [100]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(10) "myFunction" + [2]=> + int(19) + } + [101]=> + string(1) "(" + [102]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(19) + } + [103]=> + string(1) ")" + [104]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(19) + } + [105]=> + string(1) "{" + [106]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) " + " + [2]=> + int(20) + } + [107]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(21) + } + [108]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(21) + } + [109]=> + string(1) "=" + [110]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(21) + } + [111]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "new" + [2]=> + int(21) + } + [112]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(21) + } + [113]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(10) "ChildClass" + [2]=> + int(21) + } + [114]=> + string(1) "(" + [115]=> + string(1) ")" + [116]=> + string(1) ";" + [117]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) " + " + [2]=> + int(21) + } + [118]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "if" + [2]=> + int(22) + } + [119]=> + string(1) "(" + [120]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(22) + } + [121]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(22) + } + [122]=> + array(3) { + [0]=> + int(288) + [1]=> + string(10) "instanceof" + [2]=> + int(22) + } + [123]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(22) + } + [124]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "MyClass" + [2]=> + int(22) + } + [125]=> + string(1) ")" + [126]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) " + " + [2]=> + int(22) + } + [127]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(23) + } + [128]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(23) + } + [129]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(16) ""object created"" + [2]=> + int(23) + } + [130]=> + string(1) ";" + [131]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(23) + } + [132]=> + string(1) "}" + [133]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(24) + } + [134]=> + string(1) "}" + [135]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(25) + } + [136]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(10) "ChildClass" + [2]=> + int(26) + } + [137]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "::" + [2]=> + int(26) + } + [138]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(10) "myFunction" + [2]=> + int(26) + } + [139]=> + string(1) "(" + [140]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "10" + [2]=> + int(26) + } + [141]=> + string(1) ")" + [142]=> + string(1) ";" + [143]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(26) + } + [144]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(27) + } +} +Done diff --git a/ext/tokenizer/tests/token_get_all_variation14.phpt b/ext/tokenizer/tests/token_get_all_variation14.phpt new file mode 100644 index 0000000..5fc390e --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_variation14.phpt @@ -0,0 +1,282 @@ +--TEST-- +Test token_get_all() function : usage variations - invalid token values +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description: splits the given source into an array of PHP languange tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +/* + * Testing token_get_all() with 'source' string containing invalid/unknown token value + * unknown tokens - T_UNKNOWN(307) +*/ + +echo "*** Testing token_get_all() : with invalid/unknown tokens ***\n"; + +// with valid php tags and invalid tokens +echo "-- with valid PHP tags & invlid tokens --\n"; +$source = '<?php +struct myStruct { + variable $a; + method() { display $a; } +} +?>'; +var_dump( token_get_all($source)); + +// with invalid open tag for testing entire source to be unkown token +echo "-- with invlalid PHP open tag & valid tokens --\n"; +$source = '<pli +echo "hello world"; ?>'; +var_dump( token_get_all($source)); + +// with invalid PHP tags and invalid tokens +echo "-- with invalid PHP tags and tokens --\n"; +$source = '<PDP display $a; <'; +var_dump( token_get_all($source)); + +echo "Done" +?> +--EXPECTF-- +*** Testing token_get_all() : with invalid/unknown tokens *** +-- with valid PHP tags & invlid tokens -- +array(29) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "struct" + [2]=> + int(2) + } + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "myStruct" + [2]=> + int(2) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [6]=> + string(1) "{" + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(2) + } + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "variable" + [2]=> + int(%d) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(%d) + } + [11]=> + string(1) ";" + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(%d) + } + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "method" + [2]=> + int(4) + } + [14]=> + string(1) "(" + [15]=> + string(1) ")" + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [17]=> + string(1) "{" + [18]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [19]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "display" + [2]=> + int(4) + } + [20]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [21]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(4) + } + [22]=> + string(1) ";" + [23]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [24]=> + string(1) "}" + [25]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(4) + } + [26]=> + string(1) "}" + [27]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(5) + } + [28]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(6) + } +} +-- with invlalid PHP open tag & valid tokens -- +array(1) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(28) "<pli +echo "hello world"; ?>" + [2]=> + int(1) + } +} +-- with invalid PHP tags and tokens -- +array(1) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(19) "<PDP display $a; <" + [2]=> + int(1) + } +} +Done diff --git a/ext/tokenizer/tests/token_get_all_variation15.phpt b/ext/tokenizer/tests/token_get_all_variation15.phpt new file mode 100644 index 0000000..fbb50de --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_variation15.phpt @@ -0,0 +1,780 @@ +--TEST-- +Test token_get_all() function : usage variations - heredoc string for 'source' +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--INI-- +short_open_tag=On +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description: splits the given source into an array of PHP languange tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +/* + * Testing token_get_all() with heredoc 'source' string with all different types of token and heredoc string within + * <<<EOT - T_START_HEREDOC(371) + * EOT - T_END_HEREDOC(372) +*/ + +echo "*** Testing token_get_all() : with heredoc source string ***\n"; + +$source = <<<EOT +<?= + \$a = 2; + \$b = 1; + \$c = <<<EOS + This is to test + heredoc string +EOS; + echo \$a + \$b; + function myFunction(\$a) + { + var_dump(\$a); + } + if(\$b < 10) { + \$b++; + } + else + \$b--; + while(\$a > 0) { + echo "*"; + \$a--; + } + myFunction(10); +?> +EOT; +var_dump( token_get_all($source)); + +echo "Done" +?> +--EXPECTF-- +*** Testing token_get_all() : with heredoc source string *** +array(103) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "<?=" + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) " + " + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(2) + } + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [4]=> + string(1) "=" + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(2) + } + [7]=> + string(1) ";" + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(2) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(%d) + } + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [11]=> + string(1) "=" + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(%d) + } + [14]=> + string(1) ";" + [15]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(%d) + } + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$c" + [2]=> + int(4) + } + [17]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [18]=> + string(1) "=" + [19]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [20]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "<<<EOS +" + [2]=> + int(4) + } + [21]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(36) " This is to test + heredoc string +" + [2]=> + int(5) + } + [22]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "EOS" + [2]=> + int(7) + } + [23]=> + string(1) ";" + [24]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(7) + } + [25]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(8) + } + [26]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [27]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(8) + } + [28]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [29]=> + string(1) "+" + [30]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [31]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(8) + } + [32]=> + string(1) ";" + [33]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(8) + } + [34]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "function" + [2]=> + int(9) + } + [35]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(9) + } + [36]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(10) "myFunction" + [2]=> + int(9) + } + [37]=> + string(1) "(" + [38]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(9) + } + [39]=> + string(1) ")" + [40]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(9) + } + [41]=> + string(1) "{" + [42]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) " + " + [2]=> + int(10) + } + [43]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "var_dump" + [2]=> + int(11) + } + [44]=> + string(1) "(" + [45]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(11) + } + [46]=> + string(1) ")" + [47]=> + string(1) ";" + [48]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(11) + } + [49]=> + string(1) "}" + [50]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(12) + } + [51]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "if" + [2]=> + int(13) + } + [52]=> + string(1) "(" + [53]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(13) + } + [54]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(13) + } + [55]=> + string(1) "<" + [56]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(13) + } + [57]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "10" + [2]=> + int(13) + } + [58]=> + string(1) ")" + [59]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(13) + } + [60]=> + string(1) "{" + [61]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) " + " + [2]=> + int(13) + } + [62]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(14) + } + [63]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "++" + [2]=> + int(14) + } + [64]=> + string(1) ";" + [65]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(14) + } + [66]=> + string(1) "}" + [67]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(15) + } + [68]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "else" + [2]=> + int(16) + } + [69]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) " + " + [2]=> + int(16) + } + [70]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(17) + } + [71]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "--" + [2]=> + int(17) + } + [72]=> + string(1) ";" + [73]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(17) + } + [74]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "while" + [2]=> + int(18) + } + [75]=> + string(1) "(" + [76]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(18) + } + [77]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(18) + } + [78]=> + string(1) ">" + [79]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(18) + } + [80]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "0" + [2]=> + int(18) + } + [81]=> + string(1) ")" + [82]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(18) + } + [83]=> + string(1) "{" + [84]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) " + " + [2]=> + int(18) + } + [85]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(19) + } + [86]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(19) + } + [87]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) ""*"" + [2]=> + int(19) + } + [88]=> + string(1) ";" + [89]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) " + " + [2]=> + int(19) + } + [90]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(20) + } + [91]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "--" + [2]=> + int(20) + } + [92]=> + string(1) ";" + [93]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(20) + } + [94]=> + string(1) "}" + [95]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(21) + } + [96]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(10) "myFunction" + [2]=> + int(22) + } + [97]=> + string(1) "(" + [98]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "10" + [2]=> + int(22) + } + [99]=> + string(1) ")" + [100]=> + string(1) ";" + [101]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(22) + } + [102]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(23) + } +} +Done diff --git a/ext/tokenizer/tests/token_get_all_variation16.phpt b/ext/tokenizer/tests/token_get_all_variation16.phpt new file mode 100644 index 0000000..db9bde1 --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_variation16.phpt @@ -0,0 +1,1003 @@ +--TEST-- +Test token_get_all() function : usage variations - with function constructs +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description: splits the given source into an array of PHP languange tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +/* + * Testing token_get_all() with different function keywords + * function - T_FUNCTION(333), return - T_RETURN(335) + * different functions: + * include() - T_INCLUDE(262), print() - T_PRINT(266), + * isset() - T_ISSET(349), list() - T_LIST(358), + * require() - T_REQUIRE(259), empty() - T_EMPTY(350), + * declare() - T_DECLARE(324), array() - T_ARRAY(359), + * __halt_compiler() - T_HALT_COMPILER(351) +*/ + +echo "*** Testing token_get_all() : with different function constructs ***\n"; + +$source = '<?php +declare(VALUE=100); +include("addfile.php"); +require("sumfile.php"); + +function myFunction($a) +{ + if($a % 2) + return 1; + else + exit; +} + +$a = VALUE; +$b = 20; +$c = array(1,2); +$b >>= 2; + +if($b <= 0) + die; +else + print($b); + +list($value1,$value2) = $c; +if(empty($value1) && !isset($value1)) { + myFunction(); +} +?>'; +$tokens = token_get_all($source); +var_dump($tokens); + +echo "Done"; +?> +--EXPECTF-- +*** Testing token_get_all() : with different function constructs *** +array(142) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php +" + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "declare" + [2]=> + int(2) + } + [2]=> + string(1) "(" + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "VALUE" + [2]=> + int(2) + } + [4]=> + string(1) "=" + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "100" + [2]=> + int(2) + } + [6]=> + string(1) ")" + [7]=> + string(1) ";" + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(2) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "include" + [2]=> + int(3) + } + [10]=> + string(1) "(" + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(13) ""addfile.php"" + [2]=> + int(3) + } + [12]=> + string(1) ")" + [13]=> + string(1) ";" + [14]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(3) + } + [15]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "require" + [2]=> + int(4) + } + [16]=> + string(1) "(" + [17]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(13) ""sumfile.php"" + [2]=> + int(4) + } + [18]=> + string(1) ")" + [19]=> + string(1) ";" + [20]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) " + +" + [2]=> + int(4) + } + [21]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "function" + [2]=> + int(6) + } + [22]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(6) + } + [23]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(10) "myFunction" + [2]=> + int(6) + } + [24]=> + string(1) "(" + [25]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(6) + } + [26]=> + string(1) ")" + [27]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(6) + } + [28]=> + string(1) "{" + [29]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(7) + } + [30]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "if" + [2]=> + int(8) + } + [31]=> + string(1) "(" + [32]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(8) + } + [33]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [34]=> + string(1) "%" + [35]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [36]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(8) + } + [37]=> + string(1) ")" + [38]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) " + " + [2]=> + int(8) + } + [39]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "return" + [2]=> + int(9) + } + [40]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(9) + } + [41]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(9) + } + [42]=> + string(1) ";" + [43]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(9) + } + [44]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "else" + [2]=> + int(10) + } + [45]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) " + " + [2]=> + int(10) + } + [46]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "exit" + [2]=> + int(11) + } + [47]=> + string(1) ";" + [48]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(11) + } + [49]=> + string(1) "}" + [50]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) " + +" + [2]=> + int(12) + } + [51]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(14) + } + [52]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(14) + } + [53]=> + string(1) "=" + [54]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(14) + } + [55]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "VALUE" + [2]=> + int(14) + } + [56]=> + string(1) ";" + [57]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(14) + } + [58]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(15) + } + [59]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(15) + } + [60]=> + string(1) "=" + [61]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(15) + } + [62]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "20" + [2]=> + int(15) + } + [63]=> + string(1) ";" + [64]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(15) + } + [65]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$c" + [2]=> + int(16) + } + [66]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(16) + } + [67]=> + string(1) "=" + [68]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(16) + } + [69]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "array" + [2]=> + int(16) + } + [70]=> + string(1) "(" + [71]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(16) + } + [72]=> + string(1) "," + [73]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(16) + } + [74]=> + string(1) ")" + [75]=> + string(1) ";" + [76]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(16) + } + [77]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(17) + } + [78]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(17) + } + [79]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) ">>=" + [2]=> + int(17) + } + [80]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(17) + } + [81]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(17) + } + [82]=> + string(1) ";" + [83]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) " + +" + [2]=> + int(17) + } + [84]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "if" + [2]=> + int(19) + } + [85]=> + string(1) "(" + [86]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(19) + } + [87]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(19) + } + [88]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "<=" + [2]=> + int(19) + } + [89]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(19) + } + [90]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "0" + [2]=> + int(19) + } + [91]=> + string(1) ")" + [92]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(19) + } + [93]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "die" + [2]=> + int(20) + } + [94]=> + string(1) ";" + [95]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(20) + } + [96]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "else" + [2]=> + int(21) + } + [97]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(21) + } + [98]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "print" + [2]=> + int(22) + } + [99]=> + string(1) "(" + [100]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(22) + } + [101]=> + string(1) ")" + [102]=> + string(1) ";" + [103]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) " + +" + [2]=> + int(22) + } + [104]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "list" + [2]=> + int(24) + } + [105]=> + string(1) "(" + [106]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "$value1" + [2]=> + int(24) + } + [107]=> + string(1) "," + [108]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "$value2" + [2]=> + int(24) + } + [109]=> + string(1) ")" + [110]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(24) + } + [111]=> + string(1) "=" + [112]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(24) + } + [113]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$c" + [2]=> + int(24) + } + [114]=> + string(1) ";" + [115]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(24) + } + [116]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "if" + [2]=> + int(25) + } + [117]=> + string(1) "(" + [118]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "empty" + [2]=> + int(25) + } + [119]=> + string(1) "(" + [120]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "$value1" + [2]=> + int(25) + } + [121]=> + string(1) ")" + [122]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(25) + } + [123]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "&&" + [2]=> + int(25) + } + [124]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(25) + } + [125]=> + string(1) "!" + [126]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "isset" + [2]=> + int(25) + } + [127]=> + string(1) "(" + [128]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "$value1" + [2]=> + int(25) + } + [129]=> + string(1) ")" + [130]=> + string(1) ")" + [131]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(25) + } + [132]=> + string(1) "{" + [133]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(25) + } + [134]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(10) "myFunction" + [2]=> + int(26) + } + [135]=> + string(1) "(" + [136]=> + string(1) ")" + [137]=> + string(1) ";" + [138]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(26) + } + [139]=> + string(1) "}" + [140]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(27) + } + [141]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(28) + } +} +Done diff --git a/ext/tokenizer/tests/token_get_all_variation17.phpt b/ext/tokenizer/tests/token_get_all_variation17.phpt new file mode 100644 index 0000000..dccc4c9 --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_variation17.phpt @@ -0,0 +1,613 @@ +--TEST-- +Test token_get_all() function : usage variations - with exception keywords +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description: splits the given source into an array of PHP languange tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +/* + * Testing token_get_all() with different exception keywords + * try - T_TRY(336), + * catch - T_CATCH(337), + * throw - T_THROW(338) +*/ + +echo "*** Testing token_get_all() : with exception keywords ***\n"; + +$source = '<?php +function inverse($x) +{ + if($x == 0) { + throw new Exception("Divison by zero"); + else + return 1/$x; +} +try { + echo inverse(0); + echo inverse(5); +} catch(Exception $e) { + echo "caught exception:"; +} +?>'; +$tokens = token_get_all($source); +var_dump($tokens); + +echo "Done" +?> +--EXPECTF-- +*** Testing token_get_all() : with exception keywords *** +array(81) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php +" + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "function" + [2]=> + int(2) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "inverse" + [2]=> + int(2) + } + [4]=> + string(1) "(" + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$x" + [2]=> + int(2) + } + [6]=> + string(1) ")" + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(2) + } + [8]=> + string(1) "{" + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(%d) + } + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "if" + [2]=> + int(4) + } + [11]=> + string(1) "(" + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$x" + [2]=> + int(4) + } + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [14]=> + array(3) { + [0]=> + int(283) + [1]=> + string(2) "==" + [2]=> + int(4) + } + [15]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "0" + [2]=> + int(4) + } + [17]=> + string(1) ")" + [18]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [19]=> + string(1) "{" + [20]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) " + " + [2]=> + int(4) + } + [21]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "throw" + [2]=> + int(5) + } + [22]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(5) + } + [23]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "new" + [2]=> + int(5) + } + [24]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(5) + } + [25]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(9) "Exception" + [2]=> + int(5) + } + [26]=> + string(1) "(" + [27]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(17) ""Divison by zero"" + [2]=> + int(5) + } + [28]=> + string(1) ")" + [29]=> + string(1) ";" + [30]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(5) + } + [31]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "else" + [2]=> + int(6) + } + [32]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) " + " + [2]=> + int(6) + } + [33]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "return" + [2]=> + int(7) + } + [34]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(7) + } + [35]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(7) + } + [36]=> + string(1) "/" + [37]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$x" + [2]=> + int(7) + } + [38]=> + string(1) ";" + [39]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(7) + } + [40]=> + string(1) "}" + [41]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(8) + } + [42]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "try" + [2]=> + int(9) + } + [43]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(9) + } + [44]=> + string(1) "{" + [45]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(9) + } + [46]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(10) + } + [47]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(10) + } + [48]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "inverse" + [2]=> + int(10) + } + [49]=> + string(1) "(" + [50]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "0" + [2]=> + int(10) + } + [51]=> + string(1) ")" + [52]=> + string(1) ";" + [53]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(10) + } + [54]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(11) + } + [55]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(11) + } + [56]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "inverse" + [2]=> + int(11) + } + [57]=> + string(1) "(" + [58]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "5" + [2]=> + int(11) + } + [59]=> + string(1) ")" + [60]=> + string(1) ";" + [61]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(11) + } + [62]=> + string(1) "}" + [63]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(12) + } + [64]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "catch" + [2]=> + int(12) + } + [65]=> + string(1) "(" + [66]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(9) "Exception" + [2]=> + int(12) + } + [67]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(12) + } + [68]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$e" + [2]=> + int(12) + } + [69]=> + string(1) ")" + [70]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(12) + } + [71]=> + string(1) "{" + [72]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) " + " + [2]=> + int(12) + } + [73]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(13) + } + [74]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(13) + } + [75]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(19) ""caught exception:"" + [2]=> + int(13) + } + [76]=> + string(1) ";" + [77]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(13) + } + [78]=> + string(1) "}" + [79]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(14) + } + [80]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(15) + } +} +Done diff --git a/ext/tokenizer/tests/token_get_all_variation18.phpt b/ext/tokenizer/tests/token_get_all_variation18.phpt new file mode 100644 index 0000000..01219ff --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_variation18.phpt @@ -0,0 +1,120 @@ +--TEST-- +Test token_get_all() function : usage variations - with HTML code +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description: splits the given source into an array of PHP languange tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +/* + * Testing token_get_all() with source string containing HTML code with PHP + * HTML tags are considered to be T_INLINE_HTML(311) +*/ + +echo "*** Testing token_get_all() : 'source' string with HTML tags ***\n"; + +$source = ' +<html> +<body> + Testing HTML +</body> +</html>" + +<?php + echo "php code with HTML"; +?>'; +var_dump( token_get_all($source)); + +echo "Done" +?> +--EXPECTF-- +*** Testing token_get_all() : 'source' string with HTML tags *** +array(9) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(48) " +<html> +<body> + Testing HTML +</body> +</html>" + +" + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(8) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(8) + } + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(9) + } + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(9) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(20) ""php code with HTML"" + [2]=> + int(9) + } + [6]=> + string(1) ";" + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(9) + } + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(10) + } +} +Done diff --git a/ext/tokenizer/tests/token_get_all_variation19.phpt b/ext/tokenizer/tests/token_get_all_variation19.phpt new file mode 100644 index 0000000..9ae6759 --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_variation19.phpt @@ -0,0 +1,90 @@ +--TEST-- +Reconstructing a script using token_get_all() +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php + +$phpstr = ' +<?php + +// A php script to test token_get_all() + +/* a different +type of +comment */ + +// a class +class TestClass { + public function foo() { + echo "Called foo()\n"; + } +} + +$a = new TestClass(); +$a->foo(); + +for ($i = 0; $i < 10; $i++) { + echo "Loop iteration $i\n"; +} + +?>'; + +$token_array = token_get_all($phpstr); + +$script = ""; +// reconstruct a script (without open/close tags) from the token array +foreach ($token_array as $token) { + if (is_array($token)) { + if (strncmp($token[1], '<?php', 5) == 0) { + continue; + } + if (strncmp($token[1], '?>', 2) == 0) { + continue; + } + $script .= $token[1]; + } else { + $script .= $token; + } +} + +var_dump($script); + +eval($script); + +?> +--EXPECT-- +string(259) " + +// A php script to test token_get_all() + +/* a different +type of +comment */ + +// a class +class TestClass { + public function foo() { + echo "Called foo()\n"; + } +} + +$a = new TestClass(); +$a->foo(); + +for ($i = 0; $i < 10; $i++) { + echo "Loop iteration $i\n"; +} + +" +Called foo() +Loop iteration 0 +Loop iteration 1 +Loop iteration 2 +Loop iteration 3 +Loop iteration 4 +Loop iteration 5 +Loop iteration 6 +Loop iteration 7 +Loop iteration 8 +Loop iteration 9 diff --git a/ext/tokenizer/tests/token_get_all_variation2.phpt b/ext/tokenizer/tests/token_get_all_variation2.phpt new file mode 100644 index 0000000..1f20626 --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_variation2.phpt @@ -0,0 +1,431 @@ +--TEST-- +Test token_get_all() function : usage variations - with different arithmetic operators +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description: splits the given source into an array of PHP languange tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +/* + * Passing 'source' argument with different arithmetic operators to test them for token + * Arithmetic operators: +, -, *, /, % are not listed as specific operator tokens, + * so they are expected to return string - T_STRING +*/ + +echo "*** Testing token_get_all() : 'source' string with different arithmetic operators ***\n"; + +// arithmetic operators - '+', '-', '*', '/', '%' +$source = array ( + '<?php $a = 1 + 2; ?>', + '<?php $b = $b - 2; ?>', + '<?php $c = $a * $b; ?>', + '<?php $a = $b % 2; ?>' +); +for($count = 0; $count < count($source); $count++) { + echo "-- Iteration ".($count + 1)." --\n"; + var_dump( token_get_all($source[$count])); +} +echo "Done" +?> +--EXPECTF-- +*** Testing token_get_all() : 'source' string with different arithmetic operators *** +-- Iteration 1 -- +array(13) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [3]=> + string(1) "=" + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(1) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [7]=> + string(1) "+" + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(1) + } + [10]=> + string(1) ";" + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(1) + } +} +-- Iteration 2 -- +array(13) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [3]=> + string(1) "=" + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(1) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [7]=> + string(1) "-" + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(1) + } + [10]=> + string(1) ";" + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(1) + } +} +-- Iteration 3 -- +array(13) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$c" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [3]=> + string(1) "=" + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [7]=> + string(1) "*" + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(1) + } + [10]=> + string(1) ";" + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(1) + } +} +-- Iteration 4 -- +array(13) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [3]=> + string(1) "=" + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(1) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [7]=> + string(1) "%" + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(1) + } + [10]=> + string(1) ";" + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(1) + } +} +Done diff --git a/ext/tokenizer/tests/token_get_all_variation3.phpt b/ext/tokenizer/tests/token_get_all_variation3.phpt new file mode 100644 index 0000000..66cf4aa --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_variation3.phpt @@ -0,0 +1,570 @@ +--TEST-- +Test token_get_all() function : usage variations - with logical operators +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description: splits the given source into an array of PHP languange tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +/* + * Passing 'source' argument with different logical operators to test them for tokens + * and - T_AND_LOGICAL_AND(265), + * or - T_LOGICAL_OR(263), + * xor - T_LOGICAL_XOR(264), + * && - T_BOOLEAN_AND(279), + * || - T_BOOLEAN_OR(278) +*/ + +echo "*** Testing token_get_all() : 'source' string with different logical operators ***\n"; + +// logical operators : 'and', 'or', 'xor', '&&', '||' +$source = array ( + '<?php $a = 1 and 024; ?>', + '<?php $b = $b or 0X1E; ?>', + '<?php $c = $a xor $b; ?>', + '<?php $a = $b && 2; ?>', + '<?php $b = $b || 1; ?>' +); +for($count = 0; $count < count($source); $count++) { + echo "-- Iteration ".($count + 1)." --\n"; + var_dump( token_get_all($source[$count])); +} + +echo "Done" +?> +--EXPECTF-- +*** Testing token_get_all() : 'source' string with different logical operators *** +-- Iteration 1 -- +array(13) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [3]=> + string(1) "=" + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(1) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "and" + [2]=> + int(1) + } + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "024" + [2]=> + int(1) + } + [10]=> + string(1) ";" + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(1) + } +} +-- Iteration 2 -- +array(13) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [3]=> + string(1) "=" + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(1) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "or" + [2]=> + int(1) + } + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "0X1E" + [2]=> + int(1) + } + [10]=> + string(1) ";" + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(1) + } +} +-- Iteration 3 -- +array(13) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$c" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [3]=> + string(1) "=" + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "xor" + [2]=> + int(1) + } + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(1) + } + [10]=> + string(1) ";" + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(1) + } +} +-- Iteration 4 -- +array(13) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [3]=> + string(1) "=" + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(1) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "&&" + [2]=> + int(1) + } + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(1) + } + [10]=> + string(1) ";" + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(1) + } +} +-- Iteration 5 -- +array(13) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [3]=> + string(1) "=" + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(1) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "||" + [2]=> + int(1) + } + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(1) + } + [10]=> + string(1) ";" + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(1) + } + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(1) + } +} +Done diff --git a/ext/tokenizer/tests/token_get_all_variation4.phpt b/ext/tokenizer/tests/token_get_all_variation4.phpt new file mode 100644 index 0000000..45e6f8a --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_variation4.phpt @@ -0,0 +1,749 @@ +--TEST-- +Test token_get_all() function : usage variations - with comparison operators +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description: splits the given source into an array of PHP languange tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +/* + * Passing 'source' argument with different comparison operators to test them for tokens + * == - T_IS_EQUAL(283), === - T_IS_IDENTICAL(281), + * >= - T_IS_GREATER_OR_EQUAL(284), <= - T_IS_LESS_OR_EQUAL(285) + * != - T_IS_NOT_EQUAL, <> - T_IS_NOT_EQUAL(282), !== - T_IS_NOT_IDENTICAL(280) +*/ + +echo "*** Testing token_get_all() : 'source' string with different comparison operators ***\n"; + +// comparison operators : '==', '===', '>=', '<=', '!=', '!==', '<>' +$source = '<?php +if($a == 0) + echo "== 0"; +elseif($a === 2) + echo "=== 2"; +elseif($a >= 10 && $a <= 20) + echo ">= 10 & <=20"; +elseif($a != 1 || $a <> 1) + echo "!= 1"; +elseif($a !== 1) + echo "!==1"; +?>'; +var_dump( token_get_all($source)); + +echo "Done" +?> +--EXPECTF-- +*** Testing token_get_all() : 'source' string with different comparison operators *** +array(89) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "if" + [2]=> + int(2) + } + [3]=> + string(1) "(" + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(2) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "==" + [2]=> + int(2) + } + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "0" + [2]=> + int(2) + } + [9]=> + string(1) ")" + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) " + " + [2]=> + int(2) + } + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(%d) + } + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) ""== 0"" + [2]=> + int(%d) + } + [14]=> + string(1) ";" + [15]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(%d) + } + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "elseif" + [2]=> + int(4) + } + [17]=> + string(1) "(" + [18]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(4) + } + [19]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [20]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "===" + [2]=> + int(4) + } + [21]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [22]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(4) + } + [23]=> + string(1) ")" + [24]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(4) + } + [25]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(5) + } + [26]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(5) + } + [27]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) ""=== 2"" + [2]=> + int(5) + } + [28]=> + string(1) ";" + [29]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(5) + } + [30]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "elseif" + [2]=> + int(6) + } + [31]=> + string(1) "(" + [32]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(6) + } + [33]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(6) + } + [34]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) ">=" + [2]=> + int(6) + } + [35]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(6) + } + [36]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "10" + [2]=> + int(6) + } + [37]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(6) + } + [38]=> + array(3) { + [0]=> + int(279) + [1]=> + string(2) "&&" + [2]=> + int(6) + } + [39]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(6) + } + [40]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(6) + } + [41]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(6) + } + [42]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "<=" + [2]=> + int(6) + } + [43]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(6) + } + [44]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "20" + [2]=> + int(6) + } + [45]=> + string(1) ")" + [46]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(6) + } + [47]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(7) + } + [48]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(7) + } + [49]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(14) "">= 10 & <=20"" + [2]=> + int(7) + } + [50]=> + string(1) ";" + [51]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(7) + } + [52]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "elseif" + [2]=> + int(8) + } + [53]=> + string(1) "(" + [54]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(8) + } + [55]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [56]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "!=" + [2]=> + int(8) + } + [57]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [58]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(8) + } + [59]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [60]=> + array(3) { + [0]=> + int(278) + [1]=> + string(2) "||" + [2]=> + int(8) + } + [61]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [62]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(8) + } + [63]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [64]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "<>" + [2]=> + int(8) + } + [65]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [66]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(8) + } + [67]=> + string(1) ")" + [68]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(8) + } + [69]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(9) + } + [70]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(9) + } + [71]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) ""!= 1"" + [2]=> + int(9) + } + [72]=> + string(1) ";" + [73]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(9) + } + [74]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "elseif" + [2]=> + int(10) + } + [75]=> + string(1) "(" + [76]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(10) + } + [77]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(10) + } + [78]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) "!==" + [2]=> + int(10) + } + [79]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(10) + } + [80]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(10) + } + [81]=> + string(1) ")" + [82]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(3) " + " + [2]=> + int(10) + } + [83]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(11) + } + [84]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(11) + } + [85]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) ""!==1"" + [2]=> + int(11) + } + [86]=> + string(1) ";" + [87]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(11) + } + [88]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(12) + } +} +Done diff --git a/ext/tokenizer/tests/token_get_all_variation5.phpt b/ext/tokenizer/tests/token_get_all_variation5.phpt new file mode 100644 index 0000000..0068f28 --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_variation5.phpt @@ -0,0 +1,800 @@ +--TEST-- +Test token_get_all() function : usage variations - with assignment operators +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description: splits the given source into an array of PHP languange tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +/* + * Passing 'source' argument with different assignment operators to test them for tokens + * += - T_PLUS_EQUAL(277), -= - T_MINUS-EQUAL(276), + * *= - T_MUL_EQUAL(275), /= - T_DIVIDE_EQUAL(274), + * %= - T_MOD_EQUAL(272), &= - T_AND_EQUAL(271), + * |= - T_OR_EQUAL(271), ^= - T_EXOR_EQUAL(269), + * >>= - T_SR_EQUAL(267), <<= - T_SL_EQUAL(268), .= - T_CONCAT_EQUAL(273) +*/ + +echo "*** Testing token_get_all() : 'source' string with different assignment operators ***\n"; + +// assignment operators : '+=', '-=', '*=', '/=', '%=', '&=', '|=', '^=', '>>=', '<<=', '.=' +$source = '<?php +$a = 1, $b = 2; +$c += $b; +$b -= $a; +$a *= 2; +$d /= 10.50; +$a %= 10.50; +$b &= $c; +$c |= 1; +$d ^= 5; +$a >>= 1; +$b <<= 2; +$d .= "hello world"; +?>'; +var_dump( token_get_all($source)); + +echo "Done" +?> +--EXPECTF-- +*** Testing token_get_all() : 'source' string with different assignment operators *** +array(94) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(2) + } + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [4]=> + string(1) "=" + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(2) + } + [7]=> + string(1) "," + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(2) + } + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [11]=> + string(1) "=" + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(2) + } + [14]=> + string(1) ";" + [15]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(2) + } + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$c" + [2]=> + int(%d) + } + [17]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [18]=> + array(3) { + [0]=> + int(277) + [1]=> + string(2) "+=" + [2]=> + int(%d) + } + [19]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [20]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(%d) + } + [21]=> + string(1) ";" + [22]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(%d) + } + [23]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(4) + } + [24]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [25]=> + array(3) { + [0]=> + int(276) + [1]=> + string(2) "-=" + [2]=> + int(4) + } + [26]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [27]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(4) + } + [28]=> + string(1) ";" + [29]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(4) + } + [30]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(5) + } + [31]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(5) + } + [32]=> + array(3) { + [0]=> + int(275) + [1]=> + string(2) "*=" + [2]=> + int(5) + } + [33]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(5) + } + [34]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(5) + } + [35]=> + string(1) ";" + [36]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(5) + } + [37]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$d" + [2]=> + int(6) + } + [38]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(6) + } + [39]=> + array(3) { + [0]=> + int(274) + [1]=> + string(2) "/=" + [2]=> + int(6) + } + [40]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(6) + } + [41]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "10.50" + [2]=> + int(6) + } + [42]=> + string(1) ";" + [43]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(6) + } + [44]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(7) + } + [45]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(7) + } + [46]=> + array(3) { + [0]=> + int(272) + [1]=> + string(2) "%=" + [2]=> + int(7) + } + [47]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(7) + } + [48]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "10.50" + [2]=> + int(7) + } + [49]=> + string(1) ";" + [50]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(7) + } + [51]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(8) + } + [52]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [53]=> + array(3) { + [0]=> + int(271) + [1]=> + string(2) "&=" + [2]=> + int(8) + } + [54]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [55]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$c" + [2]=> + int(8) + } + [56]=> + string(1) ";" + [57]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(8) + } + [58]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$c" + [2]=> + int(9) + } + [59]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(9) + } + [60]=> + array(3) { + [0]=> + int(270) + [1]=> + string(2) "|=" + [2]=> + int(9) + } + [61]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(9) + } + [62]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(9) + } + [63]=> + string(1) ";" + [64]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(9) + } + [65]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$d" + [2]=> + int(10) + } + [66]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(10) + } + [67]=> + array(3) { + [0]=> + int(269) + [1]=> + string(2) "^=" + [2]=> + int(10) + } + [68]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(10) + } + [69]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "5" + [2]=> + int(10) + } + [70]=> + string(1) ";" + [71]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(10) + } + [72]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(11) + } + [73]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(11) + } + [74]=> + array(3) { + [0]=> + int(267) + [1]=> + string(3) ">>=" + [2]=> + int(11) + } + [75]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(11) + } + [76]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(11) + } + [77]=> + string(1) ";" + [78]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(11) + } + [79]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(12) + } + [80]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(12) + } + [81]=> + array(3) { + [0]=> + int(268) + [1]=> + string(3) "<<=" + [2]=> + int(12) + } + [82]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(12) + } + [83]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(12) + } + [84]=> + string(1) ";" + [85]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(12) + } + [86]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$d" + [2]=> + int(13) + } + [87]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(13) + } + [88]=> + array(3) { + [0]=> + int(273) + [1]=> + string(2) ".=" + [2]=> + int(13) + } + [89]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(13) + } + [90]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(13) ""hello world"" + [2]=> + int(13) + } + [91]=> + string(1) ";" + [92]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(13) + } + [93]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(14) + } +} +Done diff --git a/ext/tokenizer/tests/token_get_all_variation6.phpt b/ext/tokenizer/tests/token_get_all_variation6.phpt new file mode 100644 index 0000000..54936d0 --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_variation6.phpt @@ -0,0 +1,394 @@ +--TEST-- +Test token_get_all() function : usage variations - with bitwise operators +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description: splits the given source into an array of PHP languange tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +/* + * Passing 'source' argument with different bitwise operators to test them for token + * << - T_SL(287) + * >> - T_SR(286) +*/ + +echo "*** Testing token_get_all() : 'source' string with different bitwise operators ***\n"; + +// bitwise operators : '<<' , '>>' +$source = '<?php +$a = 2, $b = 4; +$a = $a << 2; +$b = $b >> 2; +var_dump($a); +var_dump($b); +?>'; +var_dump( token_get_all($source)); + +echo "Done" +?> +--EXPECTF-- +*** Testing token_get_all() : 'source' string with different bitwise operators *** +array(50) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php +" + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(2) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [3]=> + string(1) "=" + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(2) + } + [6]=> + string(1) "," + [7]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(2) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [10]=> + string(1) "=" + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "4" + [2]=> + int(2) + } + [13]=> + string(1) ";" + [14]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(2) + } + [15]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(%d) + } + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [17]=> + string(1) "=" + [18]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [19]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(%d) + } + [20]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [21]=> + array(3) { + [0]=> + int(287) + [1]=> + string(2) "<<" + [2]=> + int(%d) + } + [22]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [23]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(%d) + } + [24]=> + string(1) ";" + [25]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(%d) + } + [26]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(4) + } + [27]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [28]=> + string(1) "=" + [29]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [30]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(4) + } + [31]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [32]=> + array(3) { + [0]=> + int(286) + [1]=> + string(2) ">>" + [2]=> + int(4) + } + [33]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [34]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "2" + [2]=> + int(4) + } + [35]=> + string(1) ";" + [36]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(4) + } + [37]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "var_dump" + [2]=> + int(5) + } + [38]=> + string(1) "(" + [39]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(5) + } + [40]=> + string(1) ")" + [41]=> + string(1) ";" + [42]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(5) + } + [43]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "var_dump" + [2]=> + int(6) + } + [44]=> + string(1) "(" + [45]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(6) + } + [46]=> + string(1) ")" + [47]=> + string(1) ";" + [48]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(6) + } + [49]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(7) + } +} +Done diff --git a/ext/tokenizer/tests/token_get_all_variation7.phpt b/ext/tokenizer/tests/token_get_all_variation7.phpt new file mode 100644 index 0000000..36fda32 --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_variation7.phpt @@ -0,0 +1,261 @@ +--TEST-- +Test token_get_all() function : usage variations - with increment/decrement operators +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description: splits the given source into an array of PHP languange tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +/* + * Passing 'source' argument with different increment/decrement operators to test them for token + * ++ - T_INC(297) + * -- - T_DEC(296) +*/ + +echo "*** Testing token_get_all() : 'source' string with different increment/decrement operators ***\n"; + +// increment/decrement operators : '++' , '--' +$source = '<?php +$a = 10, $b = 5; +$a++; +$b--; +echo $a; +?>'; +var_dump( token_get_all($source)); + +echo "Done" +?> +--EXPECTF-- +*** Testing token_get_all() : 'source' string with different increment/decrement operators *** +array(30) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(2) + } + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [4]=> + string(1) "=" + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "10" + [2]=> + int(2) + } + [7]=> + string(1) "," + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(2) + } + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [11]=> + string(1) "=" + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "5" + [2]=> + int(2) + } + [14]=> + string(1) ";" + [15]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(2) + } + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(%d) + } + [17]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "++" + [2]=> + int(%d) + } + [18]=> + string(1) ";" + [19]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(%d) + } + [20]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(4) + } + [21]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "--" + [2]=> + int(4) + } + [22]=> + string(1) ";" + [23]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(4) + } + [24]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(5) + } + [25]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(5) + } + [26]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(5) + } + [27]=> + string(1) ";" + [28]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(5) + } + [29]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(6) + } +} +Done diff --git a/ext/tokenizer/tests/token_get_all_variation8.phpt b/ext/tokenizer/tests/token_get_all_variation8.phpt new file mode 100644 index 0000000..0cf1d63 --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_variation8.phpt @@ -0,0 +1,834 @@ +--TEST-- +Test token_get_all() function : usage variations - with type casting operators +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description: splits the given source into an array of PHP languange tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +/* + * Passing 'source' argument with different type casting operators to test them for token + * (int)/(integer) - T_INT_CAST(295), (float)/(real)/(double) - T_DOUBLE_CAST(294), + * (string) - T_STRING_CAST(293), (bool)/(boolean) - T_BOOL_CAST(290), + * (unset) - T_UNSET_CAST(289) +*/ + +echo "*** Testing token_get_all() : 'source' string with different type casting operators ***\n"; + +// type casting operators : (int), (integer), (float), (real), (double), (string), (array), (object), (bool), (boolean),(unset) +$source = '<?php +$a = 1, $b = 10.5 +$c = (int)$b + $a; +$d = (float)$a + $b; +$e = (string)$a.(string)$b; +if((bool)$a) echo "true"; +if(!(boolean)$b) echo "false"; +$c = $c + (integer) 123.4e2; +$d = $c - (real) 12; +$b = (unset)$a; +?>'; +var_dump( token_get_all($source)); + +echo "Done" +?> +--EXPECTF-- +*** Testing token_get_all() : 'source' string with different type casting operators *** +array(108) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(2) + } + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [4]=> + string(1) "=" + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) "1" + [2]=> + int(2) + } + [7]=> + string(1) "," + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(2) + } + [10]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [11]=> + string(1) "=" + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(2) + } + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "10.5" + [2]=> + int(2) + } + [14]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(2) + } + [15]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$c" + [2]=> + int(%d) + } + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [17]=> + string(1) "=" + [18]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [19]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(5) "(int)" + [2]=> + int(%d) + } + [20]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(%d) + } + [21]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [22]=> + string(1) "+" + [23]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(%d) + } + [24]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(%d) + } + [25]=> + string(1) ";" + [26]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(%d) + } + [27]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$d" + [2]=> + int(4) + } + [28]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [29]=> + string(1) "=" + [30]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [31]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "(float)" + [2]=> + int(4) + } + [32]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(4) + } + [33]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [34]=> + string(1) "+" + [35]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(4) + } + [36]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(4) + } + [37]=> + string(1) ";" + [38]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(4) + } + [39]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$e" + [2]=> + int(5) + } + [40]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(5) + } + [41]=> + string(1) "=" + [42]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(5) + } + [43]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "(string)" + [2]=> + int(5) + } + [44]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(5) + } + [45]=> + string(1) "." + [46]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "(string)" + [2]=> + int(5) + } + [47]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(5) + } + [48]=> + string(1) ";" + [49]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(5) + } + [50]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "if" + [2]=> + int(6) + } + [51]=> + string(1) "(" + [52]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "(bool)" + [2]=> + int(6) + } + [53]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(6) + } + [54]=> + string(1) ")" + [55]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(6) + } + [56]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(6) + } + [57]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(6) + } + [58]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) ""true"" + [2]=> + int(6) + } + [59]=> + string(1) ";" + [60]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(6) + } + [61]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "if" + [2]=> + int(7) + } + [62]=> + string(1) "(" + [63]=> + string(1) "!" + [64]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(9) "(boolean)" + [2]=> + int(7) + } + [65]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(7) + } + [66]=> + string(1) ")" + [67]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(7) + } + [68]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "echo" + [2]=> + int(7) + } + [69]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(7) + } + [70]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) ""false"" + [2]=> + int(7) + } + [71]=> + string(1) ";" + [72]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(7) + } + [73]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$c" + [2]=> + int(8) + } + [74]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [75]=> + string(1) "=" + [76]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [77]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$c" + [2]=> + int(8) + } + [78]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [79]=> + string(1) "+" + [80]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [81]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(9) "(integer)" + [2]=> + int(8) + } + [82]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [83]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(7) "123.4e2" + [2]=> + int(8) + } + [84]=> + string(1) ";" + [85]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(8) + } + [86]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$d" + [2]=> + int(9) + } + [87]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(9) + } + [88]=> + string(1) "=" + [89]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(9) + } + [90]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$c" + [2]=> + int(9) + } + [91]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(9) + } + [92]=> + string(1) "-" + [93]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(9) + } + [94]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "(real)" + [2]=> + int(9) + } + [95]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(9) + } + [96]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "12" + [2]=> + int(9) + } + [97]=> + string(1) ";" + [98]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(9) + } + [99]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(10) + } + [100]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(10) + } + [101]=> + string(1) "=" + [102]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(10) + } + [103]=> + array(3) { + [0]=> + int(289) + [1]=> + string(7) "(unset)" + [2]=> + int(10) + } + [104]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(10) + } + [105]=> + string(1) ";" + [106]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(10) + } + [107]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(11) + } +} +Done diff --git a/ext/tokenizer/tests/token_get_all_variation9.phpt b/ext/tokenizer/tests/token_get_all_variation9.phpt new file mode 100644 index 0000000..1662fb2 --- /dev/null +++ b/ext/tokenizer/tests/token_get_all_variation9.phpt @@ -0,0 +1,443 @@ +--TEST-- +Test token_get_all() function : usage variations - with different types of comments +--SKIPIF-- +<?php if (!extension_loaded("tokenizer")) print "skip"; ?> +--FILE-- +<?php +/* Prototype : array token_get_all(string $source) + * Description: splits the given source into an array of PHP languange tokens + * Source code: ext/tokenizer/tokenizer.c +*/ + +/* + * Passing 'source' argument with different style of comments + */ +// '//', '/* */', '#' - T_COMMENT(365) +// '/** */' - T_DOC_COMMENT(366) + + +echo "*** Testing token_get_all() : 'source' string with different comments ***\n"; + +// types of comments: '//', '/* */', '#' & /** */ + +$source = '<?php +/** Performing addition operation on given values : + * a, b + */ + +// int value +$a = 10; +$b = 20; +$c = true; // bool value + +/* + * Performing operation on $a,$b + * display result + */ +$c = $a + $b; +var_dump($c); # expected: int(30) + +# end of program +?>'; +var_dump( token_get_all($source)); + +echo "Done" +?> +--EXPECTF-- +*** Testing token_get_all() : 'source' string with different comments *** +array(51) { + [0]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(6) "<?php " + [2]=> + int(1) + } + [1]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(1) + } + [2]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(65) "/** Performing addition operation on given values : + * a, b + */" + [2]=> + int(2) + } + [3]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) " + +" + [2]=> + int(4) + } + [4]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(13) "// int value +" + [2]=> + int(6) + } + [5]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(7) + } + [6]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(7) + } + [7]=> + string(1) "=" + [8]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(7) + } + [9]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "10" + [2]=> + int(7) + } + [10]=> + string(1) ";" + [11]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(7) + } + [12]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(8) + } + [13]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [14]=> + string(1) "=" + [15]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(8) + } + [16]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "20" + [2]=> + int(8) + } + [17]=> + string(1) ";" + [18]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(8) + } + [19]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$c" + [2]=> + int(9) + } + [20]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(9) + } + [21]=> + string(1) "=" + [22]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(9) + } + [23]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(4) "true" + [2]=> + int(9) + } + [24]=> + string(1) ";" + [25]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(9) + } + [26]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(14) "// bool value +" + [2]=> + int(9) + } + [27]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(10) + } + [28]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(59) "/* + * Performing operation on $a,$b + * display result + */" + [2]=> + int(11) + } + [29]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(14) + } + [30]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$c" + [2]=> + int(15) + } + [31]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(15) + } + [32]=> + string(1) "=" + [33]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(15) + } + [34]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$a" + [2]=> + int(15) + } + [35]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(15) + } + [36]=> + string(1) "+" + [37]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(15) + } + [38]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$b" + [2]=> + int(15) + } + [39]=> + string(1) ";" + [40]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(15) + } + [41]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(8) "var_dump" + [2]=> + int(16) + } + [42]=> + string(1) "(" + [43]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "$c" + [2]=> + int(16) + } + [44]=> + string(1) ")" + [45]=> + string(1) ";" + [46]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " " + [2]=> + int(16) + } + [47]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(20) "# expected: int(%d) +" + [2]=> + int(16) + } + [48]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(1) " +" + [2]=> + int(17) + } + [49]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(17) "# end of program +" + [2]=> + int(18) + } + [50]=> + array(3) { + [0]=> + int(%d) + [1]=> + string(2) "?>" + [2]=> + int(19) + } +} +Done diff --git a/ext/tokenizer/tokenizer.c b/ext/tokenizer/tokenizer.c new file mode 100644 index 0000000..132a7f2 --- /dev/null +++ b/ext/tokenizer/tokenizer.c @@ -0,0 +1,233 @@ +/* + +----------------------------------------------------------------------+ + | PHP Version 5 | + +----------------------------------------------------------------------+ + | Copyright (c) 1997-2013 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Author: Andrei Zmievski <andrei@php.net> | + +----------------------------------------------------------------------+ +*/ + +/* $Id$ */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "php.h" +#include "php_ini.h" +#include "ext/standard/info.h" +#include "php_tokenizer.h" + +#include "zend.h" +#include "zend_language_scanner.h" +#include "zend_language_scanner_defs.h" +#include <zend_language_parser.h> + +#define zendtext LANG_SCNG(yy_text) +#define zendleng LANG_SCNG(yy_leng) +#define zendcursor LANG_SCNG(yy_cursor) +#define zendlimit LANG_SCNG(yy_limit) + +/* {{{ arginfo */ +ZEND_BEGIN_ARG_INFO_EX(arginfo_token_get_all, 0, 0, 1) + ZEND_ARG_INFO(0, source) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_token_name, 0, 0, 1) + ZEND_ARG_INFO(0, token) +ZEND_END_ARG_INFO() +/* }}} */ + +/* {{{ tokenizer_functions[] + * + * Every user visible function must have an entry in tokenizer_functions[]. + */ +const zend_function_entry tokenizer_functions[] = { + PHP_FE(token_get_all, arginfo_token_get_all) + PHP_FE(token_name, arginfo_token_name) + PHP_FE_END +}; +/* }}} */ + +/* {{{ tokenizer_module_entry + */ +zend_module_entry tokenizer_module_entry = { +#if ZEND_MODULE_API_NO >= 20010901 + STANDARD_MODULE_HEADER, +#endif + "tokenizer", + tokenizer_functions, + PHP_MINIT(tokenizer), + NULL, + NULL, + NULL, + PHP_MINFO(tokenizer), +#if ZEND_MODULE_API_NO >= 20010901 + "0.1", /* Replace with version number for your extension */ +#endif + STANDARD_MODULE_PROPERTIES +}; +/* }}} */ + +#ifdef COMPILE_DL_TOKENIZER +ZEND_GET_MODULE(tokenizer) +#endif + +/* {{{ PHP_MINIT_FUNCTION + */ +PHP_MINIT_FUNCTION(tokenizer) +{ + tokenizer_register_constants(INIT_FUNC_ARGS_PASSTHRU); + return SUCCESS; +} +/* }}} */ + +/* {{{ PHP_MINFO_FUNCTION + */ +PHP_MINFO_FUNCTION(tokenizer) +{ + php_info_print_table_start(); + php_info_print_table_row(2, "Tokenizer Support", "enabled"); + php_info_print_table_end(); +} +/* }}} */ + +static void tokenize(zval *return_value TSRMLS_DC) +{ + zval token; + zval *keyword; + int token_type; + zend_bool destroy; + int token_line = 1; + int need_tokens = -1; // for __halt_compiler lexing. -1 = disabled + + array_init(return_value); + + ZVAL_NULL(&token); + while ((token_type = lex_scan(&token TSRMLS_CC))) { + destroy = 1; + switch (token_type) { + case T_CLOSE_TAG: + if (zendtext[zendleng - 1] != '>') { + CG(zend_lineno)++; + } + case T_OPEN_TAG: + case T_OPEN_TAG_WITH_ECHO: + case T_WHITESPACE: + case T_COMMENT: + case T_DOC_COMMENT: + destroy = 0; + break; + } + + if (token_type >= 256) { + MAKE_STD_ZVAL(keyword); + array_init(keyword); + add_next_index_long(keyword, token_type); + if (token_type == T_END_HEREDOC) { + if (CG(increment_lineno)) { + token_line = ++CG(zend_lineno); + CG(increment_lineno) = 0; + } + add_next_index_stringl(keyword, Z_STRVAL(token), Z_STRLEN(token), 1); + efree(Z_STRVAL(token)); + } else { + add_next_index_stringl(keyword, (char *)zendtext, zendleng, 1); + } + add_next_index_long(keyword, token_line); + add_next_index_zval(return_value, keyword); + } else { + add_next_index_stringl(return_value, (char *)zendtext, zendleng, 1); + } + if (destroy && Z_TYPE(token) != IS_NULL) { + zval_dtor(&token); + } + ZVAL_NULL(&token); + + // after T_HALT_COMPILER collect the next three non-dropped tokens + if (need_tokens != -1) { + if (token_type != T_WHITESPACE && token_type != T_OPEN_TAG + && token_type != T_COMMENT && token_type != T_DOC_COMMENT + && --need_tokens == 0 + ) { + // fetch the rest into a T_INLINE_HTML + if (zendcursor != zendlimit) { + MAKE_STD_ZVAL(keyword); + array_init(keyword); + add_next_index_long(keyword, T_INLINE_HTML); + add_next_index_stringl(keyword, (char *)zendcursor, zendlimit - zendcursor, 1); + add_next_index_long(keyword, token_line); + add_next_index_zval(return_value, keyword); + } + break; + } + } else if (token_type == T_HALT_COMPILER) { + need_tokens = 3; + } + + token_line = CG(zend_lineno); + } +} + +/* {{{ proto array token_get_all(string source) + */ +PHP_FUNCTION(token_get_all) +{ + char *source = NULL; + int argc = ZEND_NUM_ARGS(); + int source_len; + zval source_z; + zend_lex_state original_lex_state; + + if (zend_parse_parameters(argc TSRMLS_CC, "s", &source, &source_len) == FAILURE) { + return; + } + + ZVAL_STRINGL(&source_z, source, source_len, 1); + zend_save_lexical_state(&original_lex_state TSRMLS_CC); + + if (zend_prepare_string_for_scanning(&source_z, "" TSRMLS_CC) == FAILURE) { + zend_restore_lexical_state(&original_lex_state TSRMLS_CC); + RETURN_FALSE; + } + + LANG_SCNG(yy_state) = yycINITIAL; + + tokenize(return_value TSRMLS_CC); + + zend_restore_lexical_state(&original_lex_state TSRMLS_CC); + zval_dtor(&source_z); +} +/* }}} */ + +/* {{{ proto string token_name(int type) + */ +PHP_FUNCTION(token_name) +{ + int argc = ZEND_NUM_ARGS(); + long type; + + if (zend_parse_parameters(argc TSRMLS_CC, "l", &type) == FAILURE) { + return; + } + RETVAL_STRING(get_token_type_name(type), 1); +} +/* }}} */ + +/* + * Local variables: + * tab-width: 4 + * c-basic-offset: 4 + * End: + * vim600: noet sw=4 ts=4 fdm=marker + * vim<600: noet sw=4 ts=4 + */ diff --git a/ext/tokenizer/tokenizer.dsp b/ext/tokenizer/tokenizer.dsp new file mode 100644 index 0000000..9844574 --- /dev/null +++ b/ext/tokenizer/tokenizer.dsp @@ -0,0 +1,108 @@ +# Microsoft Developer Studio Project File - Name="tokenizer" - Package Owner=<4>
+# Microsoft Developer Studio Generated Build File, Format Version 6.00
+# ** DO NOT EDIT **
+
+# TARGTYPE "Win32 (x86) Dynamic-Link Library" 0x0102
+
+CFG=tokenizer - Win32 Debug_TS
+!MESSAGE This is not a valid makefile. To build this project using NMAKE,
+!MESSAGE use the Export Makefile command and run
+!MESSAGE
+!MESSAGE NMAKE /f "tokenizer.mak".
+!MESSAGE
+!MESSAGE You can specify a configuration when running NMAKE
+!MESSAGE by defining the macro CFG on the command line. For example:
+!MESSAGE
+!MESSAGE NMAKE /f "tokenizer.mak" CFG="tokenizer - Win32 Debug_TS"
+!MESSAGE
+!MESSAGE Possible choices for configuration are:
+!MESSAGE
+!MESSAGE "tokenizer - Win32 Release_TS" (based on "Win32 (x86) Dynamic-Link Library")
+!MESSAGE "tokenizer - Win32 Debug_TS" (based on "Win32 (x86) Dynamic-Link Library")
+!MESSAGE
+
+# Begin Project
+# PROP AllowPerConfigDependencies 0
+# PROP Scc_ProjName ""
+# PROP Scc_LocalPath ""
+CPP=cl.exe
+MTL=midl.exe
+RSC=rc.exe
+
+!IF "$(CFG)" == "tokenizer - Win32 Release_TS"
+
+# PROP BASE Use_MFC 0
+# PROP BASE Use_Debug_Libraries 0
+# PROP BASE Output_Dir "Release_TS"
+# PROP BASE Intermediate_Dir "Release_TS"
+# PROP BASE Ignore_Export_Lib 0
+# PROP BASE Target_Dir ""
+# PROP Use_MFC 0
+# PROP Use_Debug_Libraries 0
+# PROP Output_Dir "Release_TS"
+# PROP Intermediate_Dir "Release_TS"
+# PROP Ignore_Export_Lib 0
+# PROP Target_Dir ""
+# ADD BASE CPP /nologo /MD /W3 /GX /O2 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "TOKENIZER_EXPORTS" /YX /FD /c
+# ADD CPP /nologo /MD /W3 /GX /O2 /I "..\.." /I "..\..\main" /I "..\..\Zend" /I "..\..\TSRM" /D ZEND_DEBUG=0 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /D "PHP_EXPORTS" /D "COMPILE_DL_TOKENIZER" /D ZTS=1 /D "ZEND_WIN32" /D "PHP_WIN32" /D HAVE_TOKENIZER=1 /YX /FD /c
+# ADD BASE MTL /nologo /D "NDEBUG" /mktyplib203 /win32
+# ADD MTL /nologo /D "NDEBUG" /mktyplib203 /win32
+# ADD BASE RSC /l 0x407 /d "NDEBUG"
+# ADD RSC /l 0x407 /d "NDEBUG"
+BSC32=bscmake.exe
+# ADD BASE BSC32 /nologo
+# ADD BSC32 /nologo
+LINK32=link.exe
+# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /machine:I386
+# ADD LINK32 php5ts.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /machine:I386 /out:"..\..\Release_TS/php_tokenizer.dll" /libpath:"..\..\Release_TS" /libpath:"..\..\Release_TS_Inline"
+
+!ELSEIF "$(CFG)" == "tokenizer - Win32 Debug_TS"
+
+# PROP BASE Use_MFC 0
+# PROP BASE Use_Debug_Libraries 1
+# PROP BASE Output_Dir "Debug_TS"
+# PROP BASE Intermediate_Dir "Debug_TS"
+# PROP BASE Target_Dir ""
+# PROP Use_MFC 0
+# PROP Use_Debug_Libraries 1
+# PROP Output_Dir "Debug_TS"
+# PROP Intermediate_Dir "Debug_TS"
+# PROP Ignore_Export_Lib 0
+# PROP Target_Dir ""
+# ADD BASE CPP /nologo /MDd /W3 /Gm /GX /ZI /Od /D "WIN32" /D "_DEBUG" /D "_WINDOWS" /D "_MBCS" /D "_USRDLL" /D "TOKENIZER_EXPORTS" /YX /FD /GZ /c
+# ADD CPP /nologo /MDd /W3 /Gm /GX /ZI /Od /I "..\.." /I "..\..\main" /I "..\..\Zend" /I "..\..\TSRM" /D ZEND_DEBUG=1 /D "WIN32" /D "NDEBUG" /D "_WINDOWS" /D "PHP_EXPORTS" /D "COMPILE_DL_TOKENIZER" /D ZTS=1 /D "ZEND_WIN32" /D "PHP_WIN32" /D HAVE_TOKENIZER=1 /YX /FD /GZ /c
+# ADD BASE MTL /nologo /D "_DEBUG" /mktyplib203 /win32
+# ADD MTL /nologo /D "_DEBUG" /mktyplib203 /win32
+# ADD BASE RSC /l 0x407 /d "_DEBUG"
+# ADD RSC /l 0x407 /d "_DEBUG"
+BSC32=bscmake.exe
+# ADD BASE BSC32 /nologo
+# ADD BSC32 /nologo
+LINK32=link.exe
+# ADD BASE LINK32 kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /debug /machine:I386 /pdbtype:sept
+# ADD LINK32 php5ts_debug.lib kernel32.lib user32.lib gdi32.lib winspool.lib comdlg32.lib advapi32.lib shell32.lib ole32.lib oleaut32.lib uuid.lib odbc32.lib odbccp32.lib /nologo /dll /debug /machine:I386 /out:"..\..\Debug_TS/php_tokenizer.dll" /pdbtype:sept /libpath:"..\..\Debug_TS"
+
+!ENDIF
+
+# Begin Target
+
+# Name "tokenizer - Win32 Release_TS"
+# Name "tokenizer - Win32 Debug_TS"
+# Begin Group "Source Files"
+
+# PROP Default_Filter "cpp;c;cxx;rc;def;r;odl;idl;hpj;bat"
+# Begin Source File
+
+SOURCE=.\tokenizer.c
+# End Source File
+# End Group
+# Begin Group "Header Files"
+
+# PROP Default_Filter "h;hpp;hxx;hm;inl"
+# Begin Source File
+
+SOURCE=.\php_tokenizer.h
+# End Source File
+# End Group
+# End Target
+# End Project
diff --git a/ext/tokenizer/tokenizer.php b/ext/tokenizer/tokenizer.php new file mode 100644 index 0000000..c13063c --- /dev/null +++ b/ext/tokenizer/tokenizer.php @@ -0,0 +1,35 @@ +<?php + +if(!extension_loaded('tokenizer')) { + dl('tokenizer.so'); +} + +$fp = fopen('php://stdin', 'r'); +while (!feof($fp)) { + $content .= fread($fp, 4096); +} +fclose($fp); + +$tokens = token_get_all($content); + +$count = count($tokens); +$state = 0; +for ($i = 0; $i < $count; $i++) { + $token = $tokens[$i]; + if (is_array($token)) { + if ($state == 1 && $token[0] == T_STRING) { + $token[1] = preg_replace('!([a-z])([A-Z])!e', '"$1_".strtolower("$2")', $token[1]); + $state = 0; + } else if ($token[0] == T_FUNCTION) { + $state = 1; + } + $chunk = $token[1]; + } else { + $chunk = $token; + } + $output .= $chunk; +} + +print $output; + +?> diff --git a/ext/tokenizer/tokenizer_data.c b/ext/tokenizer/tokenizer_data.c new file mode 100644 index 0000000..1f563da --- /dev/null +++ b/ext/tokenizer/tokenizer_data.c @@ -0,0 +1,297 @@ +/* + +----------------------------------------------------------------------+ + | PHP Version 5 | + +----------------------------------------------------------------------+ + | Copyright (c) 1997-2013 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Author: Johannes Schlueter <johannes@php.net> | + +----------------------------------------------------------------------+ +*/ + +/* $Id$ */ + +/* + DO NOT EDIT THIS FILE! + This file is generated using tokenizer_data_gen.sh +*/ + +#include "php.h" +#include "zend.h" +#include <zend_language_parser.h> + + +void tokenizer_register_constants(INIT_FUNC_ARGS) { + REGISTER_LONG_CONSTANT("T_REQUIRE_ONCE", T_REQUIRE_ONCE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_REQUIRE", T_REQUIRE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_EVAL", T_EVAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_INCLUDE_ONCE", T_INCLUDE_ONCE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_INCLUDE", T_INCLUDE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_LOGICAL_OR", T_LOGICAL_OR, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_LOGICAL_XOR", T_LOGICAL_XOR, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_LOGICAL_AND", T_LOGICAL_AND, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_PRINT", T_PRINT, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_SR_EQUAL", T_SR_EQUAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_SL_EQUAL", T_SL_EQUAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_XOR_EQUAL", T_XOR_EQUAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_OR_EQUAL", T_OR_EQUAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_AND_EQUAL", T_AND_EQUAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_MOD_EQUAL", T_MOD_EQUAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_CONCAT_EQUAL", T_CONCAT_EQUAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_DIV_EQUAL", T_DIV_EQUAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_MUL_EQUAL", T_MUL_EQUAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_MINUS_EQUAL", T_MINUS_EQUAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_PLUS_EQUAL", T_PLUS_EQUAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_BOOLEAN_OR", T_BOOLEAN_OR, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_BOOLEAN_AND", T_BOOLEAN_AND, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_IS_NOT_IDENTICAL", T_IS_NOT_IDENTICAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_IS_IDENTICAL", T_IS_IDENTICAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_IS_NOT_EQUAL", T_IS_NOT_EQUAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_IS_EQUAL", T_IS_EQUAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_IS_GREATER_OR_EQUAL", T_IS_GREATER_OR_EQUAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_IS_SMALLER_OR_EQUAL", T_IS_SMALLER_OR_EQUAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_SR", T_SR, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_SL", T_SL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_INSTANCEOF", T_INSTANCEOF, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_UNSET_CAST", T_UNSET_CAST, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_BOOL_CAST", T_BOOL_CAST, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_OBJECT_CAST", T_OBJECT_CAST, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ARRAY_CAST", T_ARRAY_CAST, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_STRING_CAST", T_STRING_CAST, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_DOUBLE_CAST", T_DOUBLE_CAST, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_INT_CAST", T_INT_CAST, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_DEC", T_DEC, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_INC", T_INC, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_CLONE", T_CLONE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_NEW", T_NEW, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_EXIT", T_EXIT, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_IF", T_IF, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ELSEIF", T_ELSEIF, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ELSE", T_ELSE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ENDIF", T_ENDIF, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_LNUMBER", T_LNUMBER, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_DNUMBER", T_DNUMBER, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_STRING", T_STRING, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_STRING_VARNAME", T_STRING_VARNAME, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_VARIABLE", T_VARIABLE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_NUM_STRING", T_NUM_STRING, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_INLINE_HTML", T_INLINE_HTML, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_CHARACTER", T_CHARACTER, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_BAD_CHARACTER", T_BAD_CHARACTER, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ENCAPSED_AND_WHITESPACE", T_ENCAPSED_AND_WHITESPACE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_CONSTANT_ENCAPSED_STRING", T_CONSTANT_ENCAPSED_STRING, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ECHO", T_ECHO, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_DO", T_DO, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_WHILE", T_WHILE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ENDWHILE", T_ENDWHILE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_FOR", T_FOR, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ENDFOR", T_ENDFOR, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_FOREACH", T_FOREACH, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ENDFOREACH", T_ENDFOREACH, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_DECLARE", T_DECLARE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ENDDECLARE", T_ENDDECLARE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_AS", T_AS, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_SWITCH", T_SWITCH, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ENDSWITCH", T_ENDSWITCH, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_CASE", T_CASE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_DEFAULT", T_DEFAULT, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_BREAK", T_BREAK, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_CONTINUE", T_CONTINUE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_GOTO", T_GOTO, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_FUNCTION", T_FUNCTION, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_CONST", T_CONST, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_RETURN", T_RETURN, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_TRY", T_TRY, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_CATCH", T_CATCH, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_THROW", T_THROW, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_USE", T_USE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_INSTEADOF", T_INSTEADOF, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_GLOBAL", T_GLOBAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_PUBLIC", T_PUBLIC, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_PROTECTED", T_PROTECTED, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_PRIVATE", T_PRIVATE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_FINAL", T_FINAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ABSTRACT", T_ABSTRACT, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_STATIC", T_STATIC, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_VAR", T_VAR, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_UNSET", T_UNSET, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ISSET", T_ISSET, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_EMPTY", T_EMPTY, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_HALT_COMPILER", T_HALT_COMPILER, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_CLASS", T_CLASS, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_TRAIT", T_TRAIT, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_INTERFACE", T_INTERFACE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_EXTENDS", T_EXTENDS, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_IMPLEMENTS", T_IMPLEMENTS, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_OBJECT_OPERATOR", T_OBJECT_OPERATOR, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_DOUBLE_ARROW", T_DOUBLE_ARROW, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_LIST", T_LIST, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ARRAY", T_ARRAY, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_CALLABLE", T_CALLABLE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_CLASS_C", T_CLASS_C, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_TRAIT_C", T_TRAIT_C, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_METHOD_C", T_METHOD_C, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_FUNC_C", T_FUNC_C, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_LINE", T_LINE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_FILE", T_FILE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_COMMENT", T_COMMENT, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_DOC_COMMENT", T_DOC_COMMENT, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_OPEN_TAG", T_OPEN_TAG, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_OPEN_TAG_WITH_ECHO", T_OPEN_TAG_WITH_ECHO, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_CLOSE_TAG", T_CLOSE_TAG, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_WHITESPACE", T_WHITESPACE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_START_HEREDOC", T_START_HEREDOC, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_END_HEREDOC", T_END_HEREDOC, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_DOLLAR_OPEN_CURLY_BRACES", T_DOLLAR_OPEN_CURLY_BRACES, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_CURLY_OPEN", T_CURLY_OPEN, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_PAAMAYIM_NEKUDOTAYIM", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_NAMESPACE", T_NAMESPACE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_NS_C", T_NS_C, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_DIR", T_DIR, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_NS_SEPARATOR", T_NS_SEPARATOR, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_DOUBLE_COLON", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT); +} + +char *get_token_type_name(int token_type) +{ + switch (token_type) { + + case T_REQUIRE_ONCE: return "T_REQUIRE_ONCE"; + case T_REQUIRE: return "T_REQUIRE"; + case T_EVAL: return "T_EVAL"; + case T_INCLUDE_ONCE: return "T_INCLUDE_ONCE"; + case T_INCLUDE: return "T_INCLUDE"; + case T_LOGICAL_OR: return "T_LOGICAL_OR"; + case T_LOGICAL_XOR: return "T_LOGICAL_XOR"; + case T_LOGICAL_AND: return "T_LOGICAL_AND"; + case T_PRINT: return "T_PRINT"; + case T_SR_EQUAL: return "T_SR_EQUAL"; + case T_SL_EQUAL: return "T_SL_EQUAL"; + case T_XOR_EQUAL: return "T_XOR_EQUAL"; + case T_OR_EQUAL: return "T_OR_EQUAL"; + case T_AND_EQUAL: return "T_AND_EQUAL"; + case T_MOD_EQUAL: return "T_MOD_EQUAL"; + case T_CONCAT_EQUAL: return "T_CONCAT_EQUAL"; + case T_DIV_EQUAL: return "T_DIV_EQUAL"; + case T_MUL_EQUAL: return "T_MUL_EQUAL"; + case T_MINUS_EQUAL: return "T_MINUS_EQUAL"; + case T_PLUS_EQUAL: return "T_PLUS_EQUAL"; + case T_BOOLEAN_OR: return "T_BOOLEAN_OR"; + case T_BOOLEAN_AND: return "T_BOOLEAN_AND"; + case T_IS_NOT_IDENTICAL: return "T_IS_NOT_IDENTICAL"; + case T_IS_IDENTICAL: return "T_IS_IDENTICAL"; + case T_IS_NOT_EQUAL: return "T_IS_NOT_EQUAL"; + case T_IS_EQUAL: return "T_IS_EQUAL"; + case T_IS_GREATER_OR_EQUAL: return "T_IS_GREATER_OR_EQUAL"; + case T_IS_SMALLER_OR_EQUAL: return "T_IS_SMALLER_OR_EQUAL"; + case T_SR: return "T_SR"; + case T_SL: return "T_SL"; + case T_INSTANCEOF: return "T_INSTANCEOF"; + case T_UNSET_CAST: return "T_UNSET_CAST"; + case T_BOOL_CAST: return "T_BOOL_CAST"; + case T_OBJECT_CAST: return "T_OBJECT_CAST"; + case T_ARRAY_CAST: return "T_ARRAY_CAST"; + case T_STRING_CAST: return "T_STRING_CAST"; + case T_DOUBLE_CAST: return "T_DOUBLE_CAST"; + case T_INT_CAST: return "T_INT_CAST"; + case T_DEC: return "T_DEC"; + case T_INC: return "T_INC"; + case T_CLONE: return "T_CLONE"; + case T_NEW: return "T_NEW"; + case T_EXIT: return "T_EXIT"; + case T_IF: return "T_IF"; + case T_ELSEIF: return "T_ELSEIF"; + case T_ELSE: return "T_ELSE"; + case T_ENDIF: return "T_ENDIF"; + case T_LNUMBER: return "T_LNUMBER"; + case T_DNUMBER: return "T_DNUMBER"; + case T_STRING: return "T_STRING"; + case T_STRING_VARNAME: return "T_STRING_VARNAME"; + case T_VARIABLE: return "T_VARIABLE"; + case T_NUM_STRING: return "T_NUM_STRING"; + case T_INLINE_HTML: return "T_INLINE_HTML"; + case T_CHARACTER: return "T_CHARACTER"; + case T_BAD_CHARACTER: return "T_BAD_CHARACTER"; + case T_ENCAPSED_AND_WHITESPACE: return "T_ENCAPSED_AND_WHITESPACE"; + case T_CONSTANT_ENCAPSED_STRING: return "T_CONSTANT_ENCAPSED_STRING"; + case T_ECHO: return "T_ECHO"; + case T_DO: return "T_DO"; + case T_WHILE: return "T_WHILE"; + case T_ENDWHILE: return "T_ENDWHILE"; + case T_FOR: return "T_FOR"; + case T_ENDFOR: return "T_ENDFOR"; + case T_FOREACH: return "T_FOREACH"; + case T_ENDFOREACH: return "T_ENDFOREACH"; + case T_DECLARE: return "T_DECLARE"; + case T_ENDDECLARE: return "T_ENDDECLARE"; + case T_AS: return "T_AS"; + case T_SWITCH: return "T_SWITCH"; + case T_ENDSWITCH: return "T_ENDSWITCH"; + case T_CASE: return "T_CASE"; + case T_DEFAULT: return "T_DEFAULT"; + case T_BREAK: return "T_BREAK"; + case T_CONTINUE: return "T_CONTINUE"; + case T_GOTO: return "T_GOTO"; + case T_FUNCTION: return "T_FUNCTION"; + case T_CONST: return "T_CONST"; + case T_RETURN: return "T_RETURN"; + case T_TRY: return "T_TRY"; + case T_CATCH: return "T_CATCH"; + case T_THROW: return "T_THROW"; + case T_USE: return "T_USE"; + case T_INSTEADOF: return "T_INSTEADOF"; + case T_GLOBAL: return "T_GLOBAL"; + case T_PUBLIC: return "T_PUBLIC"; + case T_PROTECTED: return "T_PROTECTED"; + case T_PRIVATE: return "T_PRIVATE"; + case T_FINAL: return "T_FINAL"; + case T_ABSTRACT: return "T_ABSTRACT"; + case T_STATIC: return "T_STATIC"; + case T_VAR: return "T_VAR"; + case T_UNSET: return "T_UNSET"; + case T_ISSET: return "T_ISSET"; + case T_EMPTY: return "T_EMPTY"; + case T_HALT_COMPILER: return "T_HALT_COMPILER"; + case T_CLASS: return "T_CLASS"; + case T_TRAIT: return "T_TRAIT"; + case T_INTERFACE: return "T_INTERFACE"; + case T_EXTENDS: return "T_EXTENDS"; + case T_IMPLEMENTS: return "T_IMPLEMENTS"; + case T_OBJECT_OPERATOR: return "T_OBJECT_OPERATOR"; + case T_DOUBLE_ARROW: return "T_DOUBLE_ARROW"; + case T_LIST: return "T_LIST"; + case T_ARRAY: return "T_ARRAY"; + case T_CALLABLE: return "T_CALLABLE"; + case T_CLASS_C: return "T_CLASS_C"; + case T_TRAIT_C: return "T_TRAIT_C"; + case T_METHOD_C: return "T_METHOD_C"; + case T_FUNC_C: return "T_FUNC_C"; + case T_LINE: return "T_LINE"; + case T_FILE: return "T_FILE"; + case T_COMMENT: return "T_COMMENT"; + case T_DOC_COMMENT: return "T_DOC_COMMENT"; + case T_OPEN_TAG: return "T_OPEN_TAG"; + case T_OPEN_TAG_WITH_ECHO: return "T_OPEN_TAG_WITH_ECHO"; + case T_CLOSE_TAG: return "T_CLOSE_TAG"; + case T_WHITESPACE: return "T_WHITESPACE"; + case T_START_HEREDOC: return "T_START_HEREDOC"; + case T_END_HEREDOC: return "T_END_HEREDOC"; + case T_DOLLAR_OPEN_CURLY_BRACES: return "T_DOLLAR_OPEN_CURLY_BRACES"; + case T_CURLY_OPEN: return "T_CURLY_OPEN"; + case T_PAAMAYIM_NEKUDOTAYIM: return "T_DOUBLE_COLON"; + case T_NAMESPACE: return "T_NAMESPACE"; + case T_NS_C: return "T_NS_C"; + case T_DIR: return "T_DIR"; + case T_NS_SEPARATOR: return "T_NS_SEPARATOR"; + + } + return "UNKNOWN"; +} + diff --git a/ext/tokenizer/tokenizer_data_gen.sh b/ext/tokenizer/tokenizer_data_gen.sh new file mode 100755 index 0000000..4ccbd2a --- /dev/null +++ b/ext/tokenizer/tokenizer_data_gen.sh @@ -0,0 +1,75 @@ +#!/bin/sh + +INFILE="../../Zend/zend_language_parser.h" +OUTFILE="tokenizer_data.c" +AWK=awk + +#################################################################### + +if test ! -f "./tokenizer.c"; then + echo "Please run this script from within php-src/ext/tokenizer" + exit 0 +fi + + +echo '/* + +----------------------------------------------------------------------+ + | PHP Version 5 | + +----------------------------------------------------------------------+ + | Copyright (c) 1997-2013 The PHP Group | + +----------------------------------------------------------------------+ + | This source file is subject to version 3.01 of the PHP license, | + | that is bundled with this package in the file LICENSE, and is | + | available through the world-wide-web at the following url: | + | http://www.php.net/license/3_01.txt | + | If you did not receive a copy of the PHP license and are unable to | + | obtain it through the world-wide-web, please send a note to | + | license@php.net so we can mail you a copy immediately. | + +----------------------------------------------------------------------+ + | Author: Johannes Schlueter <johannes@php.net> | + +----------------------------------------------------------------------+ +*/ + +/* $Id$ */ + +/* + DO NOT EDIT THIS FILE! + This file is generated using tokenizer_data_gen.sh +*/ + +#include "php.h" +#include "zend.h" +#include <zend_language_parser.h> + +' > $OUTFILE + + +echo 'void tokenizer_register_constants(INIT_FUNC_ARGS) {' >> $OUTFILE +$AWK '/^#define T_/ { print " REGISTER_LONG_CONSTANT(\"" $2 "\", " $2 ", CONST_CS | CONST_PERSISTENT);" }' < $INFILE >> $OUTFILE +echo ' REGISTER_LONG_CONSTANT("T_DOUBLE_COLON", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT);' >> $OUTFILE +echo '}' >> $OUTFILE + + +echo ' +char *get_token_type_name(int token_type) +{ + switch (token_type) { +' >> $OUTFILE + +$AWK ' + /^#define T_PAAMAYIM_NEKUDOTAYIM/ { + print " case T_PAAMAYIM_NEKUDOTAYIM: return \"T_DOUBLE_COLON\";" + next + } + /^#define T_/ { + print " case " $2 ": return \"" $2 "\";" + } +' < $INFILE >> $OUTFILE + +echo ' + } + return "UNKNOWN"; +} +' >> $OUTFILE + +echo "Wrote $OUTFILE" |