summaryrefslogtreecommitdiff
path: root/ext/tokenizer
diff options
context:
space:
mode:
Diffstat (limited to 'ext/tokenizer')
-rw-r--r--ext/tokenizer/php_tokenizer.h2
-rw-r--r--ext/tokenizer/tests/bug60097.phpt121
-rw-r--r--ext/tokenizer/tests/token_get_all_variation11.phpt4
-rw-r--r--ext/tokenizer/tests/token_get_all_variation13.phpt2
-rw-r--r--ext/tokenizer/tests/token_get_all_variation17.phpt2
-rw-r--r--ext/tokenizer/tests/token_get_all_variation4.phpt4
-rw-r--r--ext/tokenizer/tests/token_get_all_variation5.phpt22
-rw-r--r--ext/tokenizer/tests/token_get_all_variation6.phpt4
-rw-r--r--ext/tokenizer/tests/token_get_all_variation8.phpt2
-rw-r--r--ext/tokenizer/tokenizer.c7
-rw-r--r--ext/tokenizer/tokenizer_data.c8
-rwxr-xr-xext/tokenizer/tokenizer_data_gen.sh2
12 files changed, 151 insertions, 29 deletions
diff --git a/ext/tokenizer/php_tokenizer.h b/ext/tokenizer/php_tokenizer.h
index 1b5490d257..761556a14b 100644
--- a/ext/tokenizer/php_tokenizer.h
+++ b/ext/tokenizer/php_tokenizer.h
@@ -2,7 +2,7 @@
+----------------------------------------------------------------------+
| PHP Version 5 |
+----------------------------------------------------------------------+
- | Copyright (c) 1997-2014 The PHP Group |
+ | Copyright (c) 1997-2013 The PHP Group |
+----------------------------------------------------------------------+
| This source file is subject to version 3.01 of the PHP license, |
| that is bundled with this package in the file LICENSE, and is |
diff --git a/ext/tokenizer/tests/bug60097.phpt b/ext/tokenizer/tests/bug60097.phpt
new file mode 100644
index 0000000000..2116866c93
--- /dev/null
+++ b/ext/tokenizer/tests/bug60097.phpt
@@ -0,0 +1,121 @@
+--TEST--
+Bug 60097: token_get_all fails to lex nested heredoc
+--FILE--
+<?php
+
+var_dump(token_get_all('<?php
+<<<DOC1
+{$s(<<<DOC2
+DOC2
+)}
+DOC1;
+'));
+
+?>
+--EXPECT--
+array(14) {
+ [0]=>
+ array(3) {
+ [0]=>
+ int(374)
+ [1]=>
+ string(6) "<?php
+"
+ [2]=>
+ int(1)
+ }
+ [1]=>
+ array(3) {
+ [0]=>
+ int(378)
+ [1]=>
+ string(8) "<<<DOC1
+"
+ [2]=>
+ int(2)
+ }
+ [2]=>
+ array(3) {
+ [0]=>
+ int(381)
+ [1]=>
+ string(1) "{"
+ [2]=>
+ int(3)
+ }
+ [3]=>
+ array(3) {
+ [0]=>
+ int(310)
+ [1]=>
+ string(2) "$s"
+ [2]=>
+ int(3)
+ }
+ [4]=>
+ string(1) "("
+ [5]=>
+ array(3) {
+ [0]=>
+ int(378)
+ [1]=>
+ string(8) "<<<DOC2
+"
+ [2]=>
+ int(3)
+ }
+ [6]=>
+ array(3) {
+ [0]=>
+ int(379)
+ [1]=>
+ string(4) "DOC2"
+ [2]=>
+ int(4)
+ }
+ [7]=>
+ array(3) {
+ [0]=>
+ int(377)
+ [1]=>
+ string(1) "
+"
+ [2]=>
+ int(4)
+ }
+ [8]=>
+ string(1) ")"
+ [9]=>
+ string(1) "}"
+ [10]=>
+ array(3) {
+ [0]=>
+ int(315)
+ [1]=>
+ string(1) "
+"
+ [2]=>
+ int(5)
+ }
+ [11]=>
+ array(3) {
+ [0]=>
+ int(379)
+ [1]=>
+ string(4) "DOC1"
+ [2]=>
+ int(6)
+ }
+ [12]=>
+ string(1) ";"
+ [13]=>
+ array(3) {
+ [0]=>
+ int(377)
+ [1]=>
+ string(1) "
+"
+ [2]=>
+ int(6)
+ }
+}
diff --git a/ext/tokenizer/tests/token_get_all_variation11.phpt b/ext/tokenizer/tests/token_get_all_variation11.phpt
index ecc86177a4..98d89961b7 100644
--- a/ext/tokenizer/tests/token_get_all_variation11.phpt
+++ b/ext/tokenizer/tests/token_get_all_variation11.phpt
@@ -130,7 +130,7 @@ array(49) {
[6]=>
array(3) {
[0]=>
- int(283)
+ int(%d)
[1]=>
string(2) "=="
[2]=>
@@ -273,7 +273,7 @@ array(49) {
[27]=>
array(3) {
[0]=>
- int(283)
+ int(%d)
[1]=>
string(2) "=="
[2]=>
diff --git a/ext/tokenizer/tests/token_get_all_variation13.phpt b/ext/tokenizer/tests/token_get_all_variation13.phpt
index 9b2f3bc94f..6f85492a99 100644
--- a/ext/tokenizer/tests/token_get_all_variation13.phpt
+++ b/ext/tokenizer/tests/token_get_all_variation13.phpt
@@ -1005,7 +1005,7 @@ array(145) {
[122]=>
array(3) {
[0]=>
- int(288)
+ int(%d)
[1]=>
string(10) "instanceof"
[2]=>
diff --git a/ext/tokenizer/tests/token_get_all_variation17.phpt b/ext/tokenizer/tests/token_get_all_variation17.phpt
index dccc4c9c23..f71444bc1e 100644
--- a/ext/tokenizer/tests/token_get_all_variation17.phpt
+++ b/ext/tokenizer/tests/token_get_all_variation17.phpt
@@ -145,7 +145,7 @@ array(81) {
[14]=>
array(3) {
[0]=>
- int(283)
+ int(%d)
[1]=>
string(2) "=="
[2]=>
diff --git a/ext/tokenizer/tests/token_get_all_variation4.phpt b/ext/tokenizer/tests/token_get_all_variation4.phpt
index 45e6f8afbd..6bc111efba 100644
--- a/ext/tokenizer/tests/token_get_all_variation4.phpt
+++ b/ext/tokenizer/tests/token_get_all_variation4.phpt
@@ -339,7 +339,7 @@ array(89) {
[38]=>
array(3) {
[0]=>
- int(279)
+ int(%d)
[1]=>
string(2) "&&"
[2]=>
@@ -518,7 +518,7 @@ array(89) {
[60]=>
array(3) {
[0]=>
- int(278)
+ int(%d)
[1]=>
string(2) "||"
[2]=>
diff --git a/ext/tokenizer/tests/token_get_all_variation5.phpt b/ext/tokenizer/tests/token_get_all_variation5.phpt
index 0068f2866f..681fb48e57 100644
--- a/ext/tokenizer/tests/token_get_all_variation5.phpt
+++ b/ext/tokenizer/tests/token_get_all_variation5.phpt
@@ -181,7 +181,7 @@ array(94) {
[18]=>
array(3) {
[0]=>
- int(277)
+ int(%d)
[1]=>
string(2) "+="
[2]=>
@@ -238,7 +238,7 @@ array(94) {
[25]=>
array(3) {
[0]=>
- int(276)
+ int(%d)
[1]=>
string(2) "-="
[2]=>
@@ -295,7 +295,7 @@ array(94) {
[32]=>
array(3) {
[0]=>
- int(275)
+ int(%d)
[1]=>
string(2) "*="
[2]=>
@@ -352,7 +352,7 @@ array(94) {
[39]=>
array(3) {
[0]=>
- int(274)
+ int(%d)
[1]=>
string(2) "/="
[2]=>
@@ -409,7 +409,7 @@ array(94) {
[46]=>
array(3) {
[0]=>
- int(272)
+ int(%d)
[1]=>
string(2) "%="
[2]=>
@@ -466,7 +466,7 @@ array(94) {
[53]=>
array(3) {
[0]=>
- int(271)
+ int(%d)
[1]=>
string(2) "&="
[2]=>
@@ -523,7 +523,7 @@ array(94) {
[60]=>
array(3) {
[0]=>
- int(270)
+ int(%d)
[1]=>
string(2) "|="
[2]=>
@@ -580,7 +580,7 @@ array(94) {
[67]=>
array(3) {
[0]=>
- int(269)
+ int(%d)
[1]=>
string(2) "^="
[2]=>
@@ -637,7 +637,7 @@ array(94) {
[74]=>
array(3) {
[0]=>
- int(267)
+ int(%d)
[1]=>
string(3) ">>="
[2]=>
@@ -694,7 +694,7 @@ array(94) {
[81]=>
array(3) {
[0]=>
- int(268)
+ int(%d)
[1]=>
string(3) "<<="
[2]=>
@@ -751,7 +751,7 @@ array(94) {
[88]=>
array(3) {
[0]=>
- int(273)
+ int(%d)
[1]=>
string(2) ".="
[2]=>
diff --git a/ext/tokenizer/tests/token_get_all_variation6.phpt b/ext/tokenizer/tests/token_get_all_variation6.phpt
index 54936d0c89..6213dab9d0 100644
--- a/ext/tokenizer/tests/token_get_all_variation6.phpt
+++ b/ext/tokenizer/tests/token_get_all_variation6.phpt
@@ -191,7 +191,7 @@ array(50) {
[21]=>
array(3) {
[0]=>
- int(287)
+ int(%d)
[1]=>
string(2) "<<"
[2]=>
@@ -277,7 +277,7 @@ array(50) {
[32]=>
array(3) {
[0]=>
- int(286)
+ int(%d)
[1]=>
string(2) ">>"
[2]=>
diff --git a/ext/tokenizer/tests/token_get_all_variation8.phpt b/ext/tokenizer/tests/token_get_all_variation8.phpt
index 0cf1d63471..c80a5d0f04 100644
--- a/ext/tokenizer/tests/token_get_all_variation8.phpt
+++ b/ext/tokenizer/tests/token_get_all_variation8.phpt
@@ -794,7 +794,7 @@ array(108) {
[103]=>
array(3) {
[0]=>
- int(289)
+ int(%d)
[1]=>
string(7) "(unset)"
[2]=>
diff --git a/ext/tokenizer/tokenizer.c b/ext/tokenizer/tokenizer.c
index 2a30a9819c..ee96e2eaa1 100644
--- a/ext/tokenizer/tokenizer.c
+++ b/ext/tokenizer/tokenizer.c
@@ -2,7 +2,7 @@
+----------------------------------------------------------------------+
| PHP Version 5 |
+----------------------------------------------------------------------+
- | Copyright (c) 1997-2014 The PHP Group |
+ | Copyright (c) 1997-2013 The PHP Group |
+----------------------------------------------------------------------+
| This source file is subject to version 3.01 of the PHP license, |
| that is bundled with this package in the file LICENSE, and is |
@@ -138,11 +138,8 @@ static void tokenize(zval *return_value TSRMLS_DC)
token_line = ++CG(zend_lineno);
CG(increment_lineno) = 0;
}
- add_next_index_stringl(keyword, Z_STRVAL(token), Z_STRLEN(token), 1);
- efree(Z_STRVAL(token));
- } else {
- add_next_index_stringl(keyword, (char *)zendtext, zendleng, 1);
}
+ add_next_index_stringl(keyword, (char *)zendtext, zendleng, 1);
add_next_index_long(keyword, token_line);
add_next_index_zval(return_value, keyword);
} else {
diff --git a/ext/tokenizer/tokenizer_data.c b/ext/tokenizer/tokenizer_data.c
index d4ef1fdf54..57b29e1dd7 100644
--- a/ext/tokenizer/tokenizer_data.c
+++ b/ext/tokenizer/tokenizer_data.c
@@ -2,7 +2,7 @@
+----------------------------------------------------------------------+
| PHP Version 5 |
+----------------------------------------------------------------------+
- | Copyright (c) 1997-2014 The PHP Group |
+ | Copyright (c) 1997-2013 The PHP Group |
+----------------------------------------------------------------------+
| This source file is subject to version 3.01 of the PHP license, |
| that is bundled with this package in the file LICENSE, and is |
@@ -21,7 +21,7 @@
/*
DO NOT EDIT THIS FILE!
This file is generated using tokenizer_data_gen.sh
-*/
+*/
#include "php.h"
#include "zend.h"
@@ -108,8 +108,10 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
REGISTER_LONG_CONSTANT("T_FUNCTION", T_FUNCTION, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_CONST", T_CONST, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_RETURN", T_RETURN, CONST_CS | CONST_PERSISTENT);
+ REGISTER_LONG_CONSTANT("T_YIELD", T_YIELD, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_TRY", T_TRY, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_CATCH", T_CATCH, CONST_CS | CONST_PERSISTENT);
+ REGISTER_LONG_CONSTANT("T_FINALLY", T_FINALLY, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_THROW", T_THROW, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_USE", T_USE, CONST_CS | CONST_PERSISTENT);
REGISTER_LONG_CONSTANT("T_INSTEADOF", T_INSTEADOF, CONST_CS | CONST_PERSISTENT);
@@ -242,8 +244,10 @@ char *get_token_type_name(int token_type)
case T_FUNCTION: return "T_FUNCTION";
case T_CONST: return "T_CONST";
case T_RETURN: return "T_RETURN";
+ case T_YIELD: return "T_YIELD";
case T_TRY: return "T_TRY";
case T_CATCH: return "T_CATCH";
+ case T_FINALLY: return "T_FINALLY";
case T_THROW: return "T_THROW";
case T_USE: return "T_USE";
case T_INSTEADOF: return "T_INSTEADOF";
diff --git a/ext/tokenizer/tokenizer_data_gen.sh b/ext/tokenizer/tokenizer_data_gen.sh
index 9a675c4f97..4ccbd2ac67 100755
--- a/ext/tokenizer/tokenizer_data_gen.sh
+++ b/ext/tokenizer/tokenizer_data_gen.sh
@@ -16,7 +16,7 @@ echo '/*
+----------------------------------------------------------------------+
| PHP Version 5 |
+----------------------------------------------------------------------+
- | Copyright (c) 1997-2014 The PHP Group |
+ | Copyright (c) 1997-2013 The PHP Group |
+----------------------------------------------------------------------+
| This source file is subject to version 3.01 of the PHP license, |
| that is bundled with this package in the file LICENSE, and is |
View Lorry Controller Status