Fixed bug #26463 (Incorrect handling of semicolons after heredoc)

2 files changed, 124 insertions, 4 deletions

diff --git a/ext/tokenizer/tests/bug26463.phpt b/ext/tokenizer/tests/bug26463.phpt
new file mode 100644
index 0000000000..d1e75b4e03
--- /dev/null
+++ b/ext/tokenizer/tests/bug26463.phpt
@@ -0,0 +1,118 @@
+--TEST--
+Bug #26463 (token_get_all() does not correctly handle semicolons after T_END_HEREDOC)
+--FILE--
+<?php
+$str = '<?php
+$x=<<<DD
+jhdsjkfhjdsh
+DD
+."";
+$a=<<<DDDD
+jhdsjkfhjdsh
+DDDD;
+?>';
+var_dump(token_get_all($str));
+?>
+--EXPECT--
+array(17) {
+  [0]=>
+  array(2) {
+    [0]=>
+    int(363)
+    [1]=>
+    string(6) "<?php
+"
+  }
+  [1]=>
+  array(2) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(2) "$x"
+  }
+  [2]=>
+  string(1) "="
+  [3]=>
+  array(2) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<<<DD
+"
+  }
+  [4]=>
+  array(2) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(13) "jhdsjkfhjdsh
+"
+  }
+  [5]=>
+  array(2) {
+    [0]=>
+    int(368)
+    [1]=>
+    string(2) "DD"
+  }
+  [6]=>
+  string(1) "."
+  [7]=>
+  array(2) {
+    [0]=>
+    int(313)
+    [1]=>
+    string(2) """"
+  }
+  [8]=>
+  string(1) ";"
+  [9]=>
+  array(2) {
+    [0]=>
+    int(366)
+    [1]=>
+    string(1) "
+"
+  }
+  [10]=>
+  array(2) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(2) "$a"
+  }
+  [11]=>
+  string(1) "="
+  [12]=>
+  array(2) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(8) "<<<DDDD
+"
+  }
+  [13]=>
+  array(2) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(13) "jhdsjkfhjdsh
+"
+  }
+  [14]=>
+  array(2) {
+    [0]=>
+    int(368)
+    [1]=>
+    string(4) "DDDD"
+  }
+  [15]=>
+  string(1) ";"
+  [16]=>
+  array(2) {
+    [0]=>
+    int(365)
+    [1]=>
+    string(2) "?>"
+  }
+}
diff --git a/ext/tokenizer/tokenizer.c b/ext/tokenizer/tokenizer.c
index 5c7346c4ba..e4c0678ec2 100644
--- a/ext/tokenizer/tokenizer.c
+++ b/ext/tokenizer/tokenizer.c
@@ -361,7 +361,12 @@ static void tokenize(zval *return_value TSRMLS_DC)
 			MAKE_STD_ZVAL(keyword);
 			array_init(keyword);
 			add_next_index_long(keyword, token_type);
-			add_next_index_stringl(keyword, zendtext, zendleng, 1);
+			if (token_type == T_END_HEREDOC) {
+				add_next_index_stringl(keyword, Z_STRVAL(token), Z_STRLEN(token), 1);
+				efree(Z_STRVAL(token));
+			} else {
+				add_next_index_stringl(keyword, zendtext, zendleng, 1);
+			}
 			add_next_index_zval(return_value, keyword);
 		} else {
 			add_next_index_stringl(return_value, zendtext, zendleng, 1);
@@ -369,9 +374,6 @@ static void tokenize(zval *return_value TSRMLS_DC)
 		if (destroy && Z_TYPE(token) != IS_NULL) {
 			zval_dtor(&token);
 		}
-		if (token_type == T_END_HEREDOC) {
-			efree(Z_STRVAL(token));
-		}
 		ZVAL_NULL(&token);
 	}
 }