Fix bug #69430

Don't throw from token_get_all() unless TOKEN_PARSE is used. Errors
are reported as T_ERROR tokens.

1 files changed, 9 insertions, 20 deletions

diff --git a/ext/tokenizer/tokenizer.c b/ext/tokenizer/tokenizer.c
index 6961eb8432..a8d1bff134 100644
--- a/ext/tokenizer/tokenizer.c
+++ b/ext/tokenizer/tokenizer.c
@@ -28,6 +28,7 @@
 #include "php_tokenizer.h"
 
 #include "zend.h"
+#include "zend_exceptions.h"
 #include "zend_language_scanner.h"
 #include "zend_language_scanner_defs.h"
 #include <zend_language_parser.h>
@@ -112,7 +113,6 @@ static zend_bool tokenize(zval *return_value, zend_string *source)
 	zval token;
 	zval keyword;
 	int token_type;
-	zend_bool destroy;
 	int token_line = 1;
 	int need_tokens = -1; /* for __halt_compiler lexing. -1 = disabled */
 
@@ -127,24 +127,10 @@ static zend_bool tokenize(zval *return_value, zend_string *source)
 	LANG_SCNG(yy_state) = yycINITIAL;
 	array_init(return_value);
 
-	ZVAL_NULL(&token);
+	ZVAL_UNDEF(&token);
 	while ((token_type = lex_scan(&token))) {
-
-		if(token_type == T_ERROR) break;
-
-		destroy = 1;
-		switch (token_type) {
-			case T_CLOSE_TAG:
-				if (zendtext[zendleng - 1] != '>') {
-					CG(zend_lineno)++;
-				}
-			case T_OPEN_TAG:
-			case T_OPEN_TAG_WITH_ECHO:
-			case T_WHITESPACE:
-			case T_COMMENT:
-			case T_DOC_COMMENT:
-				destroy = 0;
-				break;
+		if (token_type == T_CLOSE_TAG && zendtext[zendleng - 1] != '>') {
+			CG(zend_lineno)++;
 		}
 
 		if (token_type >= 256) {
@@ -162,10 +148,11 @@ static zend_bool tokenize(zval *return_value, zend_string *source)
 		} else {
 			add_next_index_stringl(return_value, (char *)zendtext, zendleng);
 		}
-		if (destroy && Z_TYPE(token) != IS_NULL) {
+
+		if (Z_TYPE(token) != IS_UNDEF) {
 			zval_dtor(&token);
+			ZVAL_UNDEF(&token);
 		}
-		ZVAL_NULL(&token);
 
 		/* after T_HALT_COMPILER collect the next three non-dropped tokens */
 		if (need_tokens != -1) {
@@ -294,6 +281,8 @@ PHP_FUNCTION(token_get_all)
 		success = tokenize_parse(return_value, source);
 	} else {
 		success = tokenize(return_value, source);
+		/* Normal token_get_all() should never throw. Errors are indicated by T_ERROR tokens. */
+		zend_clear_exception();
 	}
 
 	if (!success) RETURN_FALSE;