Don't return T_ERROR from token_get_all()

This turned out to be rather inconvenient after all. Instead just
return the same output we did on PHP 5. If people want to have an
error, use TOKEN_PARSE.

4 files changed, 10 insertions, 12 deletions

diff --git a/ext/tokenizer/tests/parse_errors.phpt b/ext/tokenizer/tests/parse_errors.phpt
index 3f376e0f9a..700326ff41 100644
--- a/ext/tokenizer/tests/parse_errors.phpt
+++ b/ext/tokenizer/tests/parse_errors.phpt
@@ -33,7 +33,7 @@ Invalid numeric literal
 T_OPEN_TAG (<?php )
 T_STRING (var_dump)
 (
-T_ERROR (078)
+T_LNUMBER (078)
 )
 ;
 
@@ -41,7 +41,7 @@ Invalid UTF-8 codepoint escape sequence
 T_OPEN_TAG (<?php )
 T_STRING (var_dump)
 (
-T_ERROR ("\u{xyz}")
+T_CONSTANT_ENCAPSED_STRING ("\u{xyz}")
 )
 ;
 
@@ -49,7 +49,7 @@ Invalid UTF-8 codepoint escape sequence: Codepoint too large
 T_OPEN_TAG (<?php )
 T_STRING (var_dump)
 (
-T_ERROR ("\u{ffffff}")
+T_CONSTANT_ENCAPSED_STRING ("\u{ffffff}")
 )
 ;
 
@@ -57,10 +57,10 @@ Invalid numeric literal
 T_OPEN_TAG (<?php )
 T_STRING (var_dump)
 (
-T_ERROR (078)
+T_LNUMBER (078)
 T_WHITESPACE ( )
 +
 T_WHITESPACE ( )
-T_ERROR (078)
+T_LNUMBER (078)
 )
 ;
diff --git a/ext/tokenizer/tokenizer.c b/ext/tokenizer/tokenizer.c
index a8d1bff134..50b47835dd 100644
--- a/ext/tokenizer/tokenizer.c
+++ b/ext/tokenizer/tokenizer.c
@@ -191,9 +191,9 @@ void on_event(zend_php_scanner_event event, int token, int line)
 	HashTable *tokens_ht;
 	zval *token_zv;
 
-	switch(event) {
+	switch (event) {
 		case ON_TOKEN:
-			if (token == T_ERROR || token == END) break;
+			if (token == END) break;
 			if (token >= 256) {
 				array_init(&keyword);
 				add_next_index_long(&keyword, token);
@@ -281,7 +281,7 @@ PHP_FUNCTION(token_get_all)
 		success = tokenize_parse(return_value, source);
 	} else {
 		success = tokenize(return_value, source);
-		/* Normal token_get_all() should never throw. Errors are indicated by T_ERROR tokens. */
+		/* Normal token_get_all() should not throw. */
 		zend_clear_exception();
 	}
 
diff --git a/ext/tokenizer/tokenizer_data.c b/ext/tokenizer/tokenizer_data.c
index 5aeff66f3f..6769202a3b 100644
--- a/ext/tokenizer/tokenizer_data.c
+++ b/ext/tokenizer/tokenizer_data.c
@@ -162,7 +162,6 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_NS_C", T_NS_C, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_NS_SEPARATOR", T_NS_SEPARATOR, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ELLIPSIS", T_ELLIPSIS, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_ERROR", T_ERROR, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_DOUBLE_COLON", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT);
 }
 
@@ -303,7 +302,6 @@ char *get_token_type_name(int token_type)
 		case T_NS_C: return "T_NS_C";
 		case T_NS_SEPARATOR: return "T_NS_SEPARATOR";
 		case T_ELLIPSIS: return "T_ELLIPSIS";
-		case T_ERROR: return "T_ERROR";
 
 	}
 	return "UNKNOWN";
diff --git a/ext/tokenizer/tokenizer_data_gen.sh b/ext/tokenizer/tokenizer_data_gen.sh
index 2c3c85793f..65bf0de83b 100755
--- a/ext/tokenizer/tokenizer_data_gen.sh
+++ b/ext/tokenizer/tokenizer_data_gen.sh
@@ -46,7 +46,7 @@ echo '/*
 
 echo 'void tokenizer_register_constants(INIT_FUNC_ARGS) {' >> $OUTFILE
 $AWK '
-	/^#define T_NOELSE/ { next }
+	/^#define T_(NOELSE|ERROR)/ { next }
 	/^#define T_/  { print "	REGISTER_LONG_CONSTANT(\"" $2 "\", " $2 ", CONST_CS | CONST_PERSISTENT);" }
 ' < $INFILE >> $OUTFILE
 echo '	REGISTER_LONG_CONSTANT("T_DOUBLE_COLON", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT);' >> $OUTFILE
@@ -64,7 +64,7 @@ $AWK '
 		print "		case T_PAAMAYIM_NEKUDOTAYIM: return \"T_DOUBLE_COLON\";"
 		next
 	}
-	/^#define T_NOELSE/ { next }
+	/^#define T_(NOELSE|ERROR)/ { next }
 	/^#define T_/ {
 		print "		case " $2 ": return \"" $2 "\";"
 	}