Regenerate tokenizer data

Also fix the tokenizer_data_gen.sh script after 64bit changes.

1 files changed, 34 insertions, 32 deletions

diff --git a/ext/tokenizer/tokenizer_data.c b/ext/tokenizer/tokenizer_data.c
index 49dec7c150..209e3ba495 100644
--- a/ext/tokenizer/tokenizer_data.c
+++ b/ext/tokenizer/tokenizer_data.c
@@ -51,6 +51,7 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_MUL_EQUAL", T_MUL_EQUAL, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_MINUS_EQUAL", T_MINUS_EQUAL, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_PLUS_EQUAL", T_PLUS_EQUAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_COALESCE", T_COALESCE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_BOOLEAN_OR", T_BOOLEAN_OR, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_BOOLEAN_AND", T_BOOLEAN_AND, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_IS_NOT_IDENTICAL", T_IS_NOT_IDENTICAL, CONST_CS | CONST_PERSISTENT);
@@ -74,22 +75,29 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_POW", T_POW, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_CLONE", T_CLONE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_NEW", T_NEW, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_EXIT", T_EXIT, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_IF", T_IF, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ELSEIF", T_ELSEIF, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ELSE", T_ELSE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ENDIF", T_ENDIF, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_PUBLIC", T_PUBLIC, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_PROTECTED", T_PROTECTED, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_PRIVATE", T_PRIVATE, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_FINAL", T_FINAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_ABSTRACT", T_ABSTRACT, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_STATIC", T_STATIC, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_DOUBLE_ARROW", T_DOUBLE_ARROW, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_LNUMBER", T_LNUMBER, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_DNUMBER", T_DNUMBER, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_STRING", T_STRING, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_STRING_VARNAME", T_STRING_VARNAME, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_VARIABLE", T_VARIABLE, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_NUM_STRING", T_NUM_STRING, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_INLINE_HTML", T_INLINE_HTML, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_CHARACTER", T_CHARACTER, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_BAD_CHARACTER", T_BAD_CHARACTER, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ENCAPSED_AND_WHITESPACE", T_ENCAPSED_AND_WHITESPACE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_CONSTANT_ENCAPSED_STRING", T_CONSTANT_ENCAPSED_STRING, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_STRING_VARNAME", T_STRING_VARNAME, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_NUM_STRING", T_NUM_STRING, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_EXIT", T_EXIT, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_IF", T_IF, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_CHARACTER", T_CHARACTER, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_BAD_CHARACTER", T_BAD_CHARACTER, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ECHO", T_ECHO, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_DO", T_DO, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_WHILE", T_WHILE, CONST_CS | CONST_PERSISTENT);
@@ -118,12 +126,6 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_USE", T_USE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_INSTEADOF", T_INSTEADOF, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_GLOBAL", T_GLOBAL, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_PUBLIC", T_PUBLIC, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_PROTECTED", T_PROTECTED, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_PRIVATE", T_PRIVATE, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_FINAL", T_FINAL, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_ABSTRACT", T_ABSTRACT, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_STATIC", T_STATIC, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_VAR", T_VAR, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_UNSET", T_UNSET, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ISSET", T_ISSET, CONST_CS | CONST_PERSISTENT);
@@ -135,16 +137,16 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_EXTENDS", T_EXTENDS, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_IMPLEMENTS", T_IMPLEMENTS, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_OBJECT_OPERATOR", T_OBJECT_OPERATOR, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_DOUBLE_ARROW", T_DOUBLE_ARROW, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_LIST", T_LIST, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ARRAY", T_ARRAY, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_CALLABLE", T_CALLABLE, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_LINE", T_LINE, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_FILE", T_FILE, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_DIR", T_DIR, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_CLASS_C", T_CLASS_C, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_TRAIT_C", T_TRAIT_C, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_METHOD_C", T_METHOD_C, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_FUNC_C", T_FUNC_C, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_LINE", T_LINE, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_FILE", T_FILE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_COMMENT", T_COMMENT, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_DOC_COMMENT", T_DOC_COMMENT, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_OPEN_TAG", T_OPEN_TAG, CONST_CS | CONST_PERSISTENT);
@@ -158,7 +160,6 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_PAAMAYIM_NEKUDOTAYIM", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_NAMESPACE", T_NAMESPACE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_NS_C", T_NS_C, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_DIR", T_DIR, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_NS_SEPARATOR", T_NS_SEPARATOR, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ELLIPSIS", T_ELLIPSIS, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_DOUBLE_COLON", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT);
@@ -190,6 +191,7 @@ char *get_token_type_name(int token_type)
 		case T_MUL_EQUAL: return "T_MUL_EQUAL";
 		case T_MINUS_EQUAL: return "T_MINUS_EQUAL";
 		case T_PLUS_EQUAL: return "T_PLUS_EQUAL";
+		case T_COALESCE: return "T_COALESCE";
 		case T_BOOLEAN_OR: return "T_BOOLEAN_OR";
 		case T_BOOLEAN_AND: return "T_BOOLEAN_AND";
 		case T_IS_NOT_IDENTICAL: return "T_IS_NOT_IDENTICAL";
@@ -213,22 +215,29 @@ char *get_token_type_name(int token_type)
 		case T_POW: return "T_POW";
 		case T_CLONE: return "T_CLONE";
 		case T_NEW: return "T_NEW";
-		case T_EXIT: return "T_EXIT";
-		case T_IF: return "T_IF";
 		case T_ELSEIF: return "T_ELSEIF";
 		case T_ELSE: return "T_ELSE";
 		case T_ENDIF: return "T_ENDIF";
+		case T_PUBLIC: return "T_PUBLIC";
+		case T_PROTECTED: return "T_PROTECTED";
+		case T_PRIVATE: return "T_PRIVATE";
+		case T_FINAL: return "T_FINAL";
+		case T_ABSTRACT: return "T_ABSTRACT";
+		case T_STATIC: return "T_STATIC";
+		case T_DOUBLE_ARROW: return "T_DOUBLE_ARROW";
 		case T_LNUMBER: return "T_LNUMBER";
 		case T_DNUMBER: return "T_DNUMBER";
 		case T_STRING: return "T_STRING";
-		case T_STRING_VARNAME: return "T_STRING_VARNAME";
 		case T_VARIABLE: return "T_VARIABLE";
-		case T_NUM_STRING: return "T_NUM_STRING";
 		case T_INLINE_HTML: return "T_INLINE_HTML";
-		case T_CHARACTER: return "T_CHARACTER";
-		case T_BAD_CHARACTER: return "T_BAD_CHARACTER";
 		case T_ENCAPSED_AND_WHITESPACE: return "T_ENCAPSED_AND_WHITESPACE";
 		case T_CONSTANT_ENCAPSED_STRING: return "T_CONSTANT_ENCAPSED_STRING";
+		case T_STRING_VARNAME: return "T_STRING_VARNAME";
+		case T_NUM_STRING: return "T_NUM_STRING";
+		case T_EXIT: return "T_EXIT";
+		case T_IF: return "T_IF";
+		case T_CHARACTER: return "T_CHARACTER";
+		case T_BAD_CHARACTER: return "T_BAD_CHARACTER";
 		case T_ECHO: return "T_ECHO";
 		case T_DO: return "T_DO";
 		case T_WHILE: return "T_WHILE";
@@ -257,12 +266,6 @@ char *get_token_type_name(int token_type)
 		case T_USE: return "T_USE";
 		case T_INSTEADOF: return "T_INSTEADOF";
 		case T_GLOBAL: return "T_GLOBAL";
-		case T_PUBLIC: return "T_PUBLIC";
-		case T_PROTECTED: return "T_PROTECTED";
-		case T_PRIVATE: return "T_PRIVATE";
-		case T_FINAL: return "T_FINAL";
-		case T_ABSTRACT: return "T_ABSTRACT";
-		case T_STATIC: return "T_STATIC";
 		case T_VAR: return "T_VAR";
 		case T_UNSET: return "T_UNSET";
 		case T_ISSET: return "T_ISSET";
@@ -274,16 +277,16 @@ char *get_token_type_name(int token_type)
 		case T_EXTENDS: return "T_EXTENDS";
 		case T_IMPLEMENTS: return "T_IMPLEMENTS";
 		case T_OBJECT_OPERATOR: return "T_OBJECT_OPERATOR";
-		case T_DOUBLE_ARROW: return "T_DOUBLE_ARROW";
 		case T_LIST: return "T_LIST";
 		case T_ARRAY: return "T_ARRAY";
 		case T_CALLABLE: return "T_CALLABLE";
+		case T_LINE: return "T_LINE";
+		case T_FILE: return "T_FILE";
+		case T_DIR: return "T_DIR";
 		case T_CLASS_C: return "T_CLASS_C";
 		case T_TRAIT_C: return "T_TRAIT_C";
 		case T_METHOD_C: return "T_METHOD_C";
 		case T_FUNC_C: return "T_FUNC_C";
-		case T_LINE: return "T_LINE";
-		case T_FILE: return "T_FILE";
 		case T_COMMENT: return "T_COMMENT";
 		case T_DOC_COMMENT: return "T_DOC_COMMENT";
 		case T_OPEN_TAG: return "T_OPEN_TAG";
@@ -297,7 +300,6 @@ char *get_token_type_name(int token_type)
 		case T_PAAMAYIM_NEKUDOTAYIM: return "T_DOUBLE_COLON";
 		case T_NAMESPACE: return "T_NAMESPACE";
 		case T_NS_C: return "T_NS_C";
-		case T_DIR: return "T_DIR";
 		case T_NS_SEPARATOR: return "T_NS_SEPARATOR";
 		case T_ELLIPSIS: return "T_ELLIPSIS";