Add T_YIELD in tokenizer_data.c

Also had to fix up some tokenizer tests that were affected by the token
number changes.

8 files changed, 22 insertions, 20 deletions

diff --git a/ext/tokenizer/tests/token_get_all_variation11.phpt b/ext/tokenizer/tests/token_get_all_variation11.phpt
index ecc86177a4..98d89961b7 100644
--- a/ext/tokenizer/tests/token_get_all_variation11.phpt
+++ b/ext/tokenizer/tests/token_get_all_variation11.phpt
@@ -130,7 +130,7 @@ array(49) {
   [6]=>
   array(3) {
     [0]=>
-    int(283)
+    int(%d)
     [1]=>
     string(2) "=="
     [2]=>
@@ -273,7 +273,7 @@ array(49) {
   [27]=>
   array(3) {
     [0]=>
-    int(283)
+    int(%d)
     [1]=>
     string(2) "=="
     [2]=>
diff --git a/ext/tokenizer/tests/token_get_all_variation13.phpt b/ext/tokenizer/tests/token_get_all_variation13.phpt
index 9b2f3bc94f..6f85492a99 100644
--- a/ext/tokenizer/tests/token_get_all_variation13.phpt
+++ b/ext/tokenizer/tests/token_get_all_variation13.phpt
@@ -1005,7 +1005,7 @@ array(145) {
   [122]=>
   array(3) {
     [0]=>
-    int(288)
+    int(%d)
     [1]=>
     string(10) "instanceof"
     [2]=>
diff --git a/ext/tokenizer/tests/token_get_all_variation17.phpt b/ext/tokenizer/tests/token_get_all_variation17.phpt
index dccc4c9c23..f71444bc1e 100644
--- a/ext/tokenizer/tests/token_get_all_variation17.phpt
+++ b/ext/tokenizer/tests/token_get_all_variation17.phpt
@@ -145,7 +145,7 @@ array(81) {
   [14]=>
   array(3) {
     [0]=>
-    int(283)
+    int(%d)
     [1]=>
     string(2) "=="
     [2]=>
diff --git a/ext/tokenizer/tests/token_get_all_variation4.phpt b/ext/tokenizer/tests/token_get_all_variation4.phpt
index 45e6f8afbd..6bc111efba 100644
--- a/ext/tokenizer/tests/token_get_all_variation4.phpt
+++ b/ext/tokenizer/tests/token_get_all_variation4.phpt
@@ -339,7 +339,7 @@ array(89) {
   [38]=>
   array(3) {
     [0]=>
-    int(279)
+    int(%d)
     [1]=>
     string(2) "&&"
     [2]=>
@@ -518,7 +518,7 @@ array(89) {
   [60]=>
   array(3) {
     [0]=>
-    int(278)
+    int(%d)
     [1]=>
     string(2) "||"
     [2]=>
diff --git a/ext/tokenizer/tests/token_get_all_variation5.phpt b/ext/tokenizer/tests/token_get_all_variation5.phpt
index 0068f2866f..681fb48e57 100644
--- a/ext/tokenizer/tests/token_get_all_variation5.phpt
+++ b/ext/tokenizer/tests/token_get_all_variation5.phpt
@@ -181,7 +181,7 @@ array(94) {
   [18]=>
   array(3) {
     [0]=>
-    int(277)
+    int(%d)
     [1]=>
     string(2) "+="
     [2]=>
@@ -238,7 +238,7 @@ array(94) {
   [25]=>
   array(3) {
     [0]=>
-    int(276)
+    int(%d)
     [1]=>
     string(2) "-="
     [2]=>
@@ -295,7 +295,7 @@ array(94) {
   [32]=>
   array(3) {
     [0]=>
-    int(275)
+    int(%d)
     [1]=>
     string(2) "*="
     [2]=>
@@ -352,7 +352,7 @@ array(94) {
   [39]=>
   array(3) {
     [0]=>
-    int(274)
+    int(%d)
     [1]=>
     string(2) "/="
     [2]=>
@@ -409,7 +409,7 @@ array(94) {
   [46]=>
   array(3) {
     [0]=>
-    int(272)
+    int(%d)
     [1]=>
     string(2) "%="
     [2]=>
@@ -466,7 +466,7 @@ array(94) {
   [53]=>
   array(3) {
     [0]=>
-    int(271)
+    int(%d)
     [1]=>
     string(2) "&="
     [2]=>
@@ -523,7 +523,7 @@ array(94) {
   [60]=>
   array(3) {
     [0]=>
-    int(270)
+    int(%d)
     [1]=>
     string(2) "|="
     [2]=>
@@ -580,7 +580,7 @@ array(94) {
   [67]=>
   array(3) {
     [0]=>
-    int(269)
+    int(%d)
     [1]=>
     string(2) "^="
     [2]=>
@@ -637,7 +637,7 @@ array(94) {
   [74]=>
   array(3) {
     [0]=>
-    int(267)
+    int(%d)
     [1]=>
     string(3) ">>="
     [2]=>
@@ -694,7 +694,7 @@ array(94) {
   [81]=>
   array(3) {
     [0]=>
-    int(268)
+    int(%d)
     [1]=>
     string(3) "<<="
     [2]=>
@@ -751,7 +751,7 @@ array(94) {
   [88]=>
   array(3) {
     [0]=>
-    int(273)
+    int(%d)
     [1]=>
     string(2) ".="
     [2]=>
diff --git a/ext/tokenizer/tests/token_get_all_variation6.phpt b/ext/tokenizer/tests/token_get_all_variation6.phpt
index 54936d0c89..6213dab9d0 100644
--- a/ext/tokenizer/tests/token_get_all_variation6.phpt
+++ b/ext/tokenizer/tests/token_get_all_variation6.phpt
@@ -191,7 +191,7 @@ array(50) {
   [21]=>
   array(3) {
     [0]=>
-    int(287)
+    int(%d)
     [1]=>
     string(2) "<<"
     [2]=>
@@ -277,7 +277,7 @@ array(50) {
   [32]=>
   array(3) {
     [0]=>
-    int(286)
+    int(%d)
     [1]=>
     string(2) ">>"
     [2]=>
diff --git a/ext/tokenizer/tests/token_get_all_variation8.phpt b/ext/tokenizer/tests/token_get_all_variation8.phpt
index 0cf1d63471..c80a5d0f04 100644
--- a/ext/tokenizer/tests/token_get_all_variation8.phpt
+++ b/ext/tokenizer/tests/token_get_all_variation8.phpt
@@ -794,7 +794,7 @@ array(108) {
   [103]=>
   array(3) {
     [0]=>
-    int(289)
+    int(%d)
     [1]=>
     string(7) "(unset)"
     [2]=>
diff --git a/ext/tokenizer/tokenizer_data.c b/ext/tokenizer/tokenizer_data.c
index 85822f1994..85fa343db1 100644
--- a/ext/tokenizer/tokenizer_data.c
+++ b/ext/tokenizer/tokenizer_data.c
@@ -108,6 +108,7 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_FUNCTION", T_FUNCTION, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_CONST", T_CONST, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_RETURN", T_RETURN, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_YIELD", T_YIELD, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_TRY", T_TRY, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_CATCH", T_CATCH, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_THROW", T_THROW, CONST_CS | CONST_PERSISTENT);
@@ -242,6 +243,7 @@ char *get_token_type_name(int token_type)
 		case T_FUNCTION: return "T_FUNCTION";
 		case T_CONST: return "T_CONST";
 		case T_RETURN: return "T_RETURN";
+		case T_YIELD: return "T_YIELD";
 		case T_TRY: return "T_TRY";
 		case T_CATCH: return "T_CATCH";
 		case T_THROW: return "T_THROW";