ext tokenizer port + cleanup unused lexer states

we basically added a mechanism to store the token stream during parsing
and exposed the entire parser stack on the tokenizer extension through
an opt in flag: token_get_all($src, TOKEN_PARSE).

this change allows easy future language enhancements regarding context
aware parsing & scanning without further maintance on the tokenizer
extension while solves known inconsistencies "parseless" tokenizer
extension has when it handles `__halt_compiler()` presence.

4 files changed, 172 insertions, 4 deletions

diff --git a/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_000.phpt b/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_000.phpt
new file mode 100644
index 0000000000..03b991b1a5
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_000.phpt
@@ -0,0 +1,19 @@
+--TEST--
+Parse errors during token_get_all() with TOKEN_PARSE flag
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip"; ?>
+--FILE--
+<?php
+
+try {
+    token_get_all('<?php invalid code;', TOKEN_PARSE);
+} catch (ParseException $e) {
+    echo $e->getMessage(), PHP_EOL;
+}
+
+echo "Done";
+
+?>
+--EXPECT--
+syntax error, unexpected 'code' (T_STRING)
+Done
diff --git a/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_001.phpt b/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_001.phpt
new file mode 100644
index 0000000000..ab334358ab
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_001.phpt
@@ -0,0 +1,81 @@
+--TEST--
+Semi reserved words support: member access
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip"; ?>
+--FILE--
+<?php
+$tokens = token_get_all('<?php
+X::continue;
+X::$continue;
+$x->$continue;
+X::continue();
+$x->continue();
+X::class;
+
+class X {
+    const CONTINUE = 1;
+    public $x = self::CONTINUE + 1;
+}
+', TOKEN_PARSE);
+
+array_walk($tokens, function($tk) {
+  if(is_array($tk)) {
+    if(($t = token_name($tk[0])) == 'T_WHITESPACE') return;
+    echo "L{$tk[2]}: ".$t." {$tk[1]}", PHP_EOL;
+  }
+  else echo $tk, PHP_EOL;
+});
+
+echo "Done";
+
+?>
+--EXPECTF--
+L1: T_OPEN_TAG <?php
+
+L2: T_STRING X
+L2: T_DOUBLE_COLON ::
+L2: T_STRING continue
+;
+L3: T_STRING X
+L3: T_DOUBLE_COLON ::
+L3: T_VARIABLE $continue
+;
+L4: T_VARIABLE $x
+L4: T_OBJECT_OPERATOR ->
+L4: T_VARIABLE $continue
+;
+L5: T_STRING X
+L5: T_DOUBLE_COLON ::
+L5: T_STRING continue
+(
+)
+;
+L6: T_VARIABLE $x
+L6: T_OBJECT_OPERATOR ->
+L6: T_STRING continue
+(
+)
+;
+L7: T_STRING X
+L7: T_DOUBLE_COLON ::
+L7: T_CLASS class
+;
+L9: T_CLASS class
+L9: T_STRING X
+{
+L10: T_CONST const
+L10: T_STRING CONTINUE
+=
+L10: T_LNUMBER 1
+;
+L11: T_PUBLIC public
+L11: T_VARIABLE $x
+=
+L11: T_STRING self
+L11: T_DOUBLE_COLON ::
+L11: T_STRING CONTINUE
++
+L11: T_LNUMBER 1
+;
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_002.phpt b/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_002.phpt
new file mode 100644
index 0000000000..3dd8e14d84
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_TOKEN_PARSE_002.phpt
@@ -0,0 +1,68 @@
+--TEST--
+Semi reserved words support: class const
+--SKIPIF--
+<?php if (!extension_loaded("tokenizer")) print "skip"; ?>
+--FILE--
+<?php
+$tokens = token_get_all('<?php
+  class SomeClass {
+      const CONST = 1;
+      const CONTINUE = (self::CONST + 1);
+      const ARRAY = [1, self::CONTINUE => [3, 4], 5];
+  }
+', TOKEN_PARSE);
+
+array_walk($tokens, function($tk) {
+  if(is_array($tk)) {
+    if(($t = token_name($tk[0])) == 'T_WHITESPACE') return;
+    echo "L{$tk[2]}: ".$t." {$tk[1]}", PHP_EOL;
+  }
+  else echo $tk, PHP_EOL;
+});
+
+echo "Done";
+
+?>
+--EXPECTF--
+L1: T_OPEN_TAG <?php
+
+L2: T_CLASS class
+L2: T_STRING SomeClass
+{
+L3: T_CONST const
+L3: T_STRING CONST
+=
+L3: T_LNUMBER 1
+;
+L4: T_CONST const
+L4: T_STRING CONTINUE
+=
+(
+L4: T_STRING self
+L4: T_DOUBLE_COLON ::
+L4: T_STRING CONST
++
+L4: T_LNUMBER 1
+)
+;
+L5: T_CONST const
+L5: T_STRING ARRAY
+=
+[
+L5: T_LNUMBER 1
+,
+L5: T_STRING self
+L5: T_DOUBLE_COLON ::
+L5: T_STRING CONTINUE
+L5: T_DOUBLE_ARROW =>
+[
+L5: T_LNUMBER 3
+,
+L5: T_LNUMBER 4
+]
+,
+L5: T_LNUMBER 5
+]
+;
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_error.phpt b/ext/tokenizer/tests/token_get_all_error.phpt
index 29e97c38c4..9ded0a1774 100644
--- a/ext/tokenizer/tests/token_get_all_error.phpt
+++ b/ext/tokenizer/tests/token_get_all_error.phpt
@@ -19,7 +19,7 @@ var_dump( token_get_all());
 echo "-- Testing token_get_all() function with more than expected no. of arguments --\n";
 $source = '<?php ?>';
 $extra_arg = 10;
-var_dump( token_get_all($source, $extra_arg));
+var_dump( token_get_all($source, true, $extra_arg));
 
 echo "Done"
 ?>
@@ -28,10 +28,10 @@ echo "Done"
 
 -- Testing token_get_all() function with zero arguments --
 
-Warning: token_get_all() expects exactly 1 parameter, 0 given in %s on line %d
+Warning: token_get_all() expects at least 1 parameter, 0 given in %s on line 11
 NULL
 -- Testing token_get_all() function with more than expected no. of arguments --
 
-Warning: token_get_all() expects exactly 1 parameter, 2 given in %s on line %d
+Warning: token_get_all() expects at most 2 parameters, 3 given in %s on line 17
 NULL
-Done
+Done
\ No newline at end of file