New testcases for token_get_all() function

20 files changed, 11305 insertions, 0 deletions

diff --git a/ext/tokenizer/tests/token_get_all_basic.phpt b/ext/tokenizer/tests/token_get_all_basic.phpt
new file mode 100644
index 0000000000..0ae84a2a75
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_basic.phpt
@@ -0,0 +1,97 @@
+--TEST--
+Test token_get_all() function : basic functionality 
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description : splits the given source into an array of PHP languange tokens
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+echo "*** Testing token_get_all() : basic functionality ***\n";
+
+// with php open/close tags
+$source = '<?php echo "Hello World"; ?>';
+echo "-- source string with PHP open and close tags --\n";
+var_dump( token_get_all($source) );
+
+// without php open/close tags testing for T_INLINE_HTML
+$source = "echo 'Hello World';";
+echo "-- source string without PHP open and close tags --\n";
+var_dump( token_get_all($source) );
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : basic functionality ***
+-- source string with PHP open and close tags --
+array(7) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [3]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(13) ""Hello World""
+    [2]=>
+    int(1)
+  }
+  [4]=>
+  string(1) ";"
+  [5]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(1)
+  }
+}
+-- source string without PHP open and close tags --
+array(1) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(19) "echo 'Hello World';"
+    [2]=>
+    int(1)
+  }
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_error.phpt b/ext/tokenizer/tests/token_get_all_error.phpt
new file mode 100644
index 0000000000..52b1efc1dd
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_error.phpt
@@ -0,0 +1,35 @@
+--TEST--
+Test token_get_all() function : error conditions 
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens 
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+echo "*** Testing token_get_all() : error conditions ***\n";
+
+// with zero arguments
+echo "\n-- Testing token_get_all() function with zero arguments --\n";
+var_dump( token_get_all());
+
+// with one more than the expected number of arguments
+echo "-- Testing token_get_all() function with more than expected no. of arguments --\n";
+$source = '<?php ?>';
+$extra_arg = 10;
+var_dump( token_get_all($source, $extra_arg));
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : error conditions ***
+
+-- Testing token_get_all() function with zero arguments --
+
+Warning: token_get_all() expects exactly 1 parameter, 0 given in %s on line %d
+NULL
+-- Testing token_get_all() function with more than expected no. of arguments --
+
+Warning: token_get_all() expects exactly 1 parameter, 2 given in %s on line %d
+NULL
+Done
diff --git a/ext/tokenizer/tests/token_get_all_variation1.phpt b/ext/tokenizer/tests/token_get_all_variation1.phpt
new file mode 100644
index 0000000000..f0e10d6d5c
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_variation1.phpt
@@ -0,0 +1,285 @@
+--TEST--
+Test token_get_all() function : usage variations - unexpected values for 'source' argument
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens 
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+/*
+ * Passing different scalar/non-scalar values in place of 'source' argument
+ *   It returns either T_INLINE_HTML by converting values into string or gives warning
+*/
+
+echo "*** Testing token_get_all() : unexpected values for 'source' argument ***\n";
+
+// get an unset variable
+$unset_var = 10;
+unset ($unset_var);
+
+// class definition
+class MyClass
+{
+  public function __toString()
+  {
+    return "object";
+  }
+}
+
+// get resource
+$fp = fopen(__FILE__, 'r');
+
+// different scalar/nonscalar values for 'source'
+$source_values = array(
+
+       // int data
+/*1*/  0,
+       1,
+       12345,
+       -2345,
+
+       // float data
+/*5*/  10.5,
+       -10.5,
+       10.1234567e8,
+       10.7654321E-8,
+       .5,
+
+       // array data
+/*10*/ array(),
+       array(0),
+       array(1),
+       array(1, 2),
+       array('color' => 'red', 'item' => 'pen'),
+
+       // null data
+/*15*/ NULL,
+       null,
+
+       // boolean data
+/*17*/ true,
+       false,
+       TRUE,
+       FALSE,
+
+       // empty string
+/*21*/ "",
+       '',
+
+       // object data
+/*23*/ new MyClass(),
+ 
+       // resource data
+       $fp,
+
+       // undefined data
+       @$undefined_var,
+
+       // unset data
+/*26*/ @$unset_var,
+);
+
+for($count = 0; $count < count($source_values); $count++) {
+  echo "--Iteration ".($count + 1)." --\n";
+  var_dump( token_get_all($source_values[$count]));
+};
+
+fclose($fp);
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : unexpected values for 'source' argument ***
+--Iteration 1 --
+array(1) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(1) "0"
+    [2]=>
+    int(1)
+  }
+}
+--Iteration 2 --
+array(1) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(1)
+  }
+}
+--Iteration 3 --
+array(1) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(5) "12345"
+    [2]=>
+    int(1)
+  }
+}
+--Iteration 4 --
+array(1) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(5) "-2345"
+    [2]=>
+    int(1)
+  }
+}
+--Iteration 5 --
+array(1) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(4) "10.5"
+    [2]=>
+    int(1)
+  }
+}
+--Iteration 6 --
+array(1) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(5) "-10.5"
+    [2]=>
+    int(1)
+  }
+}
+--Iteration 7 --
+array(1) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(10) "1012345670"
+    [2]=>
+    int(1)
+  }
+}
+--Iteration 8 --
+array(1) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(13) "1.07654321E-7"
+    [2]=>
+    int(1)
+  }
+}
+--Iteration 9 --
+array(1) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(3) "0.5"
+    [2]=>
+    int(1)
+  }
+}
+--Iteration 10 --
+
+Warning: token_get_all() expects parameter 1 to be string, array given in %s on line %d
+NULL
+--Iteration 11 --
+
+Warning: token_get_all() expects parameter 1 to be string, array given in %s on line %d
+NULL
+--Iteration 12 --
+
+Warning: token_get_all() expects parameter 1 to be string, array given in %s on line %d
+NULL
+--Iteration 13 --
+
+Warning: token_get_all() expects parameter 1 to be string, array given in %s on line %d
+NULL
+--Iteration 14 --
+
+Warning: token_get_all() expects parameter 1 to be string, array given in %s on line %d
+NULL
+--Iteration 15 --
+array(0) {
+}
+--Iteration 16 --
+array(0) {
+}
+--Iteration 17 --
+array(1) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(1)
+  }
+}
+--Iteration 18 --
+array(0) {
+}
+--Iteration 19 --
+array(1) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(1)
+  }
+}
+--Iteration 20 --
+array(0) {
+}
+--Iteration 21 --
+array(0) {
+}
+--Iteration 22 --
+array(0) {
+}
+--Iteration 23 --
+array(1) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(6) "object"
+    [2]=>
+    int(1)
+  }
+}
+--Iteration 24 --
+
+Warning: token_get_all() expects parameter 1 to be string, resource given in %s on line %d
+NULL
+--Iteration 25 --
+array(0) {
+}
+--Iteration 26 --
+array(0) {
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_variation10.phpt b/ext/tokenizer/tests/token_get_all_variation10.phpt
new file mode 100644
index 0000000000..63b61e731b
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_variation10.phpt
@@ -0,0 +1,781 @@
+--TEST--
+Test token_get_all() function : usage variations - with constant tokens
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+/*
+ * Using different types of constants in 'source' string to check them for token
+ * integer const - T_LNUMBER(305)
+ * float/double/real const - T_DNUMBER(306)
+ * string cosnt - T_CONSTANT_ESCAPED_STRING(315)
+ * bool const (no tokens specified) - T_UNKNOWN(307)
+ * null const (no tokens specified) - T_UNKNOWN(307)
+*/
+
+echo "*** Testing token_get_all() :  'source' string with different constants ***\n";
+
+$a = 1;
+$b = 0;
+
+$source = array (
+  // int const
+  '<?php $a = 1 + 034; $b = $a + 0x3F; ?>', 
+  
+  // float const
+  '<?php $a = 0.23E-2 + 0.43e2 + 0.5; ?>',
+
+  // string const
+  '<?php $a = "hello ".\'world\'; ?>',  
+
+  // bool const
+  "<?php \$a = (\$b)? true : false; ?>",  
+  "<?php \$b = (\$a)? FALSE : TRUE; ?>",  
+
+  // null const
+  '<?php $b = null | NULL; ?>'
+);
+for($count = 0; $count < count($source); $count++) {
+  echo "-- Iteration ".($count + 1)." --\n";
+  var_dump( token_get_all($source[$count]));
+}
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() :  'source' string with different constants ***
+-- Iteration 1 --
+array(24) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [3]=>
+  string(1) "="
+  [4]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(1)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [7]=>
+  string(1) "+"
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(3) "034"
+    [2]=>
+    int(1)
+  }
+  [10]=>
+  string(1) ";"
+  [11]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [12]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(1)
+  }
+  [13]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [14]=>
+  string(1) "="
+  [15]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [16]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(1)
+  }
+  [17]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [18]=>
+  string(1) "+"
+  [19]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [20]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(4) "0x3F"
+    [2]=>
+    int(1)
+  }
+  [21]=>
+  string(1) ";"
+  [22]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [23]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(1)
+  }
+}
+-- Iteration 2 --
+array(17) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [3]=>
+  string(1) "="
+  [4]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(306)
+    [1]=>
+    string(7) "0.23E-2"
+    [2]=>
+    int(1)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [7]=>
+  string(1) "+"
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(306)
+    [1]=>
+    string(6) "0.43e2"
+    [2]=>
+    int(1)
+  }
+  [10]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [11]=>
+  string(1) "+"
+  [12]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [13]=>
+  array(3) {
+    [0]=>
+    int(306)
+    [1]=>
+    string(3) "0.5"
+    [2]=>
+    int(1)
+  }
+  [14]=>
+  string(1) ";"
+  [15]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [16]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(1)
+  }
+}
+-- Iteration 3 --
+array(11) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [3]=>
+  string(1) "="
+  [4]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(8) ""hello ""
+    [2]=>
+    int(1)
+  }
+  [6]=>
+  string(1) "."
+  [7]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(7) "'world'"
+    [2]=>
+    int(1)
+  }
+  [8]=>
+  string(1) ";"
+  [9]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [10]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(1)
+  }
+}
+-- Iteration 4 --
+array(18) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [3]=>
+  string(1) "="
+  [4]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [5]=>
+  string(1) "("
+  [6]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(1)
+  }
+  [7]=>
+  string(1) ")"
+  [8]=>
+  string(1) "?"
+  [9]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [10]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(4) "true"
+    [2]=>
+    int(1)
+  }
+  [11]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [12]=>
+  string(1) ":"
+  [13]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [14]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(5) "false"
+    [2]=>
+    int(1)
+  }
+  [15]=>
+  string(1) ";"
+  [16]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [17]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(1)
+  }
+}
+-- Iteration 5 --
+array(18) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [3]=>
+  string(1) "="
+  [4]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [5]=>
+  string(1) "("
+  [6]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(1)
+  }
+  [7]=>
+  string(1) ")"
+  [8]=>
+  string(1) "?"
+  [9]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [10]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(5) "FALSE"
+    [2]=>
+    int(1)
+  }
+  [11]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [12]=>
+  string(1) ":"
+  [13]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [14]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(4) "TRUE"
+    [2]=>
+    int(1)
+  }
+  [15]=>
+  string(1) ";"
+  [16]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [17]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(1)
+  }
+}
+-- Iteration 6 --
+array(13) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [3]=>
+  string(1) "="
+  [4]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(4) "null"
+    [2]=>
+    int(1)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [7]=>
+  string(1) "|"
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(4) "NULL"
+    [2]=>
+    int(1)
+  }
+  [10]=>
+  string(1) ";"
+  [11]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [12]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(1)
+  }
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_variation11.phpt b/ext/tokenizer/tests/token_get_all_variation11.phpt
new file mode 100644
index 0000000000..c38b050cb0
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_variation11.phpt
@@ -0,0 +1,1147 @@
+--TEST--
+Test token_get_all() function : usage variations - with control structure tokens
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+/*
+ * Using different control structure keywords 
+ *   if..else, elseif - T_IF(301), T_ELSEIF(302), T_ELSE(303)
+ *   while - T_WHILE(318)
+ *   do...while - T_DO(317)
+ *   for - T_ENDFOR(320)
+ *   foreach - T_ENDFOREACH(322)
+ *   switch...case - T_ENDSWITCH(327), T_CASE(329)
+ *   break - T_BREAK(331)
+ *   continue - T_CONTINUE(332)
+*/
+
+echo "*** Testing token_get_all() : for control structure tokens ***\n";
+
+// if..elseif....else
+echo "-- with if..elseif..else..tokens --\n";
+
+$source = '<?php 
+if($a == true) {
+     echo "$a = true";
+}
+elseif($a == false) {
+  echo false;
+}
+else
+  echo 1;
+?>';
+
+var_dump( token_get_all($source));
+
+// while..., do..while, break, continue
+echo "-- with while..., do..while, switch & continue tokens --\n";
+
+$source = "<?php while(true) {
+echo 'True';
+break;
+}
+do {
+continue;
+}while(false); ?>";
+
+var_dump( token_get_all($source));
+
+// for..., foreach( as )
+echo "-- with for..foreach( as ) tokens --\n";
+
+$source = '<?php for($count=0; $count < 5; $count++) {
+echo $count;
+}
+foreach($values as $index) {
+continue;
+} ?>';
+
+var_dump( token_get_all($source));
+
+// switch..case, default
+echo "-- with switch...case tokens --\n";
+
+$source = '<?php switch($var)
+case 1: break;
+default: echo "default"; ?>';
+
+var_dump( token_get_all($source));
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : for control structure tokens ***
+-- with if..elseif..else..tokens --
+array(49) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(301)
+    [1]=>
+    string(2) "if"
+    [2]=>
+    int(2)
+  }
+  [3]=>
+  string(1) "("
+  [4]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(2)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(283)
+    [1]=>
+    string(2) "=="
+    [2]=>
+    int(2)
+  }
+  [7]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [8]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(4) "true"
+    [2]=>
+    int(2)
+  }
+  [9]=>
+  string(1) ")"
+  [10]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [11]=>
+  string(1) "{"
+  [12]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(6) "
+     "
+    [2]=>
+    int(2)
+  }
+  [13]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(3)
+  }
+  [14]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [15]=>
+  string(1) """
+  [16]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(3)
+  }
+  [17]=>
+  array(3) {
+    [0]=>
+    int(314)
+    [1]=>
+    string(7) " = true"
+    [2]=>
+    int(3)
+  }
+  [18]=>
+  string(1) """
+  [19]=>
+  string(1) ";"
+  [20]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(3)
+  }
+  [21]=>
+  string(1) "}"
+  [22]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(4)
+  }
+  [23]=>
+  array(3) {
+    [0]=>
+    int(302)
+    [1]=>
+    string(6) "elseif"
+    [2]=>
+    int(5)
+  }
+  [24]=>
+  string(1) "("
+  [25]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(5)
+  }
+  [26]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(5)
+  }
+  [27]=>
+  array(3) {
+    [0]=>
+    int(283)
+    [1]=>
+    string(2) "=="
+    [2]=>
+    int(5)
+  }
+  [28]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(5)
+  }
+  [29]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(5) "false"
+    [2]=>
+    int(5)
+  }
+  [30]=>
+  string(1) ")"
+  [31]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(5)
+  }
+  [32]=>
+  string(1) "{"
+  [33]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(5)
+  }
+  [34]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(6)
+  }
+  [35]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(6)
+  }
+  [36]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(5) "false"
+    [2]=>
+    int(6)
+  }
+  [37]=>
+  string(1) ";"
+  [38]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(6)
+  }
+  [39]=>
+  string(1) "}"
+  [40]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(7)
+  }
+  [41]=>
+  array(3) {
+    [0]=>
+    int(303)
+    [1]=>
+    string(4) "else"
+    [2]=>
+    int(8)
+  }
+  [42]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(8)
+  }
+  [43]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(9)
+  }
+  [44]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(9)
+  }
+  [45]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(9)
+  }
+  [46]=>
+  string(1) ";"
+  [47]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(9)
+  }
+  [48]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(10)
+  }
+}
+-- with while..., do..while, switch & continue tokens --
+array(33) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(318)
+    [1]=>
+    string(5) "while"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  string(1) "("
+  [3]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(4) "true"
+    [2]=>
+    int(1)
+  }
+  [4]=>
+  string(1) ")"
+  [5]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [6]=>
+  string(1) "{"
+  [7]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(1)
+  }
+  [8]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(2)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [10]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(6) "'True'"
+    [2]=>
+    int(2)
+  }
+  [11]=>
+  string(1) ";"
+  [12]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(2)
+  }
+  [13]=>
+  array(3) {
+    [0]=>
+    int(331)
+    [1]=>
+    string(5) "break"
+    [2]=>
+    int(3)
+  }
+  [14]=>
+  string(1) ";"
+  [15]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(3)
+  }
+  [16]=>
+  string(1) "}"
+  [17]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(4)
+  }
+  [18]=>
+  array(3) {
+    [0]=>
+    int(317)
+    [1]=>
+    string(2) "do"
+    [2]=>
+    int(5)
+  }
+  [19]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(5)
+  }
+  [20]=>
+  string(1) "{"
+  [21]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(5)
+  }
+  [22]=>
+  array(3) {
+    [0]=>
+    int(332)
+    [1]=>
+    string(8) "continue"
+    [2]=>
+    int(6)
+  }
+  [23]=>
+  string(1) ";"
+  [24]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(6)
+  }
+  [25]=>
+  string(1) "}"
+  [26]=>
+  array(3) {
+    [0]=>
+    int(318)
+    [1]=>
+    string(5) "while"
+    [2]=>
+    int(7)
+  }
+  [27]=>
+  string(1) "("
+  [28]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(5) "false"
+    [2]=>
+    int(7)
+  }
+  [29]=>
+  string(1) ")"
+  [30]=>
+  string(1) ";"
+  [31]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(7)
+  }
+  [32]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(7)
+  }
+}
+-- with for..foreach( as ) tokens --
+array(45) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(320)
+    [1]=>
+    string(3) "for"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  string(1) "("
+  [3]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(6) "$count"
+    [2]=>
+    int(1)
+  }
+  [4]=>
+  string(1) "="
+  [5]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "0"
+    [2]=>
+    int(1)
+  }
+  [6]=>
+  string(1) ";"
+  [7]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [8]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(6) "$count"
+    [2]=>
+    int(1)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [10]=>
+  string(1) "<"
+  [11]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [12]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "5"
+    [2]=>
+    int(1)
+  }
+  [13]=>
+  string(1) ";"
+  [14]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [15]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(6) "$count"
+    [2]=>
+    int(1)
+  }
+  [16]=>
+  array(3) {
+    [0]=>
+    int(297)
+    [1]=>
+    string(2) "++"
+    [2]=>
+    int(1)
+  }
+  [17]=>
+  string(1) ")"
+  [18]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [19]=>
+  string(1) "{"
+  [20]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(1)
+  }
+  [21]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(2)
+  }
+  [22]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [23]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(6) "$count"
+    [2]=>
+    int(2)
+  }
+  [24]=>
+  string(1) ";"
+  [25]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(2)
+  }
+  [26]=>
+  string(1) "}"
+  [27]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(3)
+  }
+  [28]=>
+  array(3) {
+    [0]=>
+    int(322)
+    [1]=>
+    string(7) "foreach"
+    [2]=>
+    int(4)
+  }
+  [29]=>
+  string(1) "("
+  [30]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(7) "$values"
+    [2]=>
+    int(4)
+  }
+  [31]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [32]=>
+  array(3) {
+    [0]=>
+    int(326)
+    [1]=>
+    string(2) "as"
+    [2]=>
+    int(4)
+  }
+  [33]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [34]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(6) "$index"
+    [2]=>
+    int(4)
+  }
+  [35]=>
+  string(1) ")"
+  [36]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [37]=>
+  string(1) "{"
+  [38]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(4)
+  }
+  [39]=>
+  array(3) {
+    [0]=>
+    int(332)
+    [1]=>
+    string(8) "continue"
+    [2]=>
+    int(5)
+  }
+  [40]=>
+  string(1) ";"
+  [41]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(5)
+  }
+  [42]=>
+  string(1) "}"
+  [43]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(6)
+  }
+  [44]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(6)
+  }
+}
+-- with switch...case tokens --
+array(23) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(327)
+    [1]=>
+    string(6) "switch"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  string(1) "("
+  [3]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(4) "$var"
+    [2]=>
+    int(1)
+  }
+  [4]=>
+  string(1) ")"
+  [5]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(1)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(329)
+    [1]=>
+    string(4) "case"
+    [2]=>
+    int(2)
+  }
+  [7]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [8]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(2)
+  }
+  [9]=>
+  string(1) ":"
+  [10]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [11]=>
+  array(3) {
+    [0]=>
+    int(331)
+    [1]=>
+    string(5) "break"
+    [2]=>
+    int(2)
+  }
+  [12]=>
+  string(1) ";"
+  [13]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(2)
+  }
+  [14]=>
+  array(3) {
+    [0]=>
+    int(330)
+    [1]=>
+    string(7) "default"
+    [2]=>
+    int(3)
+  }
+  [15]=>
+  string(1) ":"
+  [16]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [17]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(3)
+  }
+  [18]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [19]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(9) ""default""
+    [2]=>
+    int(3)
+  }
+  [20]=>
+  string(1) ";"
+  [21]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [22]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(3)
+  }
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_variation12.phpt b/ext/tokenizer/tests/token_get_all_variation12.phpt
new file mode 100644
index 0000000000..8394cb6267
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_variation12.phpt
@@ -0,0 +1,532 @@
+--TEST--
+Test token_get_all() function : usage variations - with predefined language constants
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+/*
+ * Testing token_get_all() with following predefined language constants:
+ *   __FILE__ - T_FILE(364)
+ *   __CLASS__ - T_CLASS_C(360)
+ *   __FUNCTION__ - T_FUNC_C(362)
+ *   __LINE__ - T_LINE(363)
+ *   __METHOD__ - T_METHOD_C(361)
+*/
+
+echo "*** Testing token_get_all() : with language constants ***\n";
+
+// parsing __FILE__ token
+echo "-- with FILE --\n";
+$source = "<?php 
+\$fp =  fopen(__FILE__, 'r');
+?>";
+var_dump( token_get_all($source));
+
+// parsing __CLASS__ and __FUNCTION__ tokens
+echo "-- with CLASS and FUNCTION --\n";
+$source = '<?php
+class MyClass
+{
+  echo  __CLASS__;
+  function myFunction()
+  {  echo  __FUNCTION__; }
+}
+?>';
+var_dump( token_get_all($source));
+
+// parsing __LINE__ and __METHOD__ tokens
+echo "-- with LINE and METHOD --\n";
+$source = '<?php
+  $a = __LINE__;
+  $b = $b.__METHOD__;
+?>';
+var_dump( token_get_all($source));
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : with language constants ***
+-- with FILE --
+array(16) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(3) "$fp"
+    [2]=>
+    int(2)
+  }
+  [3]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [4]=>
+  string(1) "="
+  [5]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(2) "  "
+    [2]=>
+    int(2)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(5) "fopen"
+    [2]=>
+    int(2)
+  }
+  [7]=>
+  string(1) "("
+  [8]=>
+  array(3) {
+    [0]=>
+    int(364)
+    [1]=>
+    string(8) "__FILE__"
+    [2]=>
+    int(2)
+  }
+  [9]=>
+  string(1) ","
+  [10]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [11]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(3) "'r'"
+    [2]=>
+    int(2)
+  }
+  [12]=>
+  string(1) ")"
+  [13]=>
+  string(1) ";"
+  [14]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(2)
+  }
+  [15]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(3)
+  }
+}
+-- with CLASS and FUNCTION --
+array(30) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php
+"
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(352)
+    [1]=>
+    string(5) "class"
+    [2]=>
+    int(2)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [3]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(7) "MyClass"
+    [2]=>
+    int(2)
+  }
+  [4]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(2)
+  }
+  [5]=>
+  string(1) "{"
+  [6]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(3)
+  }
+  [7]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(4)
+  }
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(2) "  "
+    [2]=>
+    int(4)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(360)
+    [1]=>
+    string(9) "__CLASS__"
+    [2]=>
+    int(4)
+  }
+  [10]=>
+  string(1) ";"
+  [11]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(4)
+  }
+  [12]=>
+  array(3) {
+    [0]=>
+    int(333)
+    [1]=>
+    string(8) "function"
+    [2]=>
+    int(5)
+  }
+  [13]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(5)
+  }
+  [14]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(10) "myFunction"
+    [2]=>
+    int(5)
+  }
+  [15]=>
+  string(1) "("
+  [16]=>
+  string(1) ")"
+  [17]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(5)
+  }
+  [18]=>
+  string(1) "{"
+  [19]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(2) "  "
+    [2]=>
+    int(6)
+  }
+  [20]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(6)
+  }
+  [21]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(2) "  "
+    [2]=>
+    int(6)
+  }
+  [22]=>
+  array(3) {
+    [0]=>
+    int(362)
+    [1]=>
+    string(12) "__FUNCTION__"
+    [2]=>
+    int(6)
+  }
+  [23]=>
+  string(1) ";"
+  [24]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(6)
+  }
+  [25]=>
+  string(1) "}"
+  [26]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(6)
+  }
+  [27]=>
+  string(1) "}"
+  [28]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(7)
+  }
+  [29]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(8)
+  }
+}
+-- with LINE and METHOD --
+array(19) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php
+"
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(2) "  "
+    [2]=>
+    int(2)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(2)
+  }
+  [3]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [4]=>
+  string(1) "="
+  [5]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(363)
+    [1]=>
+    string(8) "__LINE__"
+    [2]=>
+    int(2)
+  }
+  [7]=>
+  string(1) ";"
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(2)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(3)
+  }
+  [10]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [11]=>
+  string(1) "="
+  [12]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [13]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(3)
+  }
+  [14]=>
+  string(1) "."
+  [15]=>
+  array(3) {
+    [0]=>
+    int(361)
+    [1]=>
+    string(10) "__METHOD__"
+    [2]=>
+    int(3)
+  }
+  [16]=>
+  string(1) ";"
+  [17]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(3)
+  }
+  [18]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(4)
+  }
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_variation13.phpt b/ext/tokenizer/tests/token_get_all_variation13.phpt
new file mode 100644
index 0000000000..e79545e7e4
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_variation13.phpt
@@ -0,0 +1,1167 @@
+--TEST--
+Test token_get_all() function : usage variations - with class/object constructs
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+/*
+ * Testing token_get_all() with different class/object keywords
+ *   scope related : 
+ *     static - T_STATIC(346), global - T_GLOBAL(340),
+ *     private - T_PRIVATE(343), public - T_PUBLIC(341),
+ *     protected - T_PROTECTED(342)
+ *   class/object related : 
+ *     var - T_VAR(347), abstract - T_ABSTRACT(345), 
+ *     interface - T_INTERFACE(353), class - T_CLASS(352),
+ *     extends - T_EXTENDS(354), implements - T_IMPLEMENTS(355), new - T_NEW(299)
+*/
+
+echo "*** Testing token_get_all() : with class/object constructs ***\n";
+
+$source = '<?php
+interface MyInterface
+{
+  public const var $var = 10;
+}
+abstract class MyClass
+{
+  private var $a;
+  public var $b;
+  protected var $c;
+  static $d;
+  final $e = 10;
+  
+  abstract public function myFunction($a);
+}
+class ChildClass extends MyClass implements MyInterface
+{
+  global $value;
+  function myFunction($a)
+  {
+    $a = new ChildClass();
+    if($a instanceof MyClass)
+      echo "object created";
+  }
+}
+ChildClass::myFunction(10);
+?>';
+$tokens =  token_get_all($source);
+var_dump($tokens);
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : with class/object constructs ***
+array(145) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php
+"
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(353)
+    [1]=>
+    string(9) "interface"
+    [2]=>
+    int(2)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [3]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(11) "MyInterface"
+    [2]=>
+    int(2)
+  }
+  [4]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(2)
+  }
+  [5]=>
+  string(1) "{"
+  [6]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(3)
+  }
+  [7]=>
+  array(3) {
+    [0]=>
+    int(341)
+    [1]=>
+    string(6) "public"
+    [2]=>
+    int(4)
+  }
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(334)
+    [1]=>
+    string(5) "const"
+    [2]=>
+    int(4)
+  }
+  [10]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [11]=>
+  array(3) {
+    [0]=>
+    int(347)
+    [1]=>
+    string(3) "var"
+    [2]=>
+    int(4)
+  }
+  [12]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [13]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(4) "$var"
+    [2]=>
+    int(4)
+  }
+  [14]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [15]=>
+  string(1) "="
+  [16]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [17]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(2) "10"
+    [2]=>
+    int(4)
+  }
+  [18]=>
+  string(1) ";"
+  [19]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(4)
+  }
+  [20]=>
+  string(1) "}"
+  [21]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(5)
+  }
+  [22]=>
+  array(3) {
+    [0]=>
+    int(345)
+    [1]=>
+    string(8) "abstract"
+    [2]=>
+    int(6)
+  }
+  [23]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(6)
+  }
+  [24]=>
+  array(3) {
+    [0]=>
+    int(352)
+    [1]=>
+    string(5) "class"
+    [2]=>
+    int(6)
+  }
+  [25]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(6)
+  }
+  [26]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(7) "MyClass"
+    [2]=>
+    int(6)
+  }
+  [27]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(6)
+  }
+  [28]=>
+  string(1) "{"
+  [29]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(7)
+  }
+  [30]=>
+  array(3) {
+    [0]=>
+    int(343)
+    [1]=>
+    string(7) "private"
+    [2]=>
+    int(8)
+  }
+  [31]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [32]=>
+  array(3) {
+    [0]=>
+    int(347)
+    [1]=>
+    string(3) "var"
+    [2]=>
+    int(8)
+  }
+  [33]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [34]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(8)
+  }
+  [35]=>
+  string(1) ";"
+  [36]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(8)
+  }
+  [37]=>
+  array(3) {
+    [0]=>
+    int(341)
+    [1]=>
+    string(6) "public"
+    [2]=>
+    int(9)
+  }
+  [38]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(9)
+  }
+  [39]=>
+  array(3) {
+    [0]=>
+    int(347)
+    [1]=>
+    string(3) "var"
+    [2]=>
+    int(9)
+  }
+  [40]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(9)
+  }
+  [41]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(9)
+  }
+  [42]=>
+  string(1) ";"
+  [43]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(9)
+  }
+  [44]=>
+  array(3) {
+    [0]=>
+    int(342)
+    [1]=>
+    string(9) "protected"
+    [2]=>
+    int(10)
+  }
+  [45]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(10)
+  }
+  [46]=>
+  array(3) {
+    [0]=>
+    int(347)
+    [1]=>
+    string(3) "var"
+    [2]=>
+    int(10)
+  }
+  [47]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(10)
+  }
+  [48]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$c"
+    [2]=>
+    int(10)
+  }
+  [49]=>
+  string(1) ";"
+  [50]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(10)
+  }
+  [51]=>
+  array(3) {
+    [0]=>
+    int(346)
+    [1]=>
+    string(6) "static"
+    [2]=>
+    int(11)
+  }
+  [52]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(11)
+  }
+  [53]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$d"
+    [2]=>
+    int(11)
+  }
+  [54]=>
+  string(1) ";"
+  [55]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(11)
+  }
+  [56]=>
+  array(3) {
+    [0]=>
+    int(344)
+    [1]=>
+    string(5) "final"
+    [2]=>
+    int(12)
+  }
+  [57]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(12)
+  }
+  [58]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$e"
+    [2]=>
+    int(12)
+  }
+  [59]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(12)
+  }
+  [60]=>
+  string(1) "="
+  [61]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(12)
+  }
+  [62]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(2) "10"
+    [2]=>
+    int(12)
+  }
+  [63]=>
+  string(1) ";"
+  [64]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(6) "
+  
+  "
+    [2]=>
+    int(12)
+  }
+  [65]=>
+  array(3) {
+    [0]=>
+    int(345)
+    [1]=>
+    string(8) "abstract"
+    [2]=>
+    int(14)
+  }
+  [66]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(14)
+  }
+  [67]=>
+  array(3) {
+    [0]=>
+    int(341)
+    [1]=>
+    string(6) "public"
+    [2]=>
+    int(14)
+  }
+  [68]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(14)
+  }
+  [69]=>
+  array(3) {
+    [0]=>
+    int(333)
+    [1]=>
+    string(8) "function"
+    [2]=>
+    int(14)
+  }
+  [70]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(14)
+  }
+  [71]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(10) "myFunction"
+    [2]=>
+    int(14)
+  }
+  [72]=>
+  string(1) "("
+  [73]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(14)
+  }
+  [74]=>
+  string(1) ")"
+  [75]=>
+  string(1) ";"
+  [76]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(14)
+  }
+  [77]=>
+  string(1) "}"
+  [78]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(15)
+  }
+  [79]=>
+  array(3) {
+    [0]=>
+    int(352)
+    [1]=>
+    string(5) "class"
+    [2]=>
+    int(16)
+  }
+  [80]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(16)
+  }
+  [81]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(10) "ChildClass"
+    [2]=>
+    int(16)
+  }
+  [82]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(16)
+  }
+  [83]=>
+  array(3) {
+    [0]=>
+    int(354)
+    [1]=>
+    string(7) "extends"
+    [2]=>
+    int(16)
+  }
+  [84]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(16)
+  }
+  [85]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(7) "MyClass"
+    [2]=>
+    int(16)
+  }
+  [86]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(16)
+  }
+  [87]=>
+  array(3) {
+    [0]=>
+    int(355)
+    [1]=>
+    string(10) "implements"
+    [2]=>
+    int(16)
+  }
+  [88]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(16)
+  }
+  [89]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(11) "MyInterface"
+    [2]=>
+    int(16)
+  }
+  [90]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(16)
+  }
+  [91]=>
+  string(1) "{"
+  [92]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(17)
+  }
+  [93]=>
+  array(3) {
+    [0]=>
+    int(340)
+    [1]=>
+    string(6) "global"
+    [2]=>
+    int(18)
+  }
+  [94]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(18)
+  }
+  [95]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(6) "$value"
+    [2]=>
+    int(18)
+  }
+  [96]=>
+  string(1) ";"
+  [97]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(18)
+  }
+  [98]=>
+  array(3) {
+    [0]=>
+    int(333)
+    [1]=>
+    string(8) "function"
+    [2]=>
+    int(19)
+  }
+  [99]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(19)
+  }
+  [100]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(10) "myFunction"
+    [2]=>
+    int(19)
+  }
+  [101]=>
+  string(1) "("
+  [102]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(19)
+  }
+  [103]=>
+  string(1) ")"
+  [104]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(19)
+  }
+  [105]=>
+  string(1) "{"
+  [106]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(5) "
+    "
+    [2]=>
+    int(20)
+  }
+  [107]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(21)
+  }
+  [108]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(21)
+  }
+  [109]=>
+  string(1) "="
+  [110]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(21)
+  }
+  [111]=>
+  array(3) {
+    [0]=>
+    int(299)
+    [1]=>
+    string(3) "new"
+    [2]=>
+    int(21)
+  }
+  [112]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(21)
+  }
+  [113]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(10) "ChildClass"
+    [2]=>
+    int(21)
+  }
+  [114]=>
+  string(1) "("
+  [115]=>
+  string(1) ")"
+  [116]=>
+  string(1) ";"
+  [117]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(5) "
+    "
+    [2]=>
+    int(21)
+  }
+  [118]=>
+  array(3) {
+    [0]=>
+    int(301)
+    [1]=>
+    string(2) "if"
+    [2]=>
+    int(22)
+  }
+  [119]=>
+  string(1) "("
+  [120]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(22)
+  }
+  [121]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(22)
+  }
+  [122]=>
+  array(3) {
+    [0]=>
+    int(288)
+    [1]=>
+    string(10) "instanceof"
+    [2]=>
+    int(22)
+  }
+  [123]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(22)
+  }
+  [124]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(7) "MyClass"
+    [2]=>
+    int(22)
+  }
+  [125]=>
+  string(1) ")"
+  [126]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(7) "
+      "
+    [2]=>
+    int(22)
+  }
+  [127]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(23)
+  }
+  [128]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(23)
+  }
+  [129]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(16) ""object created""
+    [2]=>
+    int(23)
+  }
+  [130]=>
+  string(1) ";"
+  [131]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(23)
+  }
+  [132]=>
+  string(1) "}"
+  [133]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(24)
+  }
+  [134]=>
+  string(1) "}"
+  [135]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(25)
+  }
+  [136]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(10) "ChildClass"
+    [2]=>
+    int(26)
+  }
+  [137]=>
+  array(3) {
+    [0]=>
+    int(375)
+    [1]=>
+    string(2) "::"
+    [2]=>
+    int(26)
+  }
+  [138]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(10) "myFunction"
+    [2]=>
+    int(26)
+  }
+  [139]=>
+  string(1) "("
+  [140]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(2) "10"
+    [2]=>
+    int(26)
+  }
+  [141]=>
+  string(1) ")"
+  [142]=>
+  string(1) ";"
+  [143]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(26)
+  }
+  [144]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(27)
+  }
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_variation14.phpt b/ext/tokenizer/tests/token_get_all_variation14.phpt
new file mode 100644
index 0000000000..d4e4ec6437
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_variation14.phpt
@@ -0,0 +1,289 @@
+--TEST--
+Test token_get_all() function : usage variations - invalid token values
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+/*
+ * Testing token_get_all() with 'source' string containing invalid/unknown token value
+ *  unknown tokens - T_UNKNOWN(307)
+*/
+
+echo "*** Testing token_get_all() : with invalid/unknown tokens ***\n";
+
+// with valid php tags and invalid tokens
+echo "-- with valid PHP tags & invlid tokens --\n";
+$source = '<?php 
+struct myStruct {
+  variable $a;
+  method() { display $a; }
+}
+?>';
+var_dump( token_get_all($source));
+
+// with invalid open tag for testing entire source to be unkown token
+echo "-- with invlalid PHP open tag & valid tokens --\n";
+$source = '<pli 
+echo "hello world"; ?>';
+var_dump( token_get_all($source));
+
+// with invalid PHP tags and invalid tokens
+echo "-- with invalid PHP tags and tokens --\n";
+$source = '<PDP display  $a; <';
+var_dump( token_get_all($source));
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : with invalid/unknown tokens ***
+-- with valid PHP tags & invlid tokens --
+array(29) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(6) "struct"
+    [2]=>
+    int(2)
+  }
+  [3]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [4]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(8) "myStruct"
+    [2]=>
+    int(2)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [6]=>
+  string(1) "{"
+  [7]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(2)
+  }
+  [8]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(8) "variable"
+    [2]=>
+    int(3)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [10]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(3)
+  }
+  [11]=>
+  string(1) ";"
+  [12]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(3)
+  }
+  [13]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(6) "method"
+    [2]=>
+    int(4)
+  }
+  [14]=>
+  string(1) "("
+  [15]=>
+  string(1) ")"
+  [16]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [17]=>
+  string(1) "{"
+  [18]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [19]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(7) "display"
+    [2]=>
+    int(4)
+  }
+  [20]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [21]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(4)
+  }
+  [22]=>
+  string(1) ";"
+  [23]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [24]=>
+  string(1) "}"
+  [25]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(4)
+  }
+  [26]=>
+  string(1) "}"
+  [27]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(5)
+  }
+  [28]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(6)
+  }
+}
+-- with invlalid PHP open tag & valid tokens --
+array(1) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(28) "<pli 
+echo "hello world"; ?>"
+    [2]=>
+    int(1)
+  }
+}
+-- with invalid PHP tags and tokens --
+array(2) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(18) "<PDP display  $a; "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(1) "<"
+    [2]=>
+    int(1)
+  }
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_variation15.phpt b/ext/tokenizer/tests/token_get_all_variation15.phpt
new file mode 100644
index 0000000000..36a8630174
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_variation15.phpt
@@ -0,0 +1,776 @@
+--TEST--
+Test token_get_all() function : usage variations - heredoc string for 'source' 
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+/*
+ * Testing token_get_all() with heredoc 'source' string with all different types of token and heredoc string within
+ *     <<<EOT - T_START_HEREDOC(371)
+ *     EOT - T_END_HEREDOC(372)
+*/
+
+echo "*** Testing token_get_all() : with heredoc source string ***\n";
+
+$source = <<<EOT
+<?= 
+  \$a = 2;
+  \$b = 1;
+  \$c = <<<EOS
+  This is to test 
+  heredoc string
+EOS;
+  echo \$a + \$b;
+  function myFunction(\$a)
+  {
+    var_dump(\$a);
+  }
+  if(\$b < 10) {
+    \$b++;
+  }
+  else
+    \$b--;
+  while(\$a > 0) {
+    echo "*";
+    \$a--;
+  }
+  myFunction(10);
+?>
+EOT;
+var_dump( token_get_all($source));
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : with heredoc source string ***
+array(103) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(368)
+    [1]=>
+    string(3) "<?="
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(4) " 
+  "
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(2)
+  }
+  [3]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [4]=>
+  string(1) "="
+  [5]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+    [2]=>
+    int(2)
+  }
+  [7]=>
+  string(1) ";"
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(2)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(3)
+  }
+  [10]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [11]=>
+  string(1) "="
+  [12]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [13]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(3)
+  }
+  [14]=>
+  string(1) ";"
+  [15]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(3)
+  }
+  [16]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$c"
+    [2]=>
+    int(4)
+  }
+  [17]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [18]=>
+  string(1) "="
+  [19]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [20]=>
+  array(3) {
+    [0]=>
+    int(371)
+    [1]=>
+    string(7) "<<<EOS
+"
+    [2]=>
+    int(4)
+  }
+  [21]=>
+  array(3) {
+    [0]=>
+    int(314)
+    [1]=>
+    string(36) "  This is to test 
+  heredoc string
+"
+    [2]=>
+    int(5)
+  }
+  [22]=>
+  array(3) {
+    [0]=>
+    int(372)
+    [1]=>
+    string(3) "EOS"
+    [2]=>
+    int(7)
+  }
+  [23]=>
+  string(1) ";"
+  [24]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(7)
+  }
+  [25]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(8)
+  }
+  [26]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [27]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(8)
+  }
+  [28]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [29]=>
+  string(1) "+"
+  [30]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [31]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(8)
+  }
+  [32]=>
+  string(1) ";"
+  [33]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(8)
+  }
+  [34]=>
+  array(3) {
+    [0]=>
+    int(333)
+    [1]=>
+    string(8) "function"
+    [2]=>
+    int(9)
+  }
+  [35]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(9)
+  }
+  [36]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(10) "myFunction"
+    [2]=>
+    int(9)
+  }
+  [37]=>
+  string(1) "("
+  [38]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(9)
+  }
+  [39]=>
+  string(1) ")"
+  [40]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(9)
+  }
+  [41]=>
+  string(1) "{"
+  [42]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(5) "
+    "
+    [2]=>
+    int(10)
+  }
+  [43]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(8) "var_dump"
+    [2]=>
+    int(11)
+  }
+  [44]=>
+  string(1) "("
+  [45]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(11)
+  }
+  [46]=>
+  string(1) ")"
+  [47]=>
+  string(1) ";"
+  [48]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(11)
+  }
+  [49]=>
+  string(1) "}"
+  [50]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(12)
+  }
+  [51]=>
+  array(3) {
+    [0]=>
+    int(301)
+    [1]=>
+    string(2) "if"
+    [2]=>
+    int(13)
+  }
+  [52]=>
+  string(1) "("
+  [53]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(13)
+  }
+  [54]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(13)
+  }
+  [55]=>
+  string(1) "<"
+  [56]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(13)
+  }
+  [57]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(2) "10"
+    [2]=>
+    int(13)
+  }
+  [58]=>
+  string(1) ")"
+  [59]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(13)
+  }
+  [60]=>
+  string(1) "{"
+  [61]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(5) "
+    "
+    [2]=>
+    int(13)
+  }
+  [62]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(14)
+  }
+  [63]=>
+  array(3) {
+    [0]=>
+    int(297)
+    [1]=>
+    string(2) "++"
+    [2]=>
+    int(14)
+  }
+  [64]=>
+  string(1) ";"
+  [65]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(14)
+  }
+  [66]=>
+  string(1) "}"
+  [67]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(15)
+  }
+  [68]=>
+  array(3) {
+    [0]=>
+    int(303)
+    [1]=>
+    string(4) "else"
+    [2]=>
+    int(16)
+  }
+  [69]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(5) "
+    "
+    [2]=>
+    int(16)
+  }
+  [70]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(17)
+  }
+  [71]=>
+  array(3) {
+    [0]=>
+    int(296)
+    [1]=>
+    string(2) "--"
+    [2]=>
+    int(17)
+  }
+  [72]=>
+  string(1) ";"
+  [73]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(17)
+  }
+  [74]=>
+  array(3) {
+    [0]=>
+    int(318)
+    [1]=>
+    string(5) "while"
+    [2]=>
+    int(18)
+  }
+  [75]=>
+  string(1) "("
+  [76]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(18)
+  }
+  [77]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(18)
+  }
+  [78]=>
+  string(1) ">"
+  [79]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(18)
+  }
+  [80]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "0"
+    [2]=>
+    int(18)
+  }
+  [81]=>
+  string(1) ")"
+  [82]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(18)
+  }
+  [83]=>
+  string(1) "{"
+  [84]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(5) "
+    "
+    [2]=>
+    int(18)
+  }
+  [85]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(19)
+  }
+  [86]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(19)
+  }
+  [87]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(3) ""*""
+    [2]=>
+    int(19)
+  }
+  [88]=>
+  string(1) ";"
+  [89]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(5) "
+    "
+    [2]=>
+    int(19)
+  }
+  [90]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(20)
+  }
+  [91]=>
+  array(3) {
+    [0]=>
+    int(296)
+    [1]=>
+    string(2) "--"
+    [2]=>
+    int(20)
+  }
+  [92]=>
+  string(1) ";"
+  [93]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(20)
+  }
+  [94]=>
+  string(1) "}"
+  [95]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(21)
+  }
+  [96]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(10) "myFunction"
+    [2]=>
+    int(22)
+  }
+  [97]=>
+  string(1) "("
+  [98]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(2) "10"
+    [2]=>
+    int(22)
+  }
+  [99]=>
+  string(1) ")"
+  [100]=>
+  string(1) ";"
+  [101]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(22)
+  }
+  [102]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(23)
+  }
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_variation16.phpt b/ext/tokenizer/tests/token_get_all_variation16.phpt
new file mode 100644
index 0000000000..87646e955d
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_variation16.phpt
@@ -0,0 +1,1001 @@
+--TEST--
+Test token_get_all() function : usage variations - with function constructs
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+/*
+ * Testing token_get_all() with different function keywords
+ *   function - T_FUNCTION(333), return - T_RETURN(335)
+ *   different functions: 
+ *     include() - T_INCLUDE(262), print() - T_PRINT(266), 
+ *     isset() - T_ISSET(349), list() - T_LIST(358), 
+ *     require() - T_REQUIRE(259), empty() - T_EMPTY(350), 
+ *     declare() - T_DECLARE(324), array() - T_ARRAY(359), 
+ *      __halt_compiler() - T_HALT_COMPILER(351)
+*/
+
+echo "*** Testing token_get_all() : with different function constructs ***\n";
+
+$source = '<?php
+declare(VALUE=100);
+include("addfile.php");
+require("sumfile.php");
+
+function myFunction($a)
+{
+  if($a % 2)
+    return 1;
+  else
+    exit;
+}
+
+$a = VALUE;
+$b = 20;
+$c = array(1,2);
+$b >>= 2;
+
+if($b <= 0)
+  die;
+else
+  print($b);
+
+list($value1,$value2) = $c;
+if(empty($value1) && !isset($value1)) {
+  __halt_compiler();
+}
+?>';
+$tokens =  token_get_all($source);
+var_dump($tokens);
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : with different function constructs ***
+array(142) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php
+"
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(324)
+    [1]=>
+    string(7) "declare"
+    [2]=>
+    int(2)
+  }
+  [2]=>
+  string(1) "("
+  [3]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(5) "VALUE"
+    [2]=>
+    int(2)
+  }
+  [4]=>
+  string(1) "="
+  [5]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(3) "100"
+    [2]=>
+    int(2)
+  }
+  [6]=>
+  string(1) ")"
+  [7]=>
+  string(1) ";"
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(2)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(262)
+    [1]=>
+    string(7) "include"
+    [2]=>
+    int(3)
+  }
+  [10]=>
+  string(1) "("
+  [11]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(13) ""addfile.php""
+    [2]=>
+    int(3)
+  }
+  [12]=>
+  string(1) ")"
+  [13]=>
+  string(1) ";"
+  [14]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(3)
+  }
+  [15]=>
+  array(3) {
+    [0]=>
+    int(259)
+    [1]=>
+    string(7) "require"
+    [2]=>
+    int(4)
+  }
+  [16]=>
+  string(1) "("
+  [17]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(13) ""sumfile.php""
+    [2]=>
+    int(4)
+  }
+  [18]=>
+  string(1) ")"
+  [19]=>
+  string(1) ";"
+  [20]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(2) "
+
+"
+    [2]=>
+    int(4)
+  }
+  [21]=>
+  array(3) {
+    [0]=>
+    int(333)
+    [1]=>
+    string(8) "function"
+    [2]=>
+    int(6)
+  }
+  [22]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(6)
+  }
+  [23]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(10) "myFunction"
+    [2]=>
+    int(6)
+  }
+  [24]=>
+  string(1) "("
+  [25]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(6)
+  }
+  [26]=>
+  string(1) ")"
+  [27]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(6)
+  }
+  [28]=>
+  string(1) "{"
+  [29]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(7)
+  }
+  [30]=>
+  array(3) {
+    [0]=>
+    int(301)
+    [1]=>
+    string(2) "if"
+    [2]=>
+    int(8)
+  }
+  [31]=>
+  string(1) "("
+  [32]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(8)
+  }
+  [33]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [34]=>
+  string(1) "%"
+  [35]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [36]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+    [2]=>
+    int(8)
+  }
+  [37]=>
+  string(1) ")"
+  [38]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(5) "
+    "
+    [2]=>
+    int(8)
+  }
+  [39]=>
+  array(3) {
+    [0]=>
+    int(335)
+    [1]=>
+    string(6) "return"
+    [2]=>
+    int(9)
+  }
+  [40]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(9)
+  }
+  [41]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(9)
+  }
+  [42]=>
+  string(1) ";"
+  [43]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(9)
+  }
+  [44]=>
+  array(3) {
+    [0]=>
+    int(303)
+    [1]=>
+    string(4) "else"
+    [2]=>
+    int(10)
+  }
+  [45]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(5) "
+    "
+    [2]=>
+    int(10)
+  }
+  [46]=>
+  array(3) {
+    [0]=>
+    int(300)
+    [1]=>
+    string(4) "exit"
+    [2]=>
+    int(11)
+  }
+  [47]=>
+  string(1) ";"
+  [48]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(11)
+  }
+  [49]=>
+  string(1) "}"
+  [50]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(2) "
+
+"
+    [2]=>
+    int(12)
+  }
+  [51]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(14)
+  }
+  [52]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(14)
+  }
+  [53]=>
+  string(1) "="
+  [54]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(14)
+  }
+  [55]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(5) "VALUE"
+    [2]=>
+    int(14)
+  }
+  [56]=>
+  string(1) ";"
+  [57]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(14)
+  }
+  [58]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(15)
+  }
+  [59]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(15)
+  }
+  [60]=>
+  string(1) "="
+  [61]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(15)
+  }
+  [62]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(2) "20"
+    [2]=>
+    int(15)
+  }
+  [63]=>
+  string(1) ";"
+  [64]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(15)
+  }
+  [65]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$c"
+    [2]=>
+    int(16)
+  }
+  [66]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(16)
+  }
+  [67]=>
+  string(1) "="
+  [68]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(16)
+  }
+  [69]=>
+  array(3) {
+    [0]=>
+    int(359)
+    [1]=>
+    string(5) "array"
+    [2]=>
+    int(16)
+  }
+  [70]=>
+  string(1) "("
+  [71]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(16)
+  }
+  [72]=>
+  string(1) ","
+  [73]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+    [2]=>
+    int(16)
+  }
+  [74]=>
+  string(1) ")"
+  [75]=>
+  string(1) ";"
+  [76]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(16)
+  }
+  [77]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(17)
+  }
+  [78]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(17)
+  }
+  [79]=>
+  array(3) {
+    [0]=>
+    int(267)
+    [1]=>
+    string(3) ">>="
+    [2]=>
+    int(17)
+  }
+  [80]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(17)
+  }
+  [81]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+    [2]=>
+    int(17)
+  }
+  [82]=>
+  string(1) ";"
+  [83]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(2) "
+
+"
+    [2]=>
+    int(17)
+  }
+  [84]=>
+  array(3) {
+    [0]=>
+    int(301)
+    [1]=>
+    string(2) "if"
+    [2]=>
+    int(19)
+  }
+  [85]=>
+  string(1) "("
+  [86]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(19)
+  }
+  [87]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(19)
+  }
+  [88]=>
+  array(3) {
+    [0]=>
+    int(285)
+    [1]=>
+    string(2) "<="
+    [2]=>
+    int(19)
+  }
+  [89]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(19)
+  }
+  [90]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "0"
+    [2]=>
+    int(19)
+  }
+  [91]=>
+  string(1) ")"
+  [92]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(19)
+  }
+  [93]=>
+  array(3) {
+    [0]=>
+    int(300)
+    [1]=>
+    string(3) "die"
+    [2]=>
+    int(20)
+  }
+  [94]=>
+  string(1) ";"
+  [95]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(20)
+  }
+  [96]=>
+  array(3) {
+    [0]=>
+    int(303)
+    [1]=>
+    string(4) "else"
+    [2]=>
+    int(21)
+  }
+  [97]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(21)
+  }
+  [98]=>
+  array(3) {
+    [0]=>
+    int(266)
+    [1]=>
+    string(5) "print"
+    [2]=>
+    int(22)
+  }
+  [99]=>
+  string(1) "("
+  [100]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(22)
+  }
+  [101]=>
+  string(1) ")"
+  [102]=>
+  string(1) ";"
+  [103]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(2) "
+
+"
+    [2]=>
+    int(22)
+  }
+  [104]=>
+  array(3) {
+    [0]=>
+    int(358)
+    [1]=>
+    string(4) "list"
+    [2]=>
+    int(24)
+  }
+  [105]=>
+  string(1) "("
+  [106]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(7) "$value1"
+    [2]=>
+    int(24)
+  }
+  [107]=>
+  string(1) ","
+  [108]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(7) "$value2"
+    [2]=>
+    int(24)
+  }
+  [109]=>
+  string(1) ")"
+  [110]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(24)
+  }
+  [111]=>
+  string(1) "="
+  [112]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(24)
+  }
+  [113]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$c"
+    [2]=>
+    int(24)
+  }
+  [114]=>
+  string(1) ";"
+  [115]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(24)
+  }
+  [116]=>
+  array(3) {
+    [0]=>
+    int(301)
+    [1]=>
+    string(2) "if"
+    [2]=>
+    int(25)
+  }
+  [117]=>
+  string(1) "("
+  [118]=>
+  array(3) {
+    [0]=>
+    int(350)
+    [1]=>
+    string(5) "empty"
+    [2]=>
+    int(25)
+  }
+  [119]=>
+  string(1) "("
+  [120]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(7) "$value1"
+    [2]=>
+    int(25)
+  }
+  [121]=>
+  string(1) ")"
+  [122]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(25)
+  }
+  [123]=>
+  array(3) {
+    [0]=>
+    int(279)
+    [1]=>
+    string(2) "&&"
+    [2]=>
+    int(25)
+  }
+  [124]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(25)
+  }
+  [125]=>
+  string(1) "!"
+  [126]=>
+  array(3) {
+    [0]=>
+    int(349)
+    [1]=>
+    string(5) "isset"
+    [2]=>
+    int(25)
+  }
+  [127]=>
+  string(1) "("
+  [128]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(7) "$value1"
+    [2]=>
+    int(25)
+  }
+  [129]=>
+  string(1) ")"
+  [130]=>
+  string(1) ")"
+  [131]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(25)
+  }
+  [132]=>
+  string(1) "{"
+  [133]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(25)
+  }
+  [134]=>
+  array(3) {
+    [0]=>
+    int(351)
+    [1]=>
+    string(15) "__halt_compiler"
+    [2]=>
+    int(26)
+  }
+  [135]=>
+  string(1) "("
+  [136]=>
+  string(1) ")"
+  [137]=>
+  string(1) ";"
+  [138]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(26)
+  }
+  [139]=>
+  string(1) "}"
+  [140]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(27)
+  }
+  [141]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(28)
+  }
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_variation17.phpt b/ext/tokenizer/tests/token_get_all_variation17.phpt
new file mode 100644
index 0000000000..e10fb5851b
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_variation17.phpt
@@ -0,0 +1,611 @@
+--TEST--
+Test token_get_all() function : usage variations - with exception keywords
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+/*
+ * Testing token_get_all() with different exception keywords
+ *   try - T_TRY(336), 
+ *   catch - T_CATCH(337),
+ *   throw - T_THROW(338)
+*/
+
+echo "*** Testing token_get_all() : with exception keywords ***\n";
+
+$source = '<?php
+function inverse($x)
+{
+  if($x == 0) {
+    throw new Exception("Divison by zero");
+  else
+    return 1/$x;
+}
+try {
+  echo inverse(0);
+  echo inverse(5);
+} catch(Exception $e) {
+    echo "caught exception:";
+}
+?>';
+$tokens =  token_get_all($source);
+var_dump($tokens);
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : with exception keywords ***
+array(81) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php
+"
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(333)
+    [1]=>
+    string(8) "function"
+    [2]=>
+    int(2)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [3]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(7) "inverse"
+    [2]=>
+    int(2)
+  }
+  [4]=>
+  string(1) "("
+  [5]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$x"
+    [2]=>
+    int(2)
+  }
+  [6]=>
+  string(1) ")"
+  [7]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(2)
+  }
+  [8]=>
+  string(1) "{"
+  [9]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(3)
+  }
+  [10]=>
+  array(3) {
+    [0]=>
+    int(301)
+    [1]=>
+    string(2) "if"
+    [2]=>
+    int(4)
+  }
+  [11]=>
+  string(1) "("
+  [12]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$x"
+    [2]=>
+    int(4)
+  }
+  [13]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [14]=>
+  array(3) {
+    [0]=>
+    int(283)
+    [1]=>
+    string(2) "=="
+    [2]=>
+    int(4)
+  }
+  [15]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [16]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "0"
+    [2]=>
+    int(4)
+  }
+  [17]=>
+  string(1) ")"
+  [18]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [19]=>
+  string(1) "{"
+  [20]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(5) "
+    "
+    [2]=>
+    int(4)
+  }
+  [21]=>
+  array(3) {
+    [0]=>
+    int(338)
+    [1]=>
+    string(5) "throw"
+    [2]=>
+    int(5)
+  }
+  [22]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(5)
+  }
+  [23]=>
+  array(3) {
+    [0]=>
+    int(299)
+    [1]=>
+    string(3) "new"
+    [2]=>
+    int(5)
+  }
+  [24]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(5)
+  }
+  [25]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(9) "Exception"
+    [2]=>
+    int(5)
+  }
+  [26]=>
+  string(1) "("
+  [27]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(17) ""Divison by zero""
+    [2]=>
+    int(5)
+  }
+  [28]=>
+  string(1) ")"
+  [29]=>
+  string(1) ";"
+  [30]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(5)
+  }
+  [31]=>
+  array(3) {
+    [0]=>
+    int(303)
+    [1]=>
+    string(4) "else"
+    [2]=>
+    int(6)
+  }
+  [32]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(5) "
+    "
+    [2]=>
+    int(6)
+  }
+  [33]=>
+  array(3) {
+    [0]=>
+    int(335)
+    [1]=>
+    string(6) "return"
+    [2]=>
+    int(7)
+  }
+  [34]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(7)
+  }
+  [35]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(7)
+  }
+  [36]=>
+  string(1) "/"
+  [37]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$x"
+    [2]=>
+    int(7)
+  }
+  [38]=>
+  string(1) ";"
+  [39]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(7)
+  }
+  [40]=>
+  string(1) "}"
+  [41]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(8)
+  }
+  [42]=>
+  array(3) {
+    [0]=>
+    int(336)
+    [1]=>
+    string(3) "try"
+    [2]=>
+    int(9)
+  }
+  [43]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(9)
+  }
+  [44]=>
+  string(1) "{"
+  [45]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(9)
+  }
+  [46]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(10)
+  }
+  [47]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(10)
+  }
+  [48]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(7) "inverse"
+    [2]=>
+    int(10)
+  }
+  [49]=>
+  string(1) "("
+  [50]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "0"
+    [2]=>
+    int(10)
+  }
+  [51]=>
+  string(1) ")"
+  [52]=>
+  string(1) ";"
+  [53]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(10)
+  }
+  [54]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(11)
+  }
+  [55]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(11)
+  }
+  [56]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(7) "inverse"
+    [2]=>
+    int(11)
+  }
+  [57]=>
+  string(1) "("
+  [58]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "5"
+    [2]=>
+    int(11)
+  }
+  [59]=>
+  string(1) ")"
+  [60]=>
+  string(1) ";"
+  [61]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(11)
+  }
+  [62]=>
+  string(1) "}"
+  [63]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(12)
+  }
+  [64]=>
+  array(3) {
+    [0]=>
+    int(337)
+    [1]=>
+    string(5) "catch"
+    [2]=>
+    int(12)
+  }
+  [65]=>
+  string(1) "("
+  [66]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(9) "Exception"
+    [2]=>
+    int(12)
+  }
+  [67]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(12)
+  }
+  [68]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$e"
+    [2]=>
+    int(12)
+  }
+  [69]=>
+  string(1) ")"
+  [70]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(12)
+  }
+  [71]=>
+  string(1) "{"
+  [72]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(5) "
+    "
+    [2]=>
+    int(12)
+  }
+  [73]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(13)
+  }
+  [74]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(13)
+  }
+  [75]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(19) ""caught exception:""
+    [2]=>
+    int(13)
+  }
+  [76]=>
+  string(1) ";"
+  [77]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(13)
+  }
+  [78]=>
+  string(1) "}"
+  [79]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(14)
+  }
+  [80]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(15)
+  }
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_variation18.phpt b/ext/tokenizer/tests/token_get_all_variation18.phpt
new file mode 100644
index 0000000000..309b7e626d
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_variation18.phpt
@@ -0,0 +1,118 @@
+--TEST--
+Test token_get_all() function : usage variations - with HTML code 
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+/*
+ * Testing token_get_all() with source string containing HTML code with PHP
+ *   HTML tags are considered to be T_INLINE_HTML(311)
+*/
+
+echo "*** Testing token_get_all() : 'source' string with HTML tags ***\n";
+
+$source = '
+<html>
+<body>
+  Testing HTML
+</body>
+</html>"
+
+<?php 
+  echo "php code with HTML";
+?>';
+var_dump( token_get_all($source));
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : 'source' string with HTML tags ***
+array(9) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(311)
+    [1]=>
+    string(48) "
+<html>
+<body>
+  Testing HTML
+</body>
+</html>"
+
+"
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(8)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(8)
+  }
+  [3]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(9)
+  }
+  [4]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(9)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(20) ""php code with HTML""
+    [2]=>
+    int(9)
+  }
+  [6]=>
+  string(1) ";"
+  [7]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(9)
+  }
+  [8]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(10)
+  }
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_variation2.phpt b/ext/tokenizer/tests/token_get_all_variation2.phpt
new file mode 100644
index 0000000000..d4f62dcab1
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_variation2.phpt
@@ -0,0 +1,429 @@
+--TEST--
+Test token_get_all() function : usage variations - with different arithmetic operators
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+/*
+ * Passing 'source' argument with different arithmetic operators to test them for token
+ * Arithmetic operators: +, -, *, /, % are not listed as specific operator tokens,
+ *    so they are expected to return string - T_STRING
+*/
+
+echo "*** Testing token_get_all() : 'source' string with different arithmetic operators ***\n";
+
+// arithmetic operators - '+', '-', '*', '/', '%' 
+$source = array (
+  '<?php $a = 1 + 2; ?>',
+  '<?php $b = $b - 2; ?>',
+  '<?php $c = $a * $b; ?>',
+  '<?php $a = $b % 2; ?>'
+);
+for($count = 0; $count < count($source); $count++) {
+  echo "-- Iteration ".($count + 1)." --\n";
+  var_dump( token_get_all($source[$count]));
+}
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : 'source' string with different arithmetic operators ***
+-- Iteration 1 --
+array(13) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [3]=>
+  string(1) "="
+  [4]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(1)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [7]=>
+  string(1) "+"
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+    [2]=>
+    int(1)
+  }
+  [10]=>
+  string(1) ";"
+  [11]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [12]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(1)
+  }
+}
+-- Iteration 2 --
+array(13) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [3]=>
+  string(1) "="
+  [4]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(1)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [7]=>
+  string(1) "-"
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+    [2]=>
+    int(1)
+  }
+  [10]=>
+  string(1) ";"
+  [11]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [12]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(1)
+  }
+}
+-- Iteration 3 --
+array(13) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$c"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [3]=>
+  string(1) "="
+  [4]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(1)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [7]=>
+  string(1) "*"
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(1)
+  }
+  [10]=>
+  string(1) ";"
+  [11]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [12]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(1)
+  }
+}
+-- Iteration 4 --
+array(13) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [3]=>
+  string(1) "="
+  [4]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(1)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [7]=>
+  string(1) "%"
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+    [2]=>
+    int(1)
+  }
+  [10]=>
+  string(1) ";"
+  [11]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [12]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(1)
+  }
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_variation3.phpt b/ext/tokenizer/tests/token_get_all_variation3.phpt
new file mode 100644
index 0000000000..267e8c5b4b
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_variation3.phpt
@@ -0,0 +1,568 @@
+--TEST--
+Test token_get_all() function : usage variations - with logical operators
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+/*
+ * Passing 'source' argument with different logical operators to test them for tokens
+ *   and - T_AND_LOGICAL_AND(265), 
+ *   or - T_LOGICAL_OR(263), 
+ *   xor - T_LOGICAL_XOR(264), 
+ *   && - T_BOOLEAN_AND(279), 
+ *   || - T_BOOLEAN_OR(278)
+*/
+
+echo "*** Testing token_get_all() : 'source' string with different logical operators ***\n";
+
+// logical operators : 'and', 'or', 'xor', '&&', '||' 
+$source = array (
+  '<?php $a = 1 and 024; ?>',
+  '<?php $b = $b or 0X1E; ?>',
+  '<?php $c = $a xor $b; ?>',
+  '<?php $a = $b && 2; ?>',
+  '<?php $b = $b || 1; ?>'
+);
+for($count = 0; $count < count($source); $count++) {
+  echo "-- Iteration ".($count + 1)." --\n";
+  var_dump( token_get_all($source[$count]));
+}
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : 'source' string with different logical operators ***
+-- Iteration 1 --
+array(13) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [3]=>
+  string(1) "="
+  [4]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(1)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [7]=>
+  array(3) {
+    [0]=>
+    int(265)
+    [1]=>
+    string(3) "and"
+    [2]=>
+    int(1)
+  }
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(3) "024"
+    [2]=>
+    int(1)
+  }
+  [10]=>
+  string(1) ";"
+  [11]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [12]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(1)
+  }
+}
+-- Iteration 2 --
+array(13) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [3]=>
+  string(1) "="
+  [4]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(1)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [7]=>
+  array(3) {
+    [0]=>
+    int(263)
+    [1]=>
+    string(2) "or"
+    [2]=>
+    int(1)
+  }
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(4) "0X1E"
+    [2]=>
+    int(1)
+  }
+  [10]=>
+  string(1) ";"
+  [11]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [12]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(1)
+  }
+}
+-- Iteration 3 --
+array(13) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$c"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [3]=>
+  string(1) "="
+  [4]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(1)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [7]=>
+  array(3) {
+    [0]=>
+    int(264)
+    [1]=>
+    string(3) "xor"
+    [2]=>
+    int(1)
+  }
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(1)
+  }
+  [10]=>
+  string(1) ";"
+  [11]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [12]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(1)
+  }
+}
+-- Iteration 4 --
+array(13) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [3]=>
+  string(1) "="
+  [4]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(1)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [7]=>
+  array(3) {
+    [0]=>
+    int(279)
+    [1]=>
+    string(2) "&&"
+    [2]=>
+    int(1)
+  }
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+    [2]=>
+    int(1)
+  }
+  [10]=>
+  string(1) ";"
+  [11]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [12]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(1)
+  }
+}
+-- Iteration 5 --
+array(13) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [3]=>
+  string(1) "="
+  [4]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(1)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [7]=>
+  array(3) {
+    [0]=>
+    int(278)
+    [1]=>
+    string(2) "||"
+    [2]=>
+    int(1)
+  }
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(1)
+  }
+  [10]=>
+  string(1) ";"
+  [11]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(1)
+  }
+  [12]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(1)
+  }
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_variation4.phpt b/ext/tokenizer/tests/token_get_all_variation4.phpt
new file mode 100644
index 0000000000..e63a917ea3
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_variation4.phpt
@@ -0,0 +1,747 @@
+--TEST--
+Test token_get_all() function : usage variations - with comparison operators
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+/*
+ * Passing 'source' argument with different comparison operators to test them for tokens
+ *  == - T_IS_EQUAL(283), === - T_IS_IDENTICAL(281), 
+ *  >= - T_IS_GREATER_OR_EQUAL(284), <= - T_IS_LESS_OR_EQUAL(285)
+ *  != - T_IS_NOT_EQUAL, <> - T_IS_NOT_EQUAL(282), !== - T_IS_NOT_IDENTICAL(280)
+*/
+
+echo "*** Testing token_get_all() : 'source' string with different comparison operators ***\n";
+
+// comparison operators : '==', '===', '>=', '<=', '!=', '!==', '<>'
+$source = '<?php 
+if($a == 0) 
+  echo "== 0";
+elseif($a === 2)
+  echo "=== 2";
+elseif($a >= 10 && $a <= 20)
+  echo ">= 10 & <=20";
+elseif($a != 1 || $a <> 1)
+  echo "!= 1";
+elseif($a !== 1)
+  echo "!==1";
+?>';
+var_dump( token_get_all($source));
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : 'source' string with different comparison operators ***
+array(89) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(301)
+    [1]=>
+    string(2) "if"
+    [2]=>
+    int(2)
+  }
+  [3]=>
+  string(1) "("
+  [4]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(2)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(283)
+    [1]=>
+    string(2) "=="
+    [2]=>
+    int(2)
+  }
+  [7]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [8]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "0"
+    [2]=>
+    int(2)
+  }
+  [9]=>
+  string(1) ")"
+  [10]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(4) " 
+  "
+    [2]=>
+    int(2)
+  }
+  [11]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(3)
+  }
+  [12]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [13]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(6) ""== 0""
+    [2]=>
+    int(3)
+  }
+  [14]=>
+  string(1) ";"
+  [15]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(3)
+  }
+  [16]=>
+  array(3) {
+    [0]=>
+    int(302)
+    [1]=>
+    string(6) "elseif"
+    [2]=>
+    int(4)
+  }
+  [17]=>
+  string(1) "("
+  [18]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(4)
+  }
+  [19]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [20]=>
+  array(3) {
+    [0]=>
+    int(281)
+    [1]=>
+    string(3) "==="
+    [2]=>
+    int(4)
+  }
+  [21]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [22]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+    [2]=>
+    int(4)
+  }
+  [23]=>
+  string(1) ")"
+  [24]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(4)
+  }
+  [25]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(5)
+  }
+  [26]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(5)
+  }
+  [27]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(7) ""=== 2""
+    [2]=>
+    int(5)
+  }
+  [28]=>
+  string(1) ";"
+  [29]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(5)
+  }
+  [30]=>
+  array(3) {
+    [0]=>
+    int(302)
+    [1]=>
+    string(6) "elseif"
+    [2]=>
+    int(6)
+  }
+  [31]=>
+  string(1) "("
+  [32]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(6)
+  }
+  [33]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(6)
+  }
+  [34]=>
+  array(3) {
+    [0]=>
+    int(284)
+    [1]=>
+    string(2) ">="
+    [2]=>
+    int(6)
+  }
+  [35]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(6)
+  }
+  [36]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(2) "10"
+    [2]=>
+    int(6)
+  }
+  [37]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(6)
+  }
+  [38]=>
+  array(3) {
+    [0]=>
+    int(279)
+    [1]=>
+    string(2) "&&"
+    [2]=>
+    int(6)
+  }
+  [39]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(6)
+  }
+  [40]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(6)
+  }
+  [41]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(6)
+  }
+  [42]=>
+  array(3) {
+    [0]=>
+    int(285)
+    [1]=>
+    string(2) "<="
+    [2]=>
+    int(6)
+  }
+  [43]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(6)
+  }
+  [44]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(2) "20"
+    [2]=>
+    int(6)
+  }
+  [45]=>
+  string(1) ")"
+  [46]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(6)
+  }
+  [47]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(7)
+  }
+  [48]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(7)
+  }
+  [49]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(14) "">= 10 & <=20""
+    [2]=>
+    int(7)
+  }
+  [50]=>
+  string(1) ";"
+  [51]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(7)
+  }
+  [52]=>
+  array(3) {
+    [0]=>
+    int(302)
+    [1]=>
+    string(6) "elseif"
+    [2]=>
+    int(8)
+  }
+  [53]=>
+  string(1) "("
+  [54]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(8)
+  }
+  [55]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [56]=>
+  array(3) {
+    [0]=>
+    int(282)
+    [1]=>
+    string(2) "!="
+    [2]=>
+    int(8)
+  }
+  [57]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [58]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(8)
+  }
+  [59]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [60]=>
+  array(3) {
+    [0]=>
+    int(278)
+    [1]=>
+    string(2) "||"
+    [2]=>
+    int(8)
+  }
+  [61]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [62]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(8)
+  }
+  [63]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [64]=>
+  array(3) {
+    [0]=>
+    int(282)
+    [1]=>
+    string(2) "<>"
+    [2]=>
+    int(8)
+  }
+  [65]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [66]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(8)
+  }
+  [67]=>
+  string(1) ")"
+  [68]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(8)
+  }
+  [69]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(9)
+  }
+  [70]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(9)
+  }
+  [71]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(6) ""!= 1""
+    [2]=>
+    int(9)
+  }
+  [72]=>
+  string(1) ";"
+  [73]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(9)
+  }
+  [74]=>
+  array(3) {
+    [0]=>
+    int(302)
+    [1]=>
+    string(6) "elseif"
+    [2]=>
+    int(10)
+  }
+  [75]=>
+  string(1) "("
+  [76]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(10)
+  }
+  [77]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(10)
+  }
+  [78]=>
+  array(3) {
+    [0]=>
+    int(280)
+    [1]=>
+    string(3) "!=="
+    [2]=>
+    int(10)
+  }
+  [79]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(10)
+  }
+  [80]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(10)
+  }
+  [81]=>
+  string(1) ")"
+  [82]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(3) "
+  "
+    [2]=>
+    int(10)
+  }
+  [83]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(11)
+  }
+  [84]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(11)
+  }
+  [85]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(6) ""!==1""
+    [2]=>
+    int(11)
+  }
+  [86]=>
+  string(1) ";"
+  [87]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(11)
+  }
+  [88]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(12)
+  }
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_variation5.phpt b/ext/tokenizer/tests/token_get_all_variation5.phpt
new file mode 100644
index 0000000000..5a7d16a36b
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_variation5.phpt
@@ -0,0 +1,798 @@
+--TEST--
+Test token_get_all() function : usage variations - with assignment operators
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+/*
+ * Passing 'source' argument with different assignment operators to test them for tokens
+ *  += - T_PLUS_EQUAL(277), -= - T_MINUS-EQUAL(276), 
+ *  *= - T_MUL_EQUAL(275), /= - T_DIVIDE_EQUAL(274),
+ *  %= - T_MOD_EQUAL(272), &= - T_AND_EQUAL(271),
+ *  |= - T_OR_EQUAL(271), ^= - T_EXOR_EQUAL(269),
+ *  >>= - T_SR_EQUAL(267), <<= - T_SL_EQUAL(268), .= - T_CONCAT_EQUAL(273)
+*/
+
+echo "*** Testing token_get_all() : 'source' string with different assignment operators ***\n";
+
+// assignment operators : '+=', '-=', '*=', '/=', '%=', '&=', '|=', '^=', '>>=', '<<=', '.='
+$source = '<?php 
+$a = 1, $b = 2;
+$c += $b;
+$b -= $a;
+$a *= 2;
+$d /= 10.50;
+$a %= 10.50;
+$b &= $c;
+$c |= 1;
+$d ^= 5;
+$a >>= 1;
+$b <<= 2;
+$d .= "hello world";
+?>';
+var_dump( token_get_all($source));
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : 'source' string with different assignment operators ***
+array(94) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(2)
+  }
+  [3]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [4]=>
+  string(1) "="
+  [5]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(2)
+  }
+  [7]=>
+  string(1) ","
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(2)
+  }
+  [10]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [11]=>
+  string(1) "="
+  [12]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [13]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+    [2]=>
+    int(2)
+  }
+  [14]=>
+  string(1) ";"
+  [15]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(2)
+  }
+  [16]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$c"
+    [2]=>
+    int(3)
+  }
+  [17]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [18]=>
+  array(3) {
+    [0]=>
+    int(277)
+    [1]=>
+    string(2) "+="
+    [2]=>
+    int(3)
+  }
+  [19]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [20]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(3)
+  }
+  [21]=>
+  string(1) ";"
+  [22]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(3)
+  }
+  [23]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(4)
+  }
+  [24]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [25]=>
+  array(3) {
+    [0]=>
+    int(276)
+    [1]=>
+    string(2) "-="
+    [2]=>
+    int(4)
+  }
+  [26]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [27]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(4)
+  }
+  [28]=>
+  string(1) ";"
+  [29]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(4)
+  }
+  [30]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(5)
+  }
+  [31]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(5)
+  }
+  [32]=>
+  array(3) {
+    [0]=>
+    int(275)
+    [1]=>
+    string(2) "*="
+    [2]=>
+    int(5)
+  }
+  [33]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(5)
+  }
+  [34]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+    [2]=>
+    int(5)
+  }
+  [35]=>
+  string(1) ";"
+  [36]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(5)
+  }
+  [37]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$d"
+    [2]=>
+    int(6)
+  }
+  [38]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(6)
+  }
+  [39]=>
+  array(3) {
+    [0]=>
+    int(274)
+    [1]=>
+    string(2) "/="
+    [2]=>
+    int(6)
+  }
+  [40]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(6)
+  }
+  [41]=>
+  array(3) {
+    [0]=>
+    int(306)
+    [1]=>
+    string(5) "10.50"
+    [2]=>
+    int(6)
+  }
+  [42]=>
+  string(1) ";"
+  [43]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(6)
+  }
+  [44]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(7)
+  }
+  [45]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(7)
+  }
+  [46]=>
+  array(3) {
+    [0]=>
+    int(272)
+    [1]=>
+    string(2) "%="
+    [2]=>
+    int(7)
+  }
+  [47]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(7)
+  }
+  [48]=>
+  array(3) {
+    [0]=>
+    int(306)
+    [1]=>
+    string(5) "10.50"
+    [2]=>
+    int(7)
+  }
+  [49]=>
+  string(1) ";"
+  [50]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(7)
+  }
+  [51]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(8)
+  }
+  [52]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [53]=>
+  array(3) {
+    [0]=>
+    int(271)
+    [1]=>
+    string(2) "&="
+    [2]=>
+    int(8)
+  }
+  [54]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [55]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$c"
+    [2]=>
+    int(8)
+  }
+  [56]=>
+  string(1) ";"
+  [57]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(8)
+  }
+  [58]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$c"
+    [2]=>
+    int(9)
+  }
+  [59]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(9)
+  }
+  [60]=>
+  array(3) {
+    [0]=>
+    int(270)
+    [1]=>
+    string(2) "|="
+    [2]=>
+    int(9)
+  }
+  [61]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(9)
+  }
+  [62]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(9)
+  }
+  [63]=>
+  string(1) ";"
+  [64]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(9)
+  }
+  [65]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$d"
+    [2]=>
+    int(10)
+  }
+  [66]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(10)
+  }
+  [67]=>
+  array(3) {
+    [0]=>
+    int(269)
+    [1]=>
+    string(2) "^="
+    [2]=>
+    int(10)
+  }
+  [68]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(10)
+  }
+  [69]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "5"
+    [2]=>
+    int(10)
+  }
+  [70]=>
+  string(1) ";"
+  [71]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(10)
+  }
+  [72]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(11)
+  }
+  [73]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(11)
+  }
+  [74]=>
+  array(3) {
+    [0]=>
+    int(267)
+    [1]=>
+    string(3) ">>="
+    [2]=>
+    int(11)
+  }
+  [75]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(11)
+  }
+  [76]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(11)
+  }
+  [77]=>
+  string(1) ";"
+  [78]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(11)
+  }
+  [79]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(12)
+  }
+  [80]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(12)
+  }
+  [81]=>
+  array(3) {
+    [0]=>
+    int(268)
+    [1]=>
+    string(3) "<<="
+    [2]=>
+    int(12)
+  }
+  [82]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(12)
+  }
+  [83]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+    [2]=>
+    int(12)
+  }
+  [84]=>
+  string(1) ";"
+  [85]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(12)
+  }
+  [86]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$d"
+    [2]=>
+    int(13)
+  }
+  [87]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(13)
+  }
+  [88]=>
+  array(3) {
+    [0]=>
+    int(273)
+    [1]=>
+    string(2) ".="
+    [2]=>
+    int(13)
+  }
+  [89]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(13)
+  }
+  [90]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(13) ""hello world""
+    [2]=>
+    int(13)
+  }
+  [91]=>
+  string(1) ";"
+  [92]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(13)
+  }
+  [93]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(14)
+  }
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_variation6.phpt b/ext/tokenizer/tests/token_get_all_variation6.phpt
new file mode 100644
index 0000000000..5d81104a24
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_variation6.phpt
@@ -0,0 +1,392 @@
+--TEST--
+Test token_get_all() function : usage variations - with bitwise operators
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+/*
+ * Passing 'source' argument with different bitwise operators to test them for token
+ *  << - T_SL(287)
+ *  >> - T_SR(286)
+*/
+
+echo "*** Testing token_get_all() : 'source' string with different bitwise operators ***\n";
+
+// bitwise operators : '<<' , '>>'
+$source = '<?php
+$a = 2, $b = 4;
+$a = $a << 2;
+$b = $b >> 2;
+var_dump($a);
+var_dump($b);
+?>';
+var_dump( token_get_all($source));
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : 'source' string with different bitwise operators ***
+array(50) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php
+"
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(2)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [3]=>
+  string(1) "="
+  [4]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+    [2]=>
+    int(2)
+  }
+  [6]=>
+  string(1) ","
+  [7]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [8]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(2)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [10]=>
+  string(1) "="
+  [11]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [12]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "4"
+    [2]=>
+    int(2)
+  }
+  [13]=>
+  string(1) ";"
+  [14]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(2)
+  }
+  [15]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(3)
+  }
+  [16]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [17]=>
+  string(1) "="
+  [18]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [19]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(3)
+  }
+  [20]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [21]=>
+  array(3) {
+    [0]=>
+    int(287)
+    [1]=>
+    string(2) "<<"
+    [2]=>
+    int(3)
+  }
+  [22]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [23]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+    [2]=>
+    int(3)
+  }
+  [24]=>
+  string(1) ";"
+  [25]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(3)
+  }
+  [26]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(4)
+  }
+  [27]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [28]=>
+  string(1) "="
+  [29]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [30]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(4)
+  }
+  [31]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [32]=>
+  array(3) {
+    [0]=>
+    int(286)
+    [1]=>
+    string(2) ">>"
+    [2]=>
+    int(4)
+  }
+  [33]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [34]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "2"
+    [2]=>
+    int(4)
+  }
+  [35]=>
+  string(1) ";"
+  [36]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(4)
+  }
+  [37]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(8) "var_dump"
+    [2]=>
+    int(5)
+  }
+  [38]=>
+  string(1) "("
+  [39]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(5)
+  }
+  [40]=>
+  string(1) ")"
+  [41]=>
+  string(1) ";"
+  [42]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(5)
+  }
+  [43]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(8) "var_dump"
+    [2]=>
+    int(6)
+  }
+  [44]=>
+  string(1) "("
+  [45]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(6)
+  }
+  [46]=>
+  string(1) ")"
+  [47]=>
+  string(1) ";"
+  [48]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(6)
+  }
+  [49]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(7)
+  }
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_variation7.phpt b/ext/tokenizer/tests/token_get_all_variation7.phpt
new file mode 100644
index 0000000000..5ba8563840
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_variation7.phpt
@@ -0,0 +1,259 @@
+--TEST--
+Test token_get_all() function : usage variations - with increment/decrement operators
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+/*
+ * Passing 'source' argument with different increment/decrement operators to test them for token
+ *  ++ - T_INC(297)
+ *  -- - T_DEC(296)
+*/
+
+echo "*** Testing token_get_all() : 'source' string with different increment/decrement operators ***\n";
+
+// increment/decrement operators : '++' , '--'
+$source = '<?php 
+$a = 10, $b = 5;
+$a++;
+$b--;
+echo $a;
+?>';
+var_dump( token_get_all($source));
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : 'source' string with different increment/decrement operators ***
+array(30) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(2)
+  }
+  [3]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [4]=>
+  string(1) "="
+  [5]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(2) "10"
+    [2]=>
+    int(2)
+  }
+  [7]=>
+  string(1) ","
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(2)
+  }
+  [10]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [11]=>
+  string(1) "="
+  [12]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [13]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "5"
+    [2]=>
+    int(2)
+  }
+  [14]=>
+  string(1) ";"
+  [15]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(2)
+  }
+  [16]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(3)
+  }
+  [17]=>
+  array(3) {
+    [0]=>
+    int(297)
+    [1]=>
+    string(2) "++"
+    [2]=>
+    int(3)
+  }
+  [18]=>
+  string(1) ";"
+  [19]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(3)
+  }
+  [20]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(4)
+  }
+  [21]=>
+  array(3) {
+    [0]=>
+    int(296)
+    [1]=>
+    string(2) "--"
+    [2]=>
+    int(4)
+  }
+  [22]=>
+  string(1) ";"
+  [23]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(4)
+  }
+  [24]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(5)
+  }
+  [25]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(5)
+  }
+  [26]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(5)
+  }
+  [27]=>
+  string(1) ";"
+  [28]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(5)
+  }
+  [29]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(6)
+  }
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_variation8.phpt b/ext/tokenizer/tests/token_get_all_variation8.phpt
new file mode 100644
index 0000000000..5c761f6cbd
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_variation8.phpt
@@ -0,0 +1,832 @@
+--TEST--
+Test token_get_all() function : usage variations - with type casting operators
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+/*
+ * Passing 'source' argument with different type casting operators to test them for token
+ *  (int)/(integer) - T_INT_CAST(295), (float)/(real)/(double) - T_DOUBLE_CAST(294),
+ *  (string) - T_STRING_CAST(293), (bool)/(boolean) - T_BOOL_CAST(290),
+ *  (unset) - T_UNSET_CAST(289)
+*/
+
+echo "*** Testing token_get_all() : 'source' string with different type casting operators ***\n";
+
+// type casting operators : (int), (integer), (float), (real), (double), (string), (array), (object), (bool), (boolean),(unset)
+$source = '<?php 
+$a = 1, $b = 10.5
+$c = (int)$b + $a;
+$d = (float)$a + $b;
+$e = (string)$a.(string)$b;
+if((bool)$a) echo "true";
+if(!(boolean)$b) echo "false";
+$c = $c + (integer) 123.4e2;
+$d = $c - (real) 12;
+$b = (unset)$a;
+?>';
+var_dump( token_get_all($source));
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : 'source' string with different type casting operators ***
+array(108) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(2)
+  }
+  [3]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [4]=>
+  string(1) "="
+  [5]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(1) "1"
+    [2]=>
+    int(2)
+  }
+  [7]=>
+  string(1) ","
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(2)
+  }
+  [10]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [11]=>
+  string(1) "="
+  [12]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(2)
+  }
+  [13]=>
+  array(3) {
+    [0]=>
+    int(306)
+    [1]=>
+    string(4) "10.5"
+    [2]=>
+    int(2)
+  }
+  [14]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(2)
+  }
+  [15]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$c"
+    [2]=>
+    int(3)
+  }
+  [16]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [17]=>
+  string(1) "="
+  [18]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [19]=>
+  array(3) {
+    [0]=>
+    int(295)
+    [1]=>
+    string(5) "(int)"
+    [2]=>
+    int(3)
+  }
+  [20]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(3)
+  }
+  [21]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [22]=>
+  string(1) "+"
+  [23]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(3)
+  }
+  [24]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(3)
+  }
+  [25]=>
+  string(1) ";"
+  [26]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(3)
+  }
+  [27]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$d"
+    [2]=>
+    int(4)
+  }
+  [28]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [29]=>
+  string(1) "="
+  [30]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [31]=>
+  array(3) {
+    [0]=>
+    int(294)
+    [1]=>
+    string(7) "(float)"
+    [2]=>
+    int(4)
+  }
+  [32]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(4)
+  }
+  [33]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [34]=>
+  string(1) "+"
+  [35]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(4)
+  }
+  [36]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(4)
+  }
+  [37]=>
+  string(1) ";"
+  [38]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(4)
+  }
+  [39]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$e"
+    [2]=>
+    int(5)
+  }
+  [40]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(5)
+  }
+  [41]=>
+  string(1) "="
+  [42]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(5)
+  }
+  [43]=>
+  array(3) {
+    [0]=>
+    int(293)
+    [1]=>
+    string(8) "(string)"
+    [2]=>
+    int(5)
+  }
+  [44]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(5)
+  }
+  [45]=>
+  string(1) "."
+  [46]=>
+  array(3) {
+    [0]=>
+    int(293)
+    [1]=>
+    string(8) "(string)"
+    [2]=>
+    int(5)
+  }
+  [47]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(5)
+  }
+  [48]=>
+  string(1) ";"
+  [49]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(5)
+  }
+  [50]=>
+  array(3) {
+    [0]=>
+    int(301)
+    [1]=>
+    string(2) "if"
+    [2]=>
+    int(6)
+  }
+  [51]=>
+  string(1) "("
+  [52]=>
+  array(3) {
+    [0]=>
+    int(290)
+    [1]=>
+    string(6) "(bool)"
+    [2]=>
+    int(6)
+  }
+  [53]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(6)
+  }
+  [54]=>
+  string(1) ")"
+  [55]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(6)
+  }
+  [56]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(6)
+  }
+  [57]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(6)
+  }
+  [58]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(6) ""true""
+    [2]=>
+    int(6)
+  }
+  [59]=>
+  string(1) ";"
+  [60]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(6)
+  }
+  [61]=>
+  array(3) {
+    [0]=>
+    int(301)
+    [1]=>
+    string(2) "if"
+    [2]=>
+    int(7)
+  }
+  [62]=>
+  string(1) "("
+  [63]=>
+  string(1) "!"
+  [64]=>
+  array(3) {
+    [0]=>
+    int(290)
+    [1]=>
+    string(9) "(boolean)"
+    [2]=>
+    int(7)
+  }
+  [65]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(7)
+  }
+  [66]=>
+  string(1) ")"
+  [67]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(7)
+  }
+  [68]=>
+  array(3) {
+    [0]=>
+    int(316)
+    [1]=>
+    string(4) "echo"
+    [2]=>
+    int(7)
+  }
+  [69]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(7)
+  }
+  [70]=>
+  array(3) {
+    [0]=>
+    int(315)
+    [1]=>
+    string(7) ""false""
+    [2]=>
+    int(7)
+  }
+  [71]=>
+  string(1) ";"
+  [72]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(7)
+  }
+  [73]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$c"
+    [2]=>
+    int(8)
+  }
+  [74]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [75]=>
+  string(1) "="
+  [76]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [77]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$c"
+    [2]=>
+    int(8)
+  }
+  [78]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [79]=>
+  string(1) "+"
+  [80]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [81]=>
+  array(3) {
+    [0]=>
+    int(295)
+    [1]=>
+    string(9) "(integer)"
+    [2]=>
+    int(8)
+  }
+  [82]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [83]=>
+  array(3) {
+    [0]=>
+    int(306)
+    [1]=>
+    string(7) "123.4e2"
+    [2]=>
+    int(8)
+  }
+  [84]=>
+  string(1) ";"
+  [85]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(8)
+  }
+  [86]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$d"
+    [2]=>
+    int(9)
+  }
+  [87]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(9)
+  }
+  [88]=>
+  string(1) "="
+  [89]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(9)
+  }
+  [90]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$c"
+    [2]=>
+    int(9)
+  }
+  [91]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(9)
+  }
+  [92]=>
+  string(1) "-"
+  [93]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(9)
+  }
+  [94]=>
+  array(3) {
+    [0]=>
+    int(294)
+    [1]=>
+    string(6) "(real)"
+    [2]=>
+    int(9)
+  }
+  [95]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(9)
+  }
+  [96]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(2) "12"
+    [2]=>
+    int(9)
+  }
+  [97]=>
+  string(1) ";"
+  [98]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(9)
+  }
+  [99]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(10)
+  }
+  [100]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(10)
+  }
+  [101]=>
+  string(1) "="
+  [102]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(10)
+  }
+  [103]=>
+  array(3) {
+    [0]=>
+    int(289)
+    [1]=>
+    string(7) "(unset)"
+    [2]=>
+    int(10)
+  }
+  [104]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(10)
+  }
+  [105]=>
+  string(1) ";"
+  [106]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(10)
+  }
+  [107]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(11)
+  }
+}
+Done
diff --git a/ext/tokenizer/tests/token_get_all_variation9.phpt b/ext/tokenizer/tests/token_get_all_variation9.phpt
new file mode 100644
index 0000000000..247b754f7c
--- /dev/null
+++ b/ext/tokenizer/tests/token_get_all_variation9.phpt
@@ -0,0 +1,441 @@
+--TEST--
+Test token_get_all() function : usage variations - with different types of comments
+--FILE--
+<?php
+/* Prototype  : array token_get_all(string $source)
+ * Description: splits the given source into an array of PHP languange tokens
+ * Source code: ext/tokenizer/tokenizer.c
+*/
+
+/*
+ * Passing 'source' argument with different style of comments
+ */
+//  '//', '/* */', '#' - T_COMMENT(365)
+// '/** */' - T_DOC_COMMENT(366)
+
+
+echo "*** Testing token_get_all() : 'source' string with different comments ***\n";
+
+// types of comments: '//', '/* */', '#' & /** */
+
+$source = '<?php 
+/** Performing addition operation on given values :
+  * a, b
+  */
+
+// int value
+$a = 10;
+$b = 20;
+$c = true; // bool value
+
+/* 
+ * Performing operation on $a,$b 
+ * display result
+ */
+$c = $a + $b;
+var_dump($c); # expected: int(30)
+
+# end of program
+?>';
+var_dump( token_get_all($source));
+
+echo "Done"
+?>
+--EXPECTF--
+*** Testing token_get_all() : 'source' string with different comments ***
+array(51) {
+  [0]=>
+  array(3) {
+    [0]=>
+    int(367)
+    [1]=>
+    string(6) "<?php "
+    [2]=>
+    int(1)
+  }
+  [1]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(1)
+  }
+  [2]=>
+  array(3) {
+    [0]=>
+    int(366)
+    [1]=>
+    string(65) "/** Performing addition operation on given values :
+  * a, b
+  */"
+    [2]=>
+    int(2)
+  }
+  [3]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(2) "
+
+"
+    [2]=>
+    int(4)
+  }
+  [4]=>
+  array(3) {
+    [0]=>
+    int(365)
+    [1]=>
+    string(13) "// int value
+"
+    [2]=>
+    int(6)
+  }
+  [5]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(7)
+  }
+  [6]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(7)
+  }
+  [7]=>
+  string(1) "="
+  [8]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(7)
+  }
+  [9]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(2) "10"
+    [2]=>
+    int(7)
+  }
+  [10]=>
+  string(1) ";"
+  [11]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(7)
+  }
+  [12]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(8)
+  }
+  [13]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [14]=>
+  string(1) "="
+  [15]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(8)
+  }
+  [16]=>
+  array(3) {
+    [0]=>
+    int(305)
+    [1]=>
+    string(2) "20"
+    [2]=>
+    int(8)
+  }
+  [17]=>
+  string(1) ";"
+  [18]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(8)
+  }
+  [19]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$c"
+    [2]=>
+    int(9)
+  }
+  [20]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(9)
+  }
+  [21]=>
+  string(1) "="
+  [22]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(9)
+  }
+  [23]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(4) "true"
+    [2]=>
+    int(9)
+  }
+  [24]=>
+  string(1) ";"
+  [25]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(9)
+  }
+  [26]=>
+  array(3) {
+    [0]=>
+    int(365)
+    [1]=>
+    string(14) "// bool value
+"
+    [2]=>
+    int(9)
+  }
+  [27]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(10)
+  }
+  [28]=>
+  array(3) {
+    [0]=>
+    int(365)
+    [1]=>
+    string(59) "/* 
+ * Performing operation on $a,$b 
+ * display result
+ */"
+    [2]=>
+    int(11)
+  }
+  [29]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(14)
+  }
+  [30]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$c"
+    [2]=>
+    int(15)
+  }
+  [31]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(15)
+  }
+  [32]=>
+  string(1) "="
+  [33]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(15)
+  }
+  [34]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$a"
+    [2]=>
+    int(15)
+  }
+  [35]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(15)
+  }
+  [36]=>
+  string(1) "+"
+  [37]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(15)
+  }
+  [38]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$b"
+    [2]=>
+    int(15)
+  }
+  [39]=>
+  string(1) ";"
+  [40]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(15)
+  }
+  [41]=>
+  array(3) {
+    [0]=>
+    int(307)
+    [1]=>
+    string(8) "var_dump"
+    [2]=>
+    int(16)
+  }
+  [42]=>
+  string(1) "("
+  [43]=>
+  array(3) {
+    [0]=>
+    int(309)
+    [1]=>
+    string(2) "$c"
+    [2]=>
+    int(16)
+  }
+  [44]=>
+  string(1) ")"
+  [45]=>
+  string(1) ";"
+  [46]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) " "
+    [2]=>
+    int(16)
+  }
+  [47]=>
+  array(3) {
+    [0]=>
+    int(365)
+    [1]=>
+    string(20) "# expected: int(30)
+"
+    [2]=>
+    int(16)
+  }
+  [48]=>
+  array(3) {
+    [0]=>
+    int(370)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(17)
+  }
+  [49]=>
+  array(3) {
+    [0]=>
+    int(365)
+    [1]=>
+    string(17) "# end of program
+"
+    [2]=>
+    int(18)
+  }
+  [50]=>
+  array(3) {
+    [0]=>
+    int(369)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(19)
+  }
+}
+Done