MDEV-28762: recursive call of some json functions without stack control

Analysis: Some recursive json functions dont check for stack control
Fix: Add check_stack_overrun(). The last argument is NULL because it is not
used

4 files changed, 196 insertions, 101 deletions

diff --git a/mysql-test/main/json_debug_nonembedded.result b/mysql-test/main/json_debug_nonembedded.result
new file mode 100644
index 00000000000..da4ea54e30c
--- /dev/null
+++ b/mysql-test/main/json_debug_nonembedded.result
@@ -0,0 +1,28 @@
+#
+# Beginning of 10.3 test
+# MDEV-28762: recursive call of some json functions without stack control
+#
+SET @saved_dbug = @@debug_dbug;
+SET debug_dbug='+d,json_check_min_stack_requirement';
+SET @json1= '{"key1":"val1"}';
+SET @json2= '{"key1":"val1"}';
+SELECT JSON_CONTAINS(@json1, @json2);
+ERROR HY000: Thread stack overrun:  'used bytes' used of a 'available' byte stack, and 'X' bytes needed.  Use 'mysqld --thread_stack=#' to specify a bigger stack
+SET debug_dbug='+d,temp';
+SET @json1= '[1, 2, 3, 4]';
+SET @json2= '[5, 6, 7, 8]';
+SELECT JSON_MERGE(@json1, @json2);
+ERROR HY000: Thread stack overrun:  'used bytes' used of a 'available' byte stack, and 'X' bytes needed.  Use 'mysqld --thread_stack=#' to specify a bigger stack
+SELECT JSON_MERGE_PATCH(@json1, @json2);
+ERROR HY000: Thread stack overrun:  'used bytes' used of a 'available' byte stack, and 'X' bytes needed.  Use 'mysqld --thread_stack=#' to specify a bigger stack
+SELECT JSON_CONTAINS_PATH('{"a":[{"c":[1,{"a":[0,1,2]},3]}], "b":[1,2,3]}', 'one', "$**.a[2]");
+ERROR HY000: Thread stack overrun:  'used bytes' used of a 'available' byte stack, and 'X' bytes needed.  Use 'mysqld --thread_stack=#' to specify a bigger stack
+SET @j = '["abc", [{"k": "10"}, "def"], {"x":"abc"}, {"y":"bcd"}]';
+SELECT JSON_SEARCH(@j, 'all', 'abc', NULL, '$[2]');
+ERROR HY000: Thread stack overrun:  'used bytes' used of a 'available' byte stack, and 'X' bytes needed.  Use 'mysqld --thread_stack=#' to specify a bigger stack
+SELECT JSON_EXTRACT('{"key1":"asd", "key2":[2,3]}', "$.key1", "$.key2");
+ERROR HY000: Thread stack overrun:  'used bytes' used of a 'available' byte stack, and 'X' bytes needed.  Use 'mysqld --thread_stack=#' to specify a bigger stack
+SET @@debug_dbug= @saved_dbug;
+#
+# End of 10.3 test
+#
diff --git a/mysql-test/main/json_debug_nonembedded.test b/mysql-test/main/json_debug_nonembedded.test
new file mode 100644
index 00000000000..1fbbc05f3b2
--- /dev/null
+++ b/mysql-test/main/json_debug_nonembedded.test
@@ -0,0 +1,47 @@
+-- source include/not_embedded.inc
+--source include/have_debug.inc
+
+--echo #
+--echo # Beginning of 10.3 test
+--echo # MDEV-28762: recursive call of some json functions without stack control
+--echo #
+
+SET @saved_dbug = @@debug_dbug;
+SET debug_dbug='+d,json_check_min_stack_requirement';
+
+SET @json1= '{"key1":"val1"}';
+SET @json2= '{"key1":"val1"}';
+
+--replace_regex /overrun:  [0-9]* bytes used of a [0-9]* byte stack, and [0-9]* bytes needed/overrun:  'used bytes' used of a 'available' byte stack, and 'X' bytes needed/
+--error ER_STACK_OVERRUN_NEED_MORE
+SELECT JSON_CONTAINS(@json1, @json2);
+
+SET debug_dbug='+d,temp';
+SET @json1= '[1, 2, 3, 4]';
+SET @json2= '[5, 6, 7, 8]';
+--replace_regex /overrun:  [0-9]* bytes used of a [0-9]* byte stack, and [0-9]* bytes needed/overrun:  'used bytes' used of a 'available' byte stack, and 'X' bytes needed/
+--error ER_STACK_OVERRUN_NEED_MORE
+SELECT JSON_MERGE(@json1, @json2);
+
+--replace_regex /overrun:  [0-9]* bytes used of a [0-9]* byte stack, and [0-9]* bytes needed/overrun:  'used bytes' used of a 'available' byte stack, and 'X' bytes needed/
+--error ER_STACK_OVERRUN_NEED_MORE
+SELECT JSON_MERGE_PATCH(@json1, @json2);
+
+--replace_regex /overrun:  [0-9]* bytes used of a [0-9]* byte stack, and [0-9]* bytes needed/overrun:  'used bytes' used of a 'available' byte stack, and 'X' bytes needed/
+--error ER_STACK_OVERRUN_NEED_MORE
+SELECT JSON_CONTAINS_PATH('{"a":[{"c":[1,{"a":[0,1,2]},3]}], "b":[1,2,3]}', 'one', "$**.a[2]");
+
+SET @j = '["abc", [{"k": "10"}, "def"], {"x":"abc"}, {"y":"bcd"}]';--replace_regex /overrun:  [0-9]* bytes used of a [0-9]* byte stack, and [0-9]* bytes needed/overrun:  'used bytes' used of a 'available' byte stack, and 'X' bytes needed/
+--replace_regex /overrun:  [0-9]* bytes used of a [0-9]* byte stack, and [0-9]* bytes needed/overrun:  'used bytes' used of a 'available' byte stack, and 'X' bytes needed/
+--error ER_STACK_OVERRUN_NEED_MORE
+SELECT JSON_SEARCH(@j, 'all', 'abc', NULL, '$[2]');
+
+--replace_regex /overrun:  [0-9]* bytes used of a [0-9]* byte stack, and [0-9]* bytes needed/overrun:  'used bytes' used of a 'available' byte stack, and 'X' bytes needed/
+--error ER_STACK_OVERRUN_NEED_MORE
+SELECT JSON_EXTRACT('{"key1":"asd", "key2":[2,3]}', "$.key1", "$.key2");
+
+SET @@debug_dbug= @saved_dbug;
+
+--echo #
+--echo # End of 10.3 test
+--echo #
diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc
index 51e8825aca8..ff6013d5eb6 100644
--- a/sql/item_jsonfunc.cc
+++ b/sql/item_jsonfunc.cc
@@ -18,6 +18,7 @@
 #include "sql_priv.h"
 #include "sql_class.h"
 #include "item.h"
+#include "sql_parse.h" // For check_stack_overrun
 
 
 /*
@@ -128,6 +129,110 @@ static int append_tab(String *js, int depth, int tab_size)
   return 0;
 }
 
+int json_path_parts_compare(
+    const json_path_step_t *a, const json_path_step_t *a_end,
+    const json_path_step_t *b, const json_path_step_t *b_end,
+    enum json_value_types vt)
+{
+  int res, res2;
+
+  DBUG_EXECUTE_IF("json_check_min_stack_requirement",
+                  {alloca(my_thread_stack_size-(STACK_MIN_SIZE));});
+  if (check_stack_overrun(current_thd, STACK_MIN_SIZE, NULL))
+    return 1;
+  while (a <= a_end)
+  {
+    if (b > b_end)
+    {
+      while (vt != JSON_VALUE_ARRAY &&
+             (a->type & JSON_PATH_ARRAY_WILD) == JSON_PATH_ARRAY &&
+             a->n_item == 0)
+      {
+        if (++a > a_end)
+          return 0;
+      }
+      return -2;
+    }
+
+    DBUG_ASSERT((b->type & (JSON_PATH_WILD | JSON_PATH_DOUBLE_WILD)) == 0);
+
+
+    if (a->type & JSON_PATH_ARRAY)
+    {
+      if (b->type & JSON_PATH_ARRAY)
+      {
+        if ((a->type & JSON_PATH_WILD) || a->n_item == b->n_item)
+          goto step_fits;
+        goto step_failed;
+      }
+      if ((a->type & JSON_PATH_WILD) == 0 && a->n_item == 0)
+        goto step_fits_autowrap;
+      goto step_failed;
+    }
+    else /* JSON_PATH_KEY */
+    {
+      if (!(b->type & JSON_PATH_KEY))
+        goto step_failed;
+
+      if (!(a->type & JSON_PATH_WILD) &&
+          (a->key_end - a->key != b->key_end - b->key ||
+           memcmp(a->key, b->key, a->key_end - a->key) != 0))
+        goto step_failed;
+
+      goto step_fits;
+    }
+step_failed:
+    if (!(a->type & JSON_PATH_DOUBLE_WILD))
+      return -1;
+    b++;
+    continue;
+
+step_fits:
+    b++;
+    if (!(a->type & JSON_PATH_DOUBLE_WILD))
+    {
+      a++;
+      continue;
+    }
+
+    /* Double wild handling needs recursions. */
+    res= json_path_parts_compare(a+1, a_end, b, b_end, vt);
+    if (res == 0)
+      return 0;
+
+    res2= json_path_parts_compare(a, a_end, b, b_end, vt);
+
+    return (res2 >= 0) ? res2 : res;
+
+step_fits_autowrap:
+    if (!(a->type & JSON_PATH_DOUBLE_WILD))
+    {
+      a++;
+      continue;
+    }
+
+    /* Double wild handling needs recursions. */
+    res= json_path_parts_compare(a+1, a_end, b+1, b_end, vt);
+    if (res == 0)
+      return 0;
+
+    res2= json_path_parts_compare(a, a_end, b+1, b_end, vt);
+
+    return (res2 >= 0) ? res2 : res;
+
+  }
+
+  return b <= b_end;
+}
+
+
+int json_path_compare(const json_path_t *a, const json_path_t *b,
+                      enum json_value_types vt)
+{
+  return json_path_parts_compare(a->steps+1, a->last_step,
+                                 b->steps+1, b->last_step, vt);
+}
+
 
 static int json_nice(json_engine_t *je, String *nice_js,
                      Item_func_json_format::formats mode, int tab_size=4)
@@ -1031,6 +1136,11 @@ static int check_contains(json_engine_t *js, json_engine_t *value)
   json_engine_t loc_js;
   bool set_js;
 
+  DBUG_EXECUTE_IF("json_check_min_stack_requirement",
+                  {alloca(my_thread_stack_size-(STACK_MIN_SIZE));});
+  if (check_stack_overrun(current_thd, STACK_MIN_SIZE, NULL))
+   return 0;
+
   switch (js->value_type)
   {
   case JSON_VALUE_OBJECT:
@@ -1919,6 +2029,12 @@ err_return:
 
 static int do_merge(String *str, json_engine_t *je1, json_engine_t *je2)
 {
+
+  DBUG_EXECUTE_IF("json_check_min_stack_requirement",
+                  {alloca(my_thread_stack_size-(STACK_MIN_SIZE));});
+  if (check_stack_overrun(current_thd, STACK_MIN_SIZE, NULL))
+   return 1;
+
   if (json_read_value(je1) || json_read_value(je2))
     return 1;
 
@@ -2251,6 +2367,11 @@ static int copy_value_patch(String *str, json_engine_t *je)
 static int do_merge_patch(String *str, json_engine_t *je1, json_engine_t *je2,
                           bool *empty_result)
 {
+  DBUG_EXECUTE_IF("json_check_min_stack_requirement",
+                  {alloca(my_thread_stack_size-(STACK_MIN_SIZE));});
+  if (check_stack_overrun(current_thd, STACK_MIN_SIZE, NULL))
+    return 1;
+
   if (json_read_value(je1) || json_read_value(je2))
     return 1;
 
diff --git a/strings/json_lib.c b/strings/json_lib.c
index 8ae921b348a..6717d0079f1 100644
--- a/strings/json_lib.c
+++ b/strings/json_lib.c
@@ -1751,104 +1751,3 @@ int json_get_path_next(json_engine_t *je, json_path_t *p)
 
   return 1;
 }
-
-
-int json_path_parts_compare(
-    const json_path_step_t *a, const json_path_step_t *a_end,
-    const json_path_step_t *b, const json_path_step_t *b_end,
-    enum json_value_types vt)
-{
-  int res, res2;
-
-  while (a <= a_end)
-  {
-    if (b > b_end)
-    {
-      while (vt != JSON_VALUE_ARRAY &&
-             (a->type & JSON_PATH_ARRAY_WILD) == JSON_PATH_ARRAY &&
-             a->n_item == 0)
-      {
-        if (++a > a_end)
-          return 0;
-      }
-      return -2;
-    }
-
-    DBUG_ASSERT((b->type & (JSON_PATH_WILD | JSON_PATH_DOUBLE_WILD)) == 0);
-
-    
-    if (a->type & JSON_PATH_ARRAY)
-    {
-      if (b->type & JSON_PATH_ARRAY)
-      {
-        if ((a->type & JSON_PATH_WILD) || a->n_item == b->n_item)
-          goto step_fits;
-        goto step_failed;
-      }
-      if ((a->type & JSON_PATH_WILD) == 0 && a->n_item == 0)
-        goto step_fits_autowrap;
-      goto step_failed;
-    }
-    else /* JSON_PATH_KEY */
-    {
-      if (!(b->type & JSON_PATH_KEY))
-        goto step_failed;
-    
-      if (!(a->type & JSON_PATH_WILD) &&
-          (a->key_end - a->key != b->key_end - b->key ||
-           memcmp(a->key, b->key, a->key_end - a->key) != 0))
-        goto step_failed;
-
-      goto step_fits;
-    }
-step_failed:
-    if (!(a->type & JSON_PATH_DOUBLE_WILD))
-      return -1;
-    b++;
-    continue;
-
-step_fits:
-    b++;
-    if (!(a->type & JSON_PATH_DOUBLE_WILD))
-    {
-      a++;
-      continue;
-    }
-
-    /* Double wild handling needs recursions. */
-    res= json_path_parts_compare(a+1, a_end, b, b_end, vt);
-    if (res == 0)
-      return 0;
-
-    res2= json_path_parts_compare(a, a_end, b, b_end, vt);
-
-    return (res2 >= 0) ? res2 : res;
-
-step_fits_autowrap:
-    if (!(a->type & JSON_PATH_DOUBLE_WILD))
-    {
-      a++;
-      continue;
-    }
-
-    /* Double wild handling needs recursions. */
-    res= json_path_parts_compare(a+1, a_end, b+1, b_end, vt);
-    if (res == 0)
-      return 0;
-
-    res2= json_path_parts_compare(a, a_end, b+1, b_end, vt);
-
-    return (res2 >= 0) ? res2 : res;
-
-  }
-
-  return b <= b_end;
-}
-
-
-int json_path_compare(const json_path_t *a, const json_path_t *b,
-                      enum json_value_types vt)
-{
-  return json_path_parts_compare(a->steps+1, a->last_step,
-                                 b->steps+1, b->last_step, vt);
-}