MDEV-12420 max_recursive_iterations did not prevent a stack-overflow and segfault

post-review fixes
* move pcre-specific variable out of mysys
* don't use current_thd
* move a commonly used macro to my_sys.h
* remove new sysvar

10 files changed, 24 insertions, 46 deletions

diff --git a/include/my_sys.h b/include/my_sys.h
index 395d0956430..22ed7616f1c 100644
--- a/include/my_sys.h
+++ b/include/my_sys.h
@@ -227,9 +227,6 @@ extern void (*fatal_error_handler_hook)(uint my_err, const char *str,
 				       myf MyFlags);
 extern uint my_file_limit;
 extern ulonglong my_thread_stack_size;
-#ifndef EMBEDDED_LIBRARY
-extern ulonglong my_pcre_frame_size;
-#endif
 extern int sf_leaking_memory; /* set to 1 to disable memleak detection */
 
 extern void (*proc_info_hook)(void *, const PSI_stage_info *, PSI_stage_info *,
@@ -909,6 +906,12 @@ extern ulonglong my_getcputime(void);
 #define hrtime_sec_part(X)              ((ulong)((X).val % HRTIME_RESOLUTION))
 #define my_time(X)                      hrtime_to_time(my_hrtime())
 
+#if STACK_DIRECTION < 0
+#define available_stack_size(CUR,END) (long) ((char*)(CUR) - (char*)(END))
+#else
+#define available_stack_size(CUR,END) (long) ((char*)(END) - (char*)(CUR))
+#endif
+
 #ifdef HAVE_SYS_MMAN_H
 #include <sys/mman.h>
 
diff --git a/mysys/lf_alloc-pin.c b/mysys/lf_alloc-pin.c
index 282433ea48d..a60a2ae657c 100644
--- a/mysys/lf_alloc-pin.c
+++ b/mysys/lf_alloc-pin.c
@@ -328,12 +328,6 @@ static int match_pins(LF_PINS *el, void *addr)
   return 0;
 }
 
-#if STACK_DIRECTION < 0
-#define available_stack_size(CUR,END) (long) ((char*)(CUR) - (char*)(END))
-#else
-#define available_stack_size(CUR,END) (long) ((char*)(END) - (char*)(CUR))
-#endif
-
 #define next_node(P, X) (*((uchar * volatile *)(((uchar *)(X)) + (P)->free_ptr_offset)))
 #define anext_node(X) next_node(&allocator->pinbox, (X))
 
diff --git a/mysys/my_init.c b/mysys/my_init.c
index 206f96827c3..dee41e1202a 100644
--- a/mysys/my_init.c
+++ b/mysys/my_init.c
@@ -45,8 +45,6 @@ my_bool my_init_done= 0;
 uint	mysys_usage_id= 0;              /* Incremented for each my_init() */
 
 ulonglong   my_thread_stack_size= (sizeof(void*) <= 4)? 65536: ((256-16)*1024);
-/* http://pcre.org/original/doc/html/pcrestack.html - replaced by init_pcre value */
-ulonglong   my_pcre_frame_size= 640 + 16;
 
 static ulong atoi_octal(const char *str)
 {
diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc
index bfad63f21f8..038a0ebabc2 100644
--- a/sql/item_cmpfunc.cc
+++ b/sql/item_cmpfunc.cc
@@ -5104,6 +5104,15 @@ int Regexp_processor_pcre::default_regex_flags()
   return default_regex_flags_pcre(current_thd);
 }
 
+void Regexp_processor_pcre::set_recursion_limit(THD *thd)
+{
+  long stack_used;
+  DBUG_ASSERT(thd == current_thd);
+  stack_used= available_stack_size(thd->thread_stack, &stack_used);
+  m_pcre_extra.match_limit_recursion=
+    (my_thread_stack_size - stack_used)/my_pcre_frame_size;
+}
+
 
 /**
   Convert string to lib_charset, if needed.
diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h
index b5b94888330..17ad1bd8c7d 100644
--- a/sql/item_cmpfunc.h
+++ b/sql/item_cmpfunc.h
@@ -25,7 +25,6 @@
 
 #include "thr_malloc.h"                         /* sql_calloc */
 #include "item_func.h"             /* Item_int_func, Item_bool_func */
-long check_stack_available(long margin, uchar *dummy);
 #define PCRE_STATIC 1             /* Important on Windows */
 #include "pcre.h"                 /* pcre header file */
 
@@ -1577,15 +1576,11 @@ public:
     m_library_charset(&my_charset_utf8_general_ci),
     m_subpatterns_needed(0)
   {
-#ifndef EMBEDDED_LIBRARY
-   uchar dummy;
-   m_pcre_extra.flags= PCRE_EXTRA_MATCH_LIMIT_RECURSION;
-   m_pcre_extra.match_limit_recursion= check_stack_available(100, &dummy) / my_pcre_frame_size;
-#else
-   m_pcre_extra.flags= 0L;
-#endif
+    m_pcre_extra.flags= PCRE_EXTRA_MATCH_LIMIT_RECURSION;
+    m_pcre_extra.match_limit_recursion= 100L;
   }
   int default_regex_flags();
+  void set_recursion_limit(THD *);
   void init(CHARSET_INFO *data_charset, int extra_flags, uint nsubpatterns)
   {
     m_library_flags= default_regex_flags() | extra_flags |
diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index f6c6778e906..a8fef1bd8a3 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -3497,6 +3497,7 @@ static void init_libstrings()
 #endif
 }
 
+ulonglong my_pcre_frame_size;
 
 static void init_pcre()
 {
@@ -3504,9 +3505,8 @@ static void init_pcre()
   pcre_free= pcre_stack_free= my_str_free_mysqld;
 #ifndef EMBEDDED_LIBRARY
   pcre_stack_guard= check_enough_stack_size_slow;
-  /* my_pcre_frame_size= -pcre_exec(NULL, NULL, NULL, -999, -999, 0, NULL, 0) + 16;
-  http://pcre.org/original/doc/html/pcrestack.html has reason for + 16 
-  my_pcre_frame_size= -pcre_match(NULL, NULL, NULL, 0, NULL, NULL, 0) + 16; */
+  /* See http://pcre.org/original/doc/html/pcrestack.html */
+  my_pcre_frame_size= -pcre_exec(NULL, NULL, NULL, -999, -999, 0, NULL, 0) + 16;
 #endif
 }
 
diff --git a/sql/mysqld.h b/sql/mysqld.h
index 28ac871d858..e771a0c0da2 100644
--- a/sql/mysqld.h
+++ b/sql/mysqld.h
@@ -488,6 +488,8 @@ extern pthread_t signal_thread;
 extern struct st_VioSSLFd * ssl_acceptor_fd;
 #endif /* HAVE_OPENSSL */
 
+extern ulonglong my_pcre_frame_size;
+
 /*
   The following variables were under INNODB_COMPABILITY_HOOKS
  */
diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc
index edeb34418b2..e57756217ca 100644
--- a/sql/sql_parse.cc
+++ b/sql/sql_parse.cc
@@ -6163,12 +6163,6 @@ bool check_fk_parent_table_access(THD *thd,
 ****************************************************************************/
 
 
-#if STACK_DIRECTION < 0
-#define used_stack(A,B) (long) (A - B)
-#else
-#define used_stack(A,B) (long) (B - A)
-#endif
-
 #ifndef DBUG_OFF
 long max_stack_used;
 #endif
@@ -6185,7 +6179,7 @@ bool check_stack_overrun(THD *thd, long margin,
 {
   long stack_used;
   DBUG_ASSERT(thd == current_thd);
-  if ((stack_used=used_stack(thd->thread_stack,(char*) &stack_used)) >=
+  if ((stack_used= available_stack_size(thd->thread_stack, &stack_used)) >=
       (long) (my_thread_stack_size - margin))
   {
     thd->is_fatal_error= 1;
@@ -6208,14 +6202,6 @@ bool check_stack_overrun(THD *thd, long margin,
   return 0;
 }
 
-long check_stack_available(long margin,
-			   uchar *buf __attribute__((unused)))
-{
-  long stack_top;
-  DBUG_ASSERT(current_thd);
-  return my_thread_stack_size - margin \
-         - used_stack(current_thd->thread_stack,(char*) &stack_top);
-}
 
 #define MY_YACC_INIT 1000			// Start with big alloc
 #define MY_YACC_MAX  32000			// Because of 'short'
diff --git a/sql/sql_parse.h b/sql/sql_parse.h
index b67692fec90..fa414911093 100644
--- a/sql/sql_parse.h
+++ b/sql/sql_parse.h
@@ -134,7 +134,6 @@ bool check_simple_select();
 Item *normalize_cond(Item *cond);
 Item *negate_expression(THD *thd, Item *expr);
 bool check_stack_overrun(THD *thd, long margin, uchar *dummy);
-long check_stack_available(long margin, uchar *dummy);
 
 /* Variables */
 
diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc
index e50217a4459..5ff79a2f235 100644
--- a/sql/sys_vars.cc
+++ b/sql/sys_vars.cc
@@ -2493,14 +2493,6 @@ static Sys_var_ulonglong Sys_thread_stack(
        VALID_RANGE(128*1024, ULONGLONG_MAX), DEFAULT(DEFAULT_THREAD_STACK),
        BLOCK_SIZE(1024));
 
-#ifndef EMBEDDED_LIBRARY
-static Sys_var_ulonglong Sys_my_pcre_frame_size(
-       "pcre_frame_size", "Frame size for pcre_recursion",
-       GLOBAL_VAR(my_pcre_frame_size), NO_CMD_LINE,
-       VALID_RANGE(500,1024), DEFAULT(640 + 16), 1, NO_MUTEX_GUARD,
-       NOT_IN_BINLOG, ON_CHECK(0), ON_UPDATE(0));
-#endif
-
 static Sys_var_charptr Sys_tmpdir(
        "tmpdir", "Path for temporary files. Several paths may "
        "be specified, separated by a "