Fixed bug#58026 - massive recursion and crash in regular expression

handling.

The problem was that parsing of nested regular expression involved
recursive calls. Such recursion didn't take into account the amount of
available stack space, which ended up leading to stack overflow crashes.

mysql-test/t/not_embedded_server.test:
  Added test for bug#58026.
regex/my_regex.h:
  added pointer to function as last argument of my_regex_init() for check
  enough memory in stack.
regex/regcomp.c:
  p_ere() was modified: added call to function for check enough memory
  in stack. Function for check available stack space specified by
  global variable my_regex_enough_mem_in_stack. This variable set to
  NULL for embedded mysqld and to a pointer to function
  check_enough_stack_size otherwise.
regex/reginit.c:
  my_regex_init was modified: pass a pointer to a function for check
  enough memory in stack space. Reset this pointer to NULL in my_regex_end.
sql/mysqld.cc:
  Added function check_enough_stack_size() for check enough memory in stack.
  Passed this function as second argument to my_regex_init. For embedded 
  mysqld passed NULL as second argument.

3 files changed, 21 insertions, 4 deletions

diff --git a/regex/my_regex.h b/regex/my_regex.h
index 0d1cedf5430..30896e29b91 100644
--- a/regex/my_regex.h
+++ b/regex/my_regex.h
@@ -28,6 +28,7 @@ typedef struct {
 
 
 /* === regcomp.c === */
+typedef int (*my_regex_stack_check_t)();
 extern int my_regcomp(my_regex_t *, const char *, int, CHARSET_INFO *charset);
 #define	REG_BASIC	0000
 #define	REG_EXTENDED	0001
@@ -76,7 +77,8 @@ extern void my_regfree(my_regex_t *);
 
 /* === reginit.c === */
 
-extern void my_regex_init(CHARSET_INFO *cs);	/* Should be called for multithread progs */
+/* Should be called for multithread progs */
+extern void my_regex_init(CHARSET_INFO *cs,  my_regex_stack_check_t func);
 extern void my_regex_end(void);	/* If one wants a clean end */
 
 #ifdef __cplusplus
diff --git a/regex/regcomp.c b/regex/regcomp.c
index 81c435ed552..e163a9ba7f4 100644
--- a/regex/regcomp.c
+++ b/regex/regcomp.c
@@ -31,6 +31,9 @@ struct parse {
 	CHARSET_INFO *charset;	/* for ctype things  */
 };
 
+/* Check if there is enough stack space for recursion. */
+my_regex_stack_check_t my_regex_enough_mem_in_stack= NULL;
+
 #include "regcomp.ih"
 
 static char nuls[10];		/* place to point scanner in event of error */
@@ -117,7 +120,7 @@ CHARSET_INFO *charset;
 #	define	GOODFLAGS(f)	((f)&~REG_DUMP)
 #endif
 
-	my_regex_init(charset);	/* Init cclass if neaded */
+	my_regex_init(charset, NULL);	/* Init cclass if neaded */
 	preg->charset=charset;
 	cflags = GOODFLAGS(cflags);
 	if ((cflags&REG_EXTENDED) && (cflags&REG_NOSPEC))
@@ -222,7 +225,15 @@ int stop;			/* character this ERE should end at */
 		/* do a bunch of concatenated expressions */
 		conc = HERE();
 		while (MORE() && (c = PEEK()) != '|' && c != stop)
-			p_ere_exp(p);
+		{
+		  if (my_regex_enough_mem_in_stack &&
+		      my_regex_enough_mem_in_stack())
+		  {
+		    SETERROR(REG_ESPACE);
+		    return;
+		  }
+		  p_ere_exp(p);
+		}
 		if(REQUIRE(HERE() != conc, REG_EMPTY)) {}/* require nonempty */
 
 		if (!EAT('|'))
diff --git a/regex/reginit.c b/regex/reginit.c
index 5980de24030..3d2cd64d1e7 100644
--- a/regex/reginit.c
+++ b/regex/reginit.c
@@ -4,10 +4,12 @@
 #include <m_ctype.h>
 #include <m_string.h>
 #include "cclass.h"
+#include "my_regex.h"
 
 static my_bool regex_inited=0;
+extern my_regex_stack_check_t my_regex_enough_mem_in_stack;
 
-void my_regex_init(CHARSET_INFO *cs)
+void my_regex_init(CHARSET_INFO *cs, my_regex_stack_check_t func)
 {
   char buff[CCLASS_LAST][256];
   int  count[CCLASS_LAST];
@@ -16,6 +18,7 @@ void my_regex_init(CHARSET_INFO *cs)
   if (!regex_inited)
   {
     regex_inited=1;
+    my_regex_enough_mem_in_stack= func;
     bzero((uchar*) &count,sizeof(count));
 
     for (i=1 ; i<= 255; i++)
@@ -74,6 +77,7 @@ void my_regex_end()
     int i;
     for (i=0; i < CCLASS_LAST ; i++)
       free((char*) cclasses[i].chars);
+    my_regex_enough_mem_in_stack= NULL;
     regex_inited=0;
   }
 }