More regex memory leak fixes and testcases

(cherry picked from commit ef06edbee6463061a7f3dcbd2f56a625b41a4810)
(cherry picked from commit e9b9cbf5e9bdcda6f0b50456658bac748202dd70)

4 files changed, 53 insertions, 5 deletions

diff --git a/posix/Makefile b/posix/Makefile
index 2a467a8f7b..ad2f2eda6f 100644
--- a/posix/Makefile
+++ b/posix/Makefile
@@ -82,7 +82,7 @@ tests		:= tstgetopt testfnm runtests runptests	     \
 		   bug-regex17 bug-regex18 bug-regex19 bug-regex20 \
 		   bug-regex21 bug-regex22 bug-regex23 bug-regex24 \
 		   bug-regex25 bug-regex26 bug-regex27 bug-regex28 \
-		   bug-regex29 bug-regex30 \
+		   bug-regex29 bug-regex30 bug-regex31 \
 		   tst-nice tst-nanosleep tst-regex2 \
 		   transbug tst-rxspencer tst-pcre tst-boost \
 		   bug-ga1 tst-vfork1 tst-vfork2 tst-vfork3 tst-waitid \
@@ -109,6 +109,7 @@ generated := $(addprefix wordexp-test-result, 1 2 3 4 5 6 7 8 9 10) \
 	     annexc annexc.out wordexp-tst.out bug-regex2-mem \
 	     bug-regex2.mtrace bug-regex14-mem bug-regex14.mtrace \
 	     bug-regex21-mem bug-regex21.mtrace \
+	     bug-regex31-mem bug-regex31.mtrace \
 	     tst-rxspencer-mem tst-rxspencer.mtrace tst-getconf.out \
 	     tst-pcre-mem tst-pcre.mtrace tst-boost-mem tst-boost.mtrace \
 	     bug-ga2.mtrace bug-ga2-mem bug-glob2.mtrace bug-glob2-mem \
@@ -222,7 +223,7 @@ endif
 tests: $(objpfx)annexc.out
 ifeq (no,$(cross-compiling))
 tests: $(objpfx)bug-regex2-mem $(objpfx)bug-regex14-mem \
-  $(objpfx)bug-regex21-mem $(objpfx)tst-rxspencer-mem \
+  $(objpfx)bug-regex21-mem $(objpfx)bug-regex31-mem $(objpfx)tst-rxspencer-mem\
   $(objpfx)tst-pcre-mem $(objpfx)tst-boost-mem $(objpfx)tst-getconf.out \
   $(objpfx)bug-glob2-mem $(objpfx)tst-vfork3-mem
 xtests: $(objpfx)bug-ga2-mem
@@ -251,6 +252,11 @@ bug-regex21-ENV = MALLOC_TRACE=$(objpfx)bug-regex21.mtrace
 $(objpfx)bug-regex21-mem: $(objpfx)bug-regex21.out
 	$(common-objpfx)malloc/mtrace $(objpfx)bug-regex21.mtrace > $@
 
+bug-regex31-ENV = MALLOC_TRACE=$(objpfx)bug-regex31.mtrace
+
+$(objpfx)bug-regex31-mem: $(objpfx)bug-regex31.out
+	$(common-objpfx)malloc/mtrace $(objpfx)bug-regex31.mtrace > $@
+
 tst-vfork3-ENV = MALLOC_TRACE=$(objpfx)tst-vfork3.mtrace
 
 $(objpfx)tst-vfork3-mem: $(objpfx)tst-vfork3.out
diff --git a/posix/bug-regex31.c b/posix/bug-regex31.c
new file mode 100644
index 0000000000..974e8603b9
--- /dev/null
+++ b/posix/bug-regex31.c
@@ -0,0 +1,36 @@
+#include <mcheck.h>
+#include <regex.h>
+#include <stdio.h>
+#include <sys/types.h>
+
+int
+main (void)
+{
+  mtrace ();
+
+  int res = 0;
+  char *buf = NULL;
+  size_t len = 0;
+  while (! feof (stdin))
+    {
+      ssize_t n = getline (&buf, &len, stdin);
+      if (n <= 0)
+	break;
+      if (buf[n - 1] == '\n')
+	buf[n - 1] = '\0';
+
+      regex_t regex;
+      int rc = regcomp (&regex, buf, REG_EXTENDED);
+      if (rc != 0)
+	printf ("%s: Error %d (expected)\n", buf, rc);
+      else
+	{
+	  printf ("%s: succeeded !\n", buf);
+	  res = 1;
+	}
+    }
+
+  free (buf);
+
+  return 0;
+}
diff --git a/posix/bug-regex31.input b/posix/bug-regex31.input
new file mode 100644
index 0000000000..eea961ccf5
--- /dev/null
+++ b/posix/bug-regex31.input
@@ -0,0 +1,3 @@
+([0]
+([0]a
+([0]([0])
diff --git a/posix/regcomp.c b/posix/regcomp.c
index e70de870bc..c6de044050 100644
--- a/posix/regcomp.c
+++ b/posix/regcomp.c
@@ -2158,12 +2158,15 @@ parse_branch (re_string_t *regexp, regex_t *preg, re_token_t *token,
 	}
       if (tree != NULL && exp != NULL)
 	{
-	  tree = create_tree (dfa, tree, exp, CONCAT);
-	  if (tree == NULL)
+	  bin_tree_t *newtree = create_tree (dfa, tree, exp, CONCAT);
+	  if (newtree == NULL)
 	    {
+	      postorder (exp, free_tree, NULL);
+	      postorder (tree, free_tree, NULL);
 	      *err = REG_ESPACE;
 	      return NULL;
 	    }
+	  tree = newtree;
 	}
       else if (tree == NULL)
 	tree = exp;
@@ -2414,7 +2417,7 @@ parse_sub_exp (re_string_t *regexp, regex_t *preg, re_token_t *token,
       if (BE (*err == REG_NOERROR && token->type != OP_CLOSE_SUBEXP, 0))
 	{
 	  if (tree != NULL)
-	    free_tree (NULL, tree);
+	    postorder (tree, free_tree, NULL);
 	  *err = REG_EPAREN;
 	}
       if (BE (*err != REG_NOERROR, 0))