Remove computing the JIT read-only data size in advance and use on-demand memory allocation.

git-svn-id: svn://vcs.exim.org/pcre/code/trunk@1530 2f5784b3-3f2a-0410-8824-cb99058d5e15

6 files changed, 81 insertions, 75 deletions

diff --git a/ChangeLog b/ChangeLog
index 20858d1..08e4f16 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -88,6 +88,11 @@ Version 8.37 xx-xxx-2015
     incorrect code to be compiled. This bug was reported as "heap overflow",
     discovered by Kai Lu of Fortinet's FortiGuard Labs.
 
+24. Computing the size of the JIT read-only data in advance has been a source
+    of various issues, and new ones are still appear unfortunately. To fix
+    existing and future issues, size computation is eliminated from the code,
+    and replaced by on-demand memory allocation.
+
 
 Version 8.36 26-September-2014
 ------------------------------
diff --git a/pcre_jit_compile.c b/pcre_jit_compile.c
index fbc574f..46ce097 100644
--- a/pcre_jit_compile.c
+++ b/pcre_jit_compile.c
@@ -179,7 +179,7 @@ typedef struct jit_arguments {
 
 typedef struct executable_functions {
   void *executable_funcs[JIT_NUMBER_OF_COMPILE_MODES];
-  sljit_uw *read_only_data[JIT_NUMBER_OF_COMPILE_MODES];
+  void *read_only_data_heads[JIT_NUMBER_OF_COMPILE_MODES];
   sljit_uw executable_sizes[JIT_NUMBER_OF_COMPILE_MODES];
   PUBL(jit_callback) callback;
   void *userdata;
@@ -322,12 +322,8 @@ typedef struct compiler_common {
   pcre_uchar *start;
   /* Maps private data offset to each opcode. */
   sljit_si *private_data_ptrs;
-  /* This read-only data is available during runtime. */
-  sljit_uw *read_only_data;
-  /* The total size of the read-only data. */
-  sljit_uw read_only_data_size;
-  /* The next free entry of the read_only_data. */
-  sljit_uw *read_only_data_ptr;
+  /* Chain list of read-only data ptrs. */
+  void *read_only_data_head;
   /* Tells whether the capturing bracket is optimized. */
   pcre_uint8 *optimized_cbracket;
   /* Tells whether the starting offset is a target of then. */
@@ -802,16 +798,6 @@ while (cc < ccend)
     cc += 1 + IMM2_SIZE;
     break;
 
-    case OP_BRA:
-    case OP_CBRA:
-    case OP_SBRA:
-    case OP_SCBRA:
-    count = no_alternatives(cc);
-    if (count > 4)
-      common->read_only_data_size += count * sizeof(sljit_uw);
-    cc += 1 + LINK_SIZE + (*cc == OP_CBRA || *cc == OP_SCBRA ? IMM2_SIZE : 0);
-    break;
-
     case OP_CBRAPOS:
     case OP_SCBRAPOS:
     common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] = 0;
@@ -2114,6 +2100,40 @@ DEFINE_COMPILER;
 OP2(SLJIT_SUB, STACK_TOP, 0, STACK_TOP, 0, SLJIT_IMM, size * sizeof(sljit_sw));
 }
 
+static sljit_uw * allocate_read_only_data(compiler_common *common, sljit_uw size)
+{
+DEFINE_COMPILER;
+sljit_uw *result;
+
+if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
+  return NULL;
+
+result = (sljit_uw *)SLJIT_MALLOC(size + sizeof(sljit_uw), common->allocator_data);
+if (SLJIT_UNLIKELY(result == NULL))
+  {
+  sljit_set_compiler_memory_error(compiler);
+  return NULL;
+  }
+
+*(void**)result = common->read_only_data_head;
+common->read_only_data_head = (void *)result;
+return result + 1;
+}
+
+static void free_read_only_data(void *current, void *allocator_data)
+{
+void *next;
+
+SLJIT_UNUSED_ARG(allocator_data);
+
+while (current != NULL)
+  {
+  next = *(void**)current;
+  SLJIT_FREE(current, allocator_data);
+  current = next;
+  }
+}
+
 static SLJIT_INLINE void reset_ovector(compiler_common *common, int length)
 {
 DEFINE_COMPILER;
@@ -3530,9 +3550,6 @@ int range_right = -1, range_len = 3 - 1;
 sljit_ub *update_table = NULL;
 BOOL in_range;
 
-/* This is even TRUE, if both are NULL. */
-SLJIT_ASSERT(common->read_only_data_ptr == common->read_only_data);
-
 for (i = 0; i < MAX_N_CHARS; i++)
   {
   chars[i << 1] = NOTACHAR;
@@ -3581,18 +3598,9 @@ for (i = 0; i <= max; i++)
 
 if (range_right >= 0)
   {
-  /* Since no data is consumed (see the assert in the beginning
-  of this function), this space can be reallocated. */
-  if (common->read_only_data)
-    SLJIT_FREE(common->read_only_data, compiler->allocator_data);
-
-  common->read_only_data_size += 256;
-  common->read_only_data = (sljit_uw *)SLJIT_MALLOC(common->read_only_data_size, compiler->allocator_data);
-  if (common->read_only_data == NULL)
+  update_table = (sljit_ub *)allocate_read_only_data(common, 256);
+  if (update_table == NULL)
     return TRUE;
-
-  update_table = (sljit_ub *)common->read_only_data;
-  common->read_only_data_ptr = (sljit_uw *)(update_table + 256);
   memset(update_table, IN_UCHARS(range_len), 256);
 
   for (i = 0; i < range_len; i++)
@@ -8982,8 +8990,9 @@ else if (has_alternatives)
   if (alt_max > 4)
     {
     /* Table jump if alt_max is greater than 4. */
-    next_update_addr = common->read_only_data_ptr;
-    common->read_only_data_ptr += alt_max;
+    next_update_addr = allocate_read_only_data(common, alt_max * sizeof(sljit_uw));
+    if (SLJIT_UNLIKELY(next_update_addr == NULL))
+      return;
     sljit_emit_ijump(compiler, SLJIT_JUMP, SLJIT_MEM1(TMP1), (sljit_sw)next_update_addr);
     add_label_addr(common, next_update_addr++);
     }
@@ -9766,9 +9775,7 @@ memset(common, 0, sizeof(compiler_common));
 rootbacktrack.cc = (pcre_uchar *)re + re->name_table_offset + re->name_count * re->name_entry_size;
 
 common->start = rootbacktrack.cc;
-common->read_only_data = NULL;
-common->read_only_data_size = 0;
-common->read_only_data_ptr = NULL;
+common->read_only_data_head = NULL;
 common->fcc = tables + fcc_offset;
 common->lcc = (sljit_sw)(tables + lcc_offset);
 common->mode = mode;
@@ -9951,25 +9958,11 @@ if (common->has_then)
   set_then_offsets(common, common->start, NULL);
   }
 
-if (common->read_only_data_size > 0)
-  {
-  common->read_only_data = (sljit_uw *)SLJIT_MALLOC(common->read_only_data_size, compiler->allocator_data);
-  if (common->read_only_data == NULL)
-    {
-    SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data);
-    SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data);
-    return;
-    }
-  common->read_only_data_ptr = common->read_only_data;
-  }
-
 compiler = sljit_create_compiler(NULL);
 if (!compiler)
   {
   SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data);
   SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data);
-  if (common->read_only_data)
-    SLJIT_FREE(common->read_only_data, compiler->allocator_data);
   return;
   }
 common->compiler = compiler;
@@ -10008,16 +10001,7 @@ if ((re->options & PCRE_ANCHORED) == 0)
   if ((re->options & PCRE_NO_START_OPTIMIZE) == 0)
     {
     if (mode == JIT_COMPILE && fast_forward_first_n_chars(common, (re->options & PCRE_FIRSTLINE) != 0))
-      {
-      /* If read_only_data is reallocated, we might have an allocation failure. */
-      if (common->read_only_data_size > 0 && common->read_only_data == NULL)
-        {
-        sljit_free_compiler(compiler);
-        SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data);
-        SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data);
-        return;
-        }
-      }
+      ;
     else if ((re->flags & PCRE_FIRSTSET) != 0)
       fast_forward_first_char(common, (pcre_uchar)re->first_char, (re->flags & PCRE_FCH_CASELESS) != 0, (re->options & PCRE_FIRSTLINE) != 0);
     else if ((re->flags & PCRE_STARTLINE) != 0)
@@ -10070,8 +10054,7 @@ if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
   sljit_free_compiler(compiler);
   SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data);
   SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data);
-  if (common->read_only_data)
-    SLJIT_FREE(common->read_only_data, compiler->allocator_data);
+  free_read_only_data(common->read_only_data_head, compiler->allocator_data);
   return;
   }
 
@@ -10111,8 +10094,7 @@ if (SLJIT_UNLIKELY(sljit_get_compiler_error(compiler)))
   sljit_free_compiler(compiler);
   SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data);
   SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data);
-  if (common->read_only_data)
-    SLJIT_FREE(common->read_only_data, compiler->allocator_data);
+  free_read_only_data(common->read_only_data_head, compiler->allocator_data);
   return;
   }
 
@@ -10192,8 +10174,7 @@ while (common->currententry != NULL)
     sljit_free_compiler(compiler);
     SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data);
     SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data);
-    if (common->read_only_data)
-      SLJIT_FREE(common->read_only_data, compiler->allocator_data);
+    free_read_only_data(common->read_only_data_head, compiler->allocator_data);
     return;
     }
   flush_stubs(common);
@@ -10303,7 +10284,6 @@ if (common->getucd != NULL)
   }
 #endif
 
-SLJIT_ASSERT(common->read_only_data + (common->read_only_data_size >> SLJIT_WORD_SHIFT) == common->read_only_data_ptr);
 SLJIT_FREE(common->optimized_cbracket, compiler->allocator_data);
 SLJIT_FREE(common->private_data_ptrs, compiler->allocator_data);
 
@@ -10318,8 +10298,7 @@ while (label_addr != NULL)
 sljit_free_compiler(compiler);
 if (executable_func == NULL)
   {
-  if (common->read_only_data)
-    SLJIT_FREE(common->read_only_data, compiler->allocator_data);
+  free_read_only_data(common->read_only_data_head, compiler->allocator_data);
   return;
   }
 
@@ -10343,8 +10322,7 @@ else
     /* This case is highly unlikely since we just recently
     freed a lot of memory. Not impossible though. */
     sljit_free_code(executable_func);
-    if (common->read_only_data)
-      SLJIT_FREE(common->read_only_data, compiler->allocator_data);
+    free_read_only_data(common->read_only_data_head, compiler->allocator_data);
     return;
     }
   memset(functions, 0, sizeof(executable_functions));
@@ -10355,7 +10333,7 @@ else
   }
 
 functions->executable_funcs[mode] = executable_func;
-functions->read_only_data[mode] = common->read_only_data;
+functions->read_only_data_heads[mode] = common->read_only_data_head;
 functions->executable_sizes[mode] = executable_size;
 }
 
@@ -10542,8 +10520,7 @@ for (i = 0; i < JIT_NUMBER_OF_COMPILE_MODES; i++)
   {
   if (functions->executable_funcs[i] != NULL)
     sljit_free_code(functions->executable_funcs[i]);
-  if (functions->read_only_data[i] != NULL)
-    SLJIT_FREE(functions->read_only_data[i], compiler->allocator_data);
+  free_read_only_data(functions->read_only_data_heads[i], NULL);
   }
 SLJIT_FREE(functions, compiler->allocator_data);
 }
diff --git a/sljit/sljitLir.h b/sljit/sljitLir.h
index 229e04f..79f1d10 100644
--- a/sljit/sljitLir.h
+++ b/sljit/sljitLir.h
@@ -429,6 +429,12 @@ SLJIT_API_FUNC_ATTRIBUTE void sljit_free_compiler(struct sljit_compiler *compile
    these checks increases the performance of the compiling process. */
 static SLJIT_INLINE sljit_si sljit_get_compiler_error(struct sljit_compiler *compiler) { return compiler->error; }
 
+/* Sets the compiler error code to SLJIT_ERR_ALLOC_FAILED. After
+   the error code is set, the compiler behaves as if itself detected
+   an allocation failure. This can greatly simplify error management,
+   since only the compiler needs to be checked after compilation. */
+static SLJIT_INLINE void sljit_set_compiler_memory_error(struct sljit_compiler *compiler) { compiler->error = SLJIT_ERR_ALLOC_FAILED; }
+
 /*
    Allocate a small amount of memory. The size must be <= 64 bytes on 32 bit,
    and <= 128 bytes on 64 bit architectures. The memory area is owned by the
diff --git a/sljit/sljitNativeARM_32.c b/sljit/sljitNativeARM_32.c
index a763b81..aca1d31 100644
--- a/sljit/sljitNativeARM_32.c
+++ b/sljit/sljitNativeARM_32.c
@@ -315,11 +315,13 @@ struct future_patch {
 	sljit_si value;
 };
 
-static SLJIT_INLINE sljit_si resolve_const_pool_index(struct sljit_compiler *compiler, struct future_patch **first_patch, sljit_uw cpool_current_index, sljit_uw *cpool_start_address, sljit_uw *buf_ptr)
+static sljit_si resolve_const_pool_index(struct sljit_compiler *compiler, struct future_patch **first_patch, sljit_uw cpool_current_index, sljit_uw *cpool_start_address, sljit_uw *buf_ptr)
 {
 	sljit_si value;
 	struct future_patch *curr_patch, *prev_patch;
 
+	SLJIT_UNUSED_ARG(compiler);
+
 	/* Using the values generated by patch_pc_relative_loads. */
 	if (!*first_patch)
 		value = (sljit_si)cpool_start_address[cpool_current_index];
diff --git a/testdata/testinput12 b/testdata/testinput12
index 5d727af..d68551f 100644
--- a/testdata/testinput12
+++ b/testdata/testinput12
@@ -87,4 +87,12 @@ and a couple of things that are different with JIT. --/
 /^12345678abcd/mS++
     12345678abcd
 
+/-- Test pattern compilation --/ 
+
+/(?:a|b|c|d|e)(?R)/S++
+
+/(?:a|b|c|d|e)(?R)(?R)/S++
+
+/(a(?:a|b|c|d|e)b){8,16}/S++
+
 /-- End of testinput12 --/
diff --git a/testdata/testoutput12 b/testdata/testoutput12
index 67ad2c8..7a1cf69 100644
--- a/testdata/testoutput12
+++ b/testdata/testoutput12
@@ -176,4 +176,12 @@ No match, mark = m (JIT)
     12345678abcd
  0: 12345678abcd (JIT)
 
+/-- Test pattern compilation --/ 
+
+/(?:a|b|c|d|e)(?R)/S++
+
+/(?:a|b|c|d|e)(?R)(?R)/S++
+
+/(a(?:a|b|c|d|e)b){8,16}/S++
+
 /-- End of testinput12 --/