Bug#12552516 LF_HASH REQUIRES MY_THREAD_INIT()

Before this fix, a thread instrumented for the performance schema,
that would perform file io operations, could crash inside the LF_HASH
implementation, in cases when my_thread_init is not called.

The crash itself has not been reported in 5.5 but similar crashes have
been found in 5.6-based development branches, using LF_HASH for
more instrumentation.

The possibility of a crash in 5.5 is confirmed by code analysis.

The problem is that, when my_thread_init() is not called,
which can happen for threads in storage engines or thirs party code,
my_thread_var is NULL.

Using my_thread_var->stacks_ends_here in mysys/lf_alloc-pin.c is unsafe.

Given that my_thread_var is used:
- only for stacks_ends_here
- only on platform with HAVE_ALLOCA
- only when there is enough room on the stack
and given that the LF_HASH implementation has a fallback
algorythm implemented already when using alloca is not possible,
using my_thread_var->stacks_ends_here is in fact not a strict requirement,
and can be relaxed.

The fix is to:
- test explicitly if my_thread_var is NULL, to account for cases
  when my_thread_init() is not used by the calling thread.
- not use alloca in this case, and rely on the fall back code already in place.
  so that the LF_HASH can be supported even without my_thread_init().

The implementation of mysys/lf_alloc-pin.c has been fixed to support this new usage.
The units tests in unittest/mysys/lf-t.c have been adjusted accordingly.

1 files changed, 27 insertions, 10 deletions

diff --git a/unittest/mysys/lf-t.c b/unittest/mysys/lf-t.c
index 61b7ae08cf5..c04ef358abe 100644
--- a/unittest/mysys/lf-t.c
+++ b/unittest/mysys/lf-t.c
@@ -27,6 +27,8 @@ int32 inserts= 0, N;
 LF_ALLOCATOR lf_allocator;
 LF_HASH lf_hash;
 
+int with_my_thread_init=0;
+
 /*
   pin allocator - alloc and release an element in a loop
 */
@@ -36,7 +38,8 @@ pthread_handler_t test_lf_pinbox(void *arg)
   int32 x= 0;
   LF_PINS *pins;
 
-  my_thread_init();
+  if (with_my_thread_init)
+    my_thread_init();
 
   pins= lf_pinbox_get_pins(&lf_allocator.pinbox);
 
@@ -49,7 +52,10 @@ pthread_handler_t test_lf_pinbox(void *arg)
   pthread_mutex_lock(&mutex);
   if (!--running_threads) pthread_cond_signal(&cond);
   pthread_mutex_unlock(&mutex);
-  my_thread_end();
+
+  if (with_my_thread_init)
+    my_thread_end();
+
   return 0;
 }
 
@@ -68,7 +74,8 @@ pthread_handler_t test_lf_alloc(void *arg)
   int32 x,y= 0;
   LF_PINS *pins;
 
-  my_thread_init();
+  if (with_my_thread_init)
+    my_thread_init();
 
   pins= lf_alloc_get_pins(&lf_allocator);
 
@@ -101,7 +108,9 @@ pthread_handler_t test_lf_alloc(void *arg)
   }
   if (!--running_threads) pthread_cond_signal(&cond);
   pthread_mutex_unlock(&mutex);
-  my_thread_end();
+
+  if (with_my_thread_init)
+    my_thread_end();
   return 0;
 }
 
@@ -112,7 +121,8 @@ pthread_handler_t test_lf_hash(void *arg)
   int32 x,y,z,sum= 0, ins= 0;
   LF_PINS *pins;
 
-  my_thread_init();
+  if (with_my_thread_init)
+    my_thread_init();
 
   pins= lf_hash_get_pins(&lf_hash);
 
@@ -152,14 +162,15 @@ pthread_handler_t test_lf_hash(void *arg)
   }
   if (!--running_threads) pthread_cond_signal(&cond);
   pthread_mutex_unlock(&mutex);
-  my_thread_end();
+  if (with_my_thread_init)
+    my_thread_end();
   return 0;
 }
 
 
 void do_tests()
 {
-  plan(4);
+  plan(7);
 
   lf_alloc_init(&lf_allocator, sizeof(TLA), offsetof(TLA, not_used));
   lf_hash_init(&lf_hash, sizeof(int), LF_HASH_UNIQUE, 0, sizeof(int), 0,
@@ -168,9 +179,15 @@ void do_tests()
   bad= my_atomic_initialize();
   ok(!bad, "my_atomic_initialize() returned %d", bad);
 
-  test_concurrently("lf_pinbox", test_lf_pinbox, N= THREADS, CYCLES);
-  test_concurrently("lf_alloc",  test_lf_alloc,  N= THREADS, CYCLES);
-  test_concurrently("lf_hash",   test_lf_hash,   N= THREADS, CYCLES/10);
+  with_my_thread_init= 1;
+  test_concurrently("lf_pinbox (with my_thread_init)", test_lf_pinbox, N= THREADS, CYCLES);
+  test_concurrently("lf_alloc (with my_thread_init)",  test_lf_alloc,  N= THREADS, CYCLES);
+  test_concurrently("lf_hash (with my_thread_init)",   test_lf_hash,   N= THREADS, CYCLES/10);
+
+  with_my_thread_init= 0;
+  test_concurrently("lf_pinbox (without my_thread_init)", test_lf_pinbox, N= THREADS, CYCLES);
+  test_concurrently("lf_alloc (without my_thread_init)",  test_lf_alloc,  N= THREADS, CYCLES);
+  test_concurrently("lf_hash (without my_thread_init)",   test_lf_hash,   N= THREADS, CYCLES/10);
 
   lf_hash_destroy(&lf_hash);
   lf_alloc_destroy(&lf_allocator);