Bug#37958 - test main.plugin crash on Mac OS X when selecting from EXAMPLE engine.

This patch contains fixes for two problems:

1. As originally reported, the server crashed on Mac OS X when trying to access
   an EXAMPLE table after the EXAMPLE plugin was installed.

   It turned out that the dynamically loaded EXAMPLE plugin called the
   function hash_earch() from a Mac OS X system library, instead of
   hash_earch() from MySQL's mysys library. Makefile.am in storage/example
   does not include libmysys. So the Mac OS X linker arranged the hash_search()
   function to be linked to the system library when the shared object is
   loaded.

   One possible solution would be to include libmysys into the linkage of
   dynamic plugins. But then we must have a libmysys.so, which must be
   used by the server too. This could have a minimal performance impact,
   but foremost the change seems to bee too risky at the current state of
   MySQL 5.1.

   The selected solution is to rename MySQL's hash_search() to my_hash_search()
   like it has been done before with hash_insert() and hash_reset().

   Since this is the third time, we need to rename a hash_*() function,
   I did renamed all hash_*() functions to my_hash_*().

   To avoid changing a zillion calls to these functions, and announcing
   this to hundreds of developers, I added defines that map the old names
   to the new names.

   This change is in hash.h and hash.c.

2. The other problem was improper implementation of the handlerton-to-plugin
   mapping. We use a fixed-size array to hold a plugin reference for each
   handlerton. On every install of a handler plugin, we allocated a new slot
   of the array. On uninstall we did not free it. After some uninstall/install
   cycles the array overflowed. We did not check for overflow.

   One fix is to check for overflow to stop the crashes.

   Another fix is to free the array slot at uninstall and search for a free slot
   at plugin install.

   This change is in handler.cc.



include/hash.h:
  Bug#37958 - test main.plugin crash on Mac OS X when selecting from EXAMPLE engine.
  Renamed hash_*() functions to my_hash_*().
  Added defines that map old names to new names.
mysys/hash.c:
  Bug#37958 - test main.plugin crash on Mac OS X when selecting from EXAMPLE engine.
  Renamed hash_*() functions to my_hash_*().
sql/handler.cc:
  Bug#37958 - test main.plugin crash on Mac OS X when selecting from EXAMPLE engine.
  Protect against a failing ha_initialize_handlerton() in ha_finalize_handlerton().
  Free hton2plugin slot on uninstall of a handler plugin.
  Reuse freed slost of the hton2plugin array.
  Protect against array overrun.

1 files changed, 63 insertions, 25 deletions

diff --git a/include/hash.h b/include/hash.h
index 4ca8dc0e8bf..1f094d48585 100644
--- a/include/hash.h
+++ b/include/hash.h
@@ -22,6 +22,40 @@ extern "C" {
 #endif
 
 /*
+  There was a problem on MacOSX with a shared object ha_example.so.
+  It used hash_search(). During build of ha_example.so no libmysys
+  was specified. Since MacOSX had a hash_search() in the system
+  library, it built the shared object so that the dynamic linker
+  linked hash_search() to the system library, which caused a crash
+  when called. To come around this, we renamed hash_search() to
+  my_hash_search(), as we did long ago with hash_insert() and
+  hash_reset(). However, this time we made the move complete with
+  all names. To keep compatibility, we redefine the old names.
+  Since every C and C++ file, that uses HASH, needs to include
+  this file, the change is complete. Both names could be used
+  in the code, but the my_* versions are recommended now.
+*/
+#define hash_get_key    my_hash_get_key
+#define hash_free_key   my_hash_free_key
+#define hash_init       my_hash_init
+#define hash_init2      my_hash_init2
+#define _hash_init      _my_hash_init
+#define hash_free       my_hash_free
+#define hash_reset      my_hash_reset
+#define hash_element    my_hash_element
+#define hash_search     my_hash_search
+#define hash_first      my_hash_first
+#define hash_next       my_hash_next
+#define hash_insert     my_hash_insert
+#define hash_delete     my_hash_delete
+#define hash_update     my_hash_update
+#define hash_replace    my_hash_replace
+#define hash_check      my_hash_check
+#define hash_clear      my_hash_clear
+#define hash_inited     my_hash_inited
+#define hash_init_opt   my_hash_init_opt
+
+/*
   Overhead to store an element in hash
   Can be used to approximate memory consumption for a hash
  */
@@ -30,8 +64,8 @@ extern "C" {
 /* flags for hash_init */
 #define HASH_UNIQUE     1       /* hash_insert fails on duplicate key */
 
-typedef uchar *(*hash_get_key)(const uchar *,size_t*,my_bool);
-typedef void (*hash_free_key)(void *);
+typedef uchar *(*my_hash_get_key)(const uchar *,size_t*,my_bool);
+typedef void (*my_hash_free_key)(void *);
 
 typedef struct st_hash {
   size_t key_offset,key_length;		/* Length of key if const length */
@@ -39,7 +73,7 @@ typedef struct st_hash {
   ulong records;
   uint flags;
   DYNAMIC_ARRAY array;				/* Place for hash_keys */
-  hash_get_key get_key;
+  my_hash_get_key get_key;
   void (*free)(void *);
   CHARSET_INFO *charset;
 } HASH;
@@ -47,30 +81,34 @@ typedef struct st_hash {
 /* A search iterator state */
 typedef uint HASH_SEARCH_STATE;
 
-#define hash_init(A,B,C,D,E,F,G,H) _hash_init(A,0,B,C,D,E,F,G,H CALLER_INFO)
-#define hash_init2(A,B,C,D,E,F,G,H,I) _hash_init(A,B,C,D,E,F,G,H,I CALLER_INFO)
-my_bool _hash_init(HASH *hash, uint growth_size,CHARSET_INFO *charset,
-		   ulong default_array_elements, size_t key_offset,
-		   size_t key_length, hash_get_key get_key,
-		   void (*free_element)(void*), uint flags CALLER_INFO_PROTO);
-void hash_free(HASH *tree);
+#define my_hash_init(A,B,C,D,E,F,G,H) \
+          _my_hash_init(A,0,B,C,D,E,F,G,H CALLER_INFO)
+#define my_hash_init2(A,B,C,D,E,F,G,H,I) \
+          _my_hash_init(A,B,C,D,E,F,G,H,I CALLER_INFO)
+my_bool _my_hash_init(HASH *hash, uint growth_size, CHARSET_INFO *charset,
+                      ulong default_array_elements, size_t key_offset,
+                      size_t key_length, my_hash_get_key get_key,
+                      void (*free_element)(void*),
+                      uint flags CALLER_INFO_PROTO);
+void my_hash_free(HASH *tree);
 void my_hash_reset(HASH *hash);
-uchar *hash_element(HASH *hash,ulong idx);
-uchar *hash_search(const HASH *info, const uchar *key, size_t length);
-uchar *hash_first(const HASH *info, const uchar *key, size_t length,
-                HASH_SEARCH_STATE *state);
-uchar *hash_next(const HASH *info, const uchar *key, size_t length,
-                 HASH_SEARCH_STATE *state);
-my_bool my_hash_insert(HASH *info,const uchar *data);
-my_bool hash_delete(HASH *hash,uchar *record);
-my_bool hash_update(HASH *hash,uchar *record,uchar *old_key,size_t old_key_length);
-void hash_replace(HASH *hash, HASH_SEARCH_STATE *state, uchar *new_row);
-my_bool hash_check(HASH *hash);			/* Only in debug library */
+uchar *my_hash_element(HASH *hash, ulong idx);
+uchar *my_hash_search(const HASH *info, const uchar *key, size_t length);
+uchar *my_hash_first(const HASH *info, const uchar *key, size_t length,
+                     HASH_SEARCH_STATE *state);
+uchar *my_hash_next(const HASH *info, const uchar *key, size_t length,
+                    HASH_SEARCH_STATE *state);
+my_bool my_hash_insert(HASH *info, const uchar *data);
+my_bool my_hash_delete(HASH *hash, uchar *record);
+my_bool my_hash_update(HASH *hash, uchar *record, uchar *old_key,
+                       size_t old_key_length);
+void my_hash_replace(HASH *hash, HASH_SEARCH_STATE *state, uchar *new_row);
+my_bool my_hash_check(HASH *hash); /* Only in debug library */
 
-#define hash_clear(H) bzero((char*) (H),sizeof(*(H)))
-#define hash_inited(H) ((H)->array.buffer != 0)
-#define hash_init_opt(A,B,C,D,E,F,G,H) \
-          (!hash_inited(A) && _hash_init(A,0,B,C,D,E,F,G, H CALLER_INFO))
+#define my_hash_clear(H) bzero((char*) (H), sizeof(*(H)))
+#define my_hash_inited(H) ((H)->array.buffer != 0)
+#define my_hash_init_opt(A,B,C,D,E,F,G,H) \
+          (!my_hash_inited(A) && _my_hash_init(A,0,B,C,D,E,F,G, H CALLER_INFO))
 
 #ifdef	__cplusplus
 }