Merge 10.3 into 10.4

13 files changed, 112 insertions, 70 deletions

diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc
index bd22e78c5b2..1f12093b634 100644
--- a/storage/innobase/fil/fil0crypt.cc
+++ b/storage/innobase/fil/fil0crypt.cc
@@ -2447,7 +2447,7 @@ static void fil_crypt_rotation_list_fill()
 	     space != NULL;
 	     space = UT_LIST_GET_NEXT(space_list, space)) {
 		if (space->purpose != FIL_TYPE_TABLESPACE
-		    || space->is_in_rotation_list()
+		    || space->is_in_rotation_list
 		    || space->is_stopping()
 		    || UT_LIST_GET_LEN(space->chain) == 0) {
 			continue;
@@ -2494,6 +2494,7 @@ static void fil_crypt_rotation_list_fill()
 		}
 
 		fil_system.rotation_list.push_back(*space);
+		space->is_in_rotation_list = true;
 	}
 }
 
diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc
index a41a95025fd..16ff86c3f91 100644
--- a/storage/innobase/fil/fil0fil.cc
+++ b/storage/innobase/fil/fil0fil.cc
@@ -693,7 +693,7 @@ static void fil_flush_low(fil_space_t* space, bool metadata = false)
 
 		/* No need to flush. User has explicitly disabled
 		buffering. */
-		ut_ad(!space->is_in_unflushed_spaces());
+		ut_ad(!space->is_in_unflushed_spaces);
 		ut_ad(fil_space_is_flushed(space));
 		ut_ad(space->n_pending_flushes == 0);
 
@@ -757,10 +757,11 @@ static void fil_flush_low(fil_space_t* space, bool metadata = false)
 skip_flush:
 #endif /* _WIN32 */
 		if (!node->needs_flush) {
-			if (space->is_in_unflushed_spaces()
+			if (space->is_in_unflushed_spaces
 			    && fil_space_is_flushed(space)) {
 
 				fil_system.unflushed_spaces.remove(*space);
+				space->is_in_unflushed_spaces = false;
 			}
 		}
 
@@ -1048,13 +1049,14 @@ fil_node_close_to_free(
 
 		if (fil_buffering_disabled(space)) {
 
-			ut_ad(!space->is_in_unflushed_spaces());
+			ut_ad(!space->is_in_unflushed_spaces);
 			ut_ad(fil_space_is_flushed(space));
 
-		} else if (space->is_in_unflushed_spaces()
+		} else if (space->is_in_unflushed_spaces
 			   && fil_space_is_flushed(space)) {
 
 			fil_system.unflushed_spaces.remove(*space);
+			space->is_in_unflushed_spaces = false;
 		}
 
 		node->close();
@@ -1074,14 +1076,16 @@ fil_space_detach(
 
 	HASH_DELETE(fil_space_t, hash, fil_system.spaces, space->id, space);
 
-	if (space->is_in_unflushed_spaces()) {
+	if (space->is_in_unflushed_spaces) {
 
 		ut_ad(!fil_buffering_disabled(space));
 		fil_system.unflushed_spaces.remove(*space);
+		space->is_in_unflushed_spaces = false;
 	}
 
-	if (space->is_in_rotation_list()) {
+	if (space->is_in_rotation_list) {
 		fil_system.rotation_list.remove(*space);
+		space->is_in_rotation_list = false;
 	}
 
 	UT_LIST_REMOVE(fil_system.space_list, space);
@@ -1297,6 +1301,7 @@ fil_space_create(
 		/* Key rotation is not enabled, need to inform background
 		encryption threads. */
 		fil_system.rotation_list.push_back(*space);
+		space->is_in_rotation_list = true;
 		mutex_exit(&fil_system.mutex);
 		os_event_set(fil_crypt_threads_event);
 	} else {
@@ -4023,13 +4028,14 @@ fil_node_complete_io(fil_node_t* node, const IORequest& type)
 			/* We don't need to keep track of unflushed
 			changes as user has explicitly disabled
 			buffering. */
-			ut_ad(!node->space->is_in_unflushed_spaces());
+			ut_ad(!node->space->is_in_unflushed_spaces);
 			ut_ad(node->needs_flush == false);
 
 		} else {
 			node->needs_flush = true;
 
-			if (!node->space->is_in_unflushed_spaces()) {
+			if (!node->space->is_in_unflushed_spaces) {
+				node->space->is_in_unflushed_spaces = true;
 				fil_system.unflushed_spaces.push_front(
 					*node->space);
 			}
@@ -4503,7 +4509,7 @@ fil_flush_file_spaces(
 
 	n_space_ids = 0;
 
-	for (intrusive::list<fil_space_t, unflushed_spaces_tag_t>::iterator it
+	for (sized_ilist<fil_space_t, unflushed_spaces_tag_t>::iterator it
 	     = fil_system.unflushed_spaces.begin(),
 	     end = fil_system.unflushed_spaces.end();
 	     it != end; ++it) {
@@ -5016,7 +5022,8 @@ fil_space_remove_from_keyrotation(fil_space_t* space)
 	ut_ad(mutex_own(&fil_system.mutex));
 	ut_ad(space);
 
-	if (!space->referenced() && space->is_in_rotation_list()) {
+	if (!space->referenced() && space->is_in_rotation_list) {
+		space->is_in_rotation_list = false;
 		ut_a(!fil_system.rotation_list.empty());
 		fil_system.rotation_list.remove(*space);
 	}
@@ -5046,8 +5053,8 @@ fil_space_t *fil_system_t::keyrotate_next(fil_space_t *prev_space,
   don't remove the last processed tablespace from the rotation list. */
   const bool remove= (!recheck || prev_space->crypt_data) &&
                      !key_version == !srv_encrypt_tables;
-  intrusive::list<fil_space_t, rotation_list_tag_t>::iterator it=
-      prev_space == NULL ? fil_system.rotation_list.end() : prev_space;
+  sized_ilist<fil_space_t, rotation_list_tag_t>::iterator it=
+    prev_space ? prev_space : fil_system.rotation_list.end();
 
   if (it == fil_system.rotation_list.end())
     it= fil_system.rotation_list.begin();
@@ -5124,24 +5131,3 @@ fil_space_found_by_id(
 	mutex_exit(&fil_system.mutex);
 	return space;
 }
-
-/** Checks that this tablespace in a list of unflushed tablespaces.
-@return true if in a list */
-bool fil_space_t::is_in_unflushed_spaces() const
-{
-  ut_ad(mutex_own(&fil_system.mutex));
-
-  return static_cast<const intrusive::list_node<unflushed_spaces_tag_t> *>(
-             this)
-      ->next;
-}
-
-/** Checks that this tablespace needs key rotation.
-@return true if in a rotation list */
-bool fil_space_t::is_in_rotation_list() const
-{
-  ut_ad(mutex_own(&fil_system.mutex));
-
-  return static_cast<const intrusive::list_node<rotation_list_tag_t> *>(this)
-      ->next;
-}
diff --git a/storage/innobase/include/dyn0buf.h b/storage/innobase/include/dyn0buf.h
index f66c7e3d405..be263e04e15 100644
--- a/storage/innobase/include/dyn0buf.h
+++ b/storage/innobase/include/dyn0buf.h
@@ -29,7 +29,7 @@ Created 2013-03-16 Sunny Bains
 
 #include "mem0mem.h"
 #include "dyn0types.h"
-#include "intrusive_list.h"
+#include "ilist.h"
 
 
 /** Class that manages dynamic buffers. It uses a UT_LIST of
@@ -43,9 +43,9 @@ class mtr_buf_t {
 public:
 	/** SIZE - sizeof(m_node) + sizeof(m_used) */
 	enum { MAX_DATA_SIZE = DYN_ARRAY_DATA_SIZE
-	       - sizeof(intrusive::list_node<>) + sizeof(uint32_t) };
+	       - sizeof(ilist_node<>) + sizeof(uint32_t) };
 
-	class block_t : public intrusive::list_node<> {
+	class block_t : public ilist_node<> {
 	public:
 
 		block_t()
@@ -160,7 +160,7 @@ public:
 		friend class mtr_buf_t;
 	};
 
-	typedef intrusive::list<block_t> list_t;
+	typedef sized_ilist<block_t> list_t;
 
 	/** Default constructor */
 	mtr_buf_t()
diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h
index a24b2a43fda..e78c9587325 100644
--- a/storage/innobase/include/fil0fil.h
+++ b/storage/innobase/include/fil0fil.h
@@ -33,7 +33,7 @@ Created 10/25/1995 Heikki Tuuri
 
 #include "log0recv.h"
 #include "dict0types.h"
-#include "intrusive_list.h"
+#include "ilist.h"
 #ifdef UNIV_LINUX
 # include <set>
 #endif
@@ -82,8 +82,8 @@ struct fil_node_t;
 
 /** Tablespace or log data space */
 #ifndef UNIV_INNOCHECKSUM
-struct fil_space_t : intrusive::list_node<unflushed_spaces_tag_t>,
-                     intrusive::list_node<rotation_list_tag_t>
+struct fil_space_t : ilist_node<unflushed_spaces_tag_t>,
+                     ilist_node<rotation_list_tag_t>
 #else
 struct fil_space_t
 #endif
@@ -160,18 +160,18 @@ struct fil_space_t
 	UT_LIST_NODE_T(fil_space_t) named_spaces;
 				/*!< list of spaces for which MLOG_FILE_NAME
 				records have been issued */
-	/** Checks that this tablespace in a list of unflushed tablespaces.
-	@return true if in a list */
-	bool is_in_unflushed_spaces() const;
 	UT_LIST_NODE_T(fil_space_t) space_list;
 				/*!< list of all spaces */
-	/** Checks that this tablespace needs key rotation.
-	@return true if in a rotation list */
-	bool is_in_rotation_list() const;
 
 	/** MariaDB encryption data */
 	fil_space_crypt_t* crypt_data;
 
+	/** Checks that this tablespace in a list of unflushed tablespaces. */
+	bool is_in_unflushed_spaces;
+
+	/** Checks that this tablespace needs key rotation. */
+	bool is_in_rotation_list;
+
 	/** True if the device this filespace is on supports atomic writes */
 	bool		atomic_write_supported;
 
@@ -933,7 +933,7 @@ public:
 					not put to this list: they are opened
 					after the startup, and kept open until
 					shutdown */
-	intrusive::list<fil_space_t, unflushed_spaces_tag_t> unflushed_spaces;
+	sized_ilist<fil_space_t, unflushed_spaces_tag_t> unflushed_spaces;
 					/*!< list of those
 					tablespaces whose files contain
 					unflushed writes; those spaces have
@@ -954,7 +954,7 @@ public:
 					record has been written since
 					the latest redo log checkpoint.
 					Protected only by log_sys.mutex. */
-	intrusive::list<fil_space_t, rotation_list_tag_t> rotation_list;
+	ilist<fil_space_t, rotation_list_tag_t> rotation_list;
 					/*!< list of all file spaces needing
 					key rotation.*/
 
diff --git a/storage/maria/ma_recovery.c b/storage/maria/ma_recovery.c
index 27c941705b7..8f108e3f03f 100644
--- a/storage/maria/ma_recovery.c
+++ b/storage/maria/ma_recovery.c
@@ -956,6 +956,7 @@ prototype_redo_exec_hook(REDO_RENAME_TABLE)
   char *old_name, *new_name;
   int error= 1;
   MARIA_HA *info= NULL;
+  my_bool from_table_is_crashed= 0;
   DBUG_ENTER("exec_REDO_LOGREC_REDO_RENAME_TABLE");
 
   if (skip_DDLs)
@@ -1025,15 +1026,15 @@ prototype_redo_exec_hook(REDO_RENAME_TABLE)
     }
     if (maria_is_crashed(info))
     {
-      tprint(tracef, ", is crashed, can't rename it");
-      ALERT_USER();
-      goto end;
+      tprint(tracef, "is crashed, can't be used for rename ; new-name table ");
+      from_table_is_crashed= 1;
     }
     if (close_one_table(info->s->open_file_name.str, rec->lsn) ||
         maria_close(info))
       goto end;
     info= NULL;
-    tprint(tracef, ", is ok for renaming; new-name table ");
+    if (!from_table_is_crashed)
+      tprint(tracef, "is ok for renaming; new-name table ");
   }
   else /* one or two files absent, or header corrupted... */
   {
@@ -1098,11 +1099,19 @@ prototype_redo_exec_hook(REDO_RENAME_TABLE)
       goto end;
     info= NULL;
     /* abnormal situation */
-    tprint(tracef, ", exists but is older than record, can't rename it");
+    tprint(tracef, "exists but is older than record, can't rename it");
     goto end;
   }
   else /* one or two files absent, or header corrupted... */
-    tprint(tracef, ", can't be opened, probably does not exist");
+    tprint(tracef, "can't be opened, probably does not exist");
+
+  if (from_table_is_crashed)
+  {
+    eprint(tracef, "Aborting rename as old table was crashed");
+    ALERT_USER();
+    goto end;
+  }
+
   tprint(tracef, ", renaming '%s'", old_name);
   if (maria_rename(old_name, new_name))
   {
diff --git a/storage/maria/ma_write.c b/storage/maria/ma_write.c
index 07be8333794..86770749a45 100644
--- a/storage/maria/ma_write.c
+++ b/storage/maria/ma_write.c
@@ -335,12 +335,6 @@ err:
       my_errno == HA_ERR_NULL_IN_SPATIAL ||
       my_errno == HA_ERR_OUT_OF_MEM)
   {
-    if (info->bulk_insert)
-    {
-      uint j;
-      for (j=0 ; j < share->base.keys ; j++)
-        maria_flush_bulk_insert(info, j);
-    }
     info->errkey= i < share->base.keys ? (int) i : -1;
     /*
       We delete keys in the reverse order of insertion. This is the order that
@@ -366,6 +360,7 @@ err:
         {
           if (_ma_ft_del(info,i,buff,record,filepos))
 	  {
+            fatal_error= 1;
 	    if (local_lock_tree)
 	      mysql_rwlock_unlock(&keyinfo->root_lock);
             break;
@@ -380,6 +375,7 @@ err:
                                                       filepos,
                                                       info->trn->trid)))
 	  {
+            fatal_error= 1;
 	    if (local_lock_tree)
 	      mysql_rwlock_unlock(&keyinfo->root_lock);
 	    break;
@@ -399,6 +395,13 @@ err:
       fatal_error= 1;
   }
 
+  if (info->bulk_insert)
+  {
+    uint j;
+    for (j=0 ; j < share->base.keys ; j++)
+      maria_flush_bulk_insert(info, j);
+  }
+
   if (fatal_error)
   {
     maria_print_error(info->s, HA_ERR_CRASHED);
diff --git a/storage/mroonga/vendor/groonga/lib/db.c b/storage/mroonga/vendor/groonga/lib/db.c
index cba22aa0e64..f3769f9aa4c 100644
--- a/storage/mroonga/vendor/groonga/lib/db.c
+++ b/storage/mroonga/vendor/groonga/lib/db.c
@@ -445,6 +445,9 @@ grn_db_close(grn_ctx *ctx, grn_obj *db)
 
   ctx_used_db = ctx->impl && ctx->impl->db == db;
   if (ctx_used_db) {
+#ifdef GRN_WITH_MECAB
+    grn_db_fin_mecab_tokenizer(ctx);
+#endif
     grn_ctx_loader_clear(ctx);
     if (ctx->impl->parser) {
       grn_expr_parser_close(ctx);
diff --git a/storage/mroonga/vendor/groonga/lib/grn_tokenizers.h b/storage/mroonga/vendor/groonga/lib/grn_tokenizers.h
index e90dbfc0b31..81ac2ab6c46 100644
--- a/storage/mroonga/vendor/groonga/lib/grn_tokenizers.h
+++ b/storage/mroonga/vendor/groonga/lib/grn_tokenizers.h
@@ -30,6 +30,7 @@ grn_rc grn_tokenizers_init(void);
 grn_rc grn_tokenizers_fin(void);
 
 grn_rc grn_db_init_mecab_tokenizer(grn_ctx *ctx);
+void grn_db_fin_mecab_tokenizer(grn_ctx *ctx);
 grn_rc grn_db_init_builtin_tokenizers(grn_ctx *ctx);
 
 #ifdef __cplusplus
diff --git a/storage/mroonga/vendor/groonga/lib/tokenizers.c b/storage/mroonga/vendor/groonga/lib/tokenizers.c
index 11f274e72db..3daacce7ef9 100644
--- a/storage/mroonga/vendor/groonga/lib/tokenizers.c
+++ b/storage/mroonga/vendor/groonga/lib/tokenizers.c
@@ -797,6 +797,36 @@ grn_db_init_mecab_tokenizer(grn_ctx *ctx)
   }
 }
 
+void
+grn_db_fin_mecab_tokenizer(grn_ctx *ctx)
+{
+  switch (GRN_CTX_GET_ENCODING(ctx)) {
+  case GRN_ENC_EUC_JP :
+  case GRN_ENC_UTF8 :
+  case GRN_ENC_SJIS :
+#if defined(GRN_EMBEDDED) && defined(GRN_WITH_MECAB)
+    {
+      GRN_PLUGIN_DECLARE_FUNCTIONS(tokenizers_mecab);
+      GRN_PLUGIN_IMPL_NAME_TAGGED(fin, tokenizers_mecab)(ctx);
+    }
+#else /* defined(GRN_EMBEDDED) && defined(GRN_WITH_MECAB) */
+    {
+      const char *mecab_plugin_name = "tokenizers/mecab";
+      char *path;
+      path = grn_plugin_find_path(ctx, mecab_plugin_name);
+      if (path) {
+        GRN_FREE(path);
+        grn_plugin_unregister(ctx, mecab_plugin_name);
+      }
+    }
+#endif /* defined(GRN_EMBEDDED) && defined(GRN_WITH_MECAB) */
+    break;
+  default :
+    break;
+  }
+  return;
+}
+
 #define DEF_TOKENIZER(name, init, next, fin, vars)\
   (grn_proc_create(ctx, (name), (sizeof(name) - 1),\
                    GRN_PROC_TOKENIZER, (init), (next), (fin), 3, (vars)))
diff --git a/storage/mroonga/vendor/groonga/plugins/tokenizers/mecab.c b/storage/mroonga/vendor/groonga/plugins/tokenizers/mecab.c
index 3dd969a89c5..cabf2c94e53 100644
--- a/storage/mroonga/vendor/groonga/plugins/tokenizers/mecab.c
+++ b/storage/mroonga/vendor/groonga/plugins/tokenizers/mecab.c
@@ -31,6 +31,7 @@
 #include <string.h>
 #include <ctype.h>
 
+static unsigned int sole_mecab_init_counter = 0;
 static mecab_t *sole_mecab = NULL;
 static grn_plugin_mutex *sole_mecab_mutex = NULL;
 static grn_encoding sole_mecab_encoding = GRN_ENC_NONE;
@@ -563,6 +564,11 @@ check_mecab_dictionary_encoding(grn_ctx *ctx)
 grn_rc
 GRN_PLUGIN_INIT(grn_ctx *ctx)
 {
+  ++sole_mecab_init_counter;
+  if (sole_mecab_init_counter > 1)
+  {
+    return GRN_SUCCESS;
+  }
   {
     char env[GRN_ENV_BUFFER_SIZE];
 
@@ -636,6 +642,11 @@ GRN_PLUGIN_REGISTER(grn_ctx *ctx)
 grn_rc
 GRN_PLUGIN_FIN(grn_ctx *ctx)
 {
+  --sole_mecab_init_counter;
+  if (sole_mecab_init_counter > 0)
+  {
+    return GRN_SUCCESS;
+  }
   if (sole_mecab) {
     mecab_destroy(sole_mecab);
     sole_mecab = NULL;
diff --git a/storage/myisam/mi_check.c b/storage/myisam/mi_check.c
index 3f3c60a4249..a0cdd37b302 100644
--- a/storage/myisam/mi_check.c
+++ b/storage/myisam/mi_check.c
@@ -1587,6 +1587,8 @@ int mi_repair(HA_CHECK *param, register MI_INFO *info,
   sort_param.filepos=new_header_length;
   param->read_cache.end_of_file=sort_info.filelength=
     mysql_file_seek(info->dfile, 0L, MY_SEEK_END, MYF(0));
+  if (info->state->data_file_length == 0)
+    info->state->data_file_length= sort_info.filelength;
   sort_info.dupp=0;
   sort_param.fix_datafile= (my_bool) (! rep_quick);
   sort_param.master=1;
@@ -2292,6 +2294,8 @@ int mi_repair_by_sort(HA_CHECK *param, register MI_INFO *info,
   sort_info.buff=0;
   param->read_cache.end_of_file=sort_info.filelength=
     mysql_file_seek(param->read_cache.file, 0L, MY_SEEK_END, MYF(0));
+  if (info->state->data_file_length == 0)
+    info->state->data_file_length= sort_info.filelength;
 
   sort_param.wordlist=NULL;
   init_alloc_root(&sort_param.wordroot, "sort", FTPARSER_MEMROOT_ALLOC_SIZE, 0,
@@ -2759,6 +2763,8 @@ int mi_repair_parallel(HA_CHECK *param, register MI_INFO *info,
   sort_info.buff=0;
   param->read_cache.end_of_file=sort_info.filelength=
     mysql_file_seek(param->read_cache.file, 0L, MY_SEEK_END, MYF(0));
+  if (info->state->data_file_length == 0)
+    info->state->data_file_length= sort_info.filelength;
 
   if (share->data_file_type == DYNAMIC_RECORD)
     rec_length=MY_MAX(share->base.min_pack_length+1,share->base.min_block_length);
diff --git a/storage/rocksdb/build_rocksdb.cmake b/storage/rocksdb/build_rocksdb.cmake
index 1a3b2076c82..2e939587811 100644
--- a/storage/rocksdb/build_rocksdb.cmake
+++ b/storage/rocksdb/build_rocksdb.cmake
@@ -67,7 +67,7 @@ if(SNAPPY_FOUND  AND (NOT WITH_ROCKSDB_snappy STREQUAL "OFF"))
 endif()
 
 include(CheckFunctionExists)
-if(ZSTD_FOUND AND (NOT WITH_ROCKSDB_zstd STREQUAL "OFF"))
+if(ZSTD_FOUND AND (NOT WITH_ROCKSDB_ZSTD STREQUAL "OFF"))
   SET(CMAKE_REQUIRED_LIBRARIES zstd)
   CHECK_FUNCTION_EXISTS(ZDICT_trainFromBuffer ZSTD_VALID)
   UNSET(CMAKE_REQUIRED_LIBRARIES)
diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc
index 1f4b53029b8..52c93c37806 100644
--- a/storage/rocksdb/ha_rocksdb.cc
+++ b/storage/rocksdb/ha_rocksdb.cc
@@ -114,7 +114,6 @@ int thd_binlog_format(const MYSQL_THD thd);
 bool thd_binlog_filter_ok(const MYSQL_THD thd);
 }
 
-MYSQL_PLUGIN_IMPORT bool my_disable_leak_check;
 extern my_bool opt_core_file;
 
 // Needed in rocksdb_init_func
@@ -5716,13 +5715,6 @@ static int rocksdb_init_func(void *const p) {
   }
 #endif
 
-  /**
-    Rocksdb does not always shutdown its threads, when
-    plugin is shut down. Disable server's leak check
-    at exit to avoid crash.
-  */
-  my_disable_leak_check = true;
-
   err = my_error_register(rdb_get_error_messages, HA_ERR_ROCKSDB_FIRST,
                           HA_ERR_ROCKSDB_LAST);
   if (err != 0) {