5.6.38

11 files changed, 192 insertions, 42 deletions

diff --git a/storage/innobase/dict/dict0stats_bg.cc b/storage/innobase/dict/dict0stats_bg.cc
index 975c8a50803..bc8fbe6ece4 100644
--- a/storage/innobase/dict/dict0stats_bg.cc
+++ b/storage/innobase/dict/dict0stats_bg.cc
@@ -195,7 +195,7 @@ dict_stats_wait_bg_to_stop_using_table(
 				unlocking/locking the data dict */
 {
 	while (!dict_stats_stop_bg(table)) {
-		DICT_STATS_BG_YIELD(trx);
+		DICT_BG_YIELD(trx);
 	}
 }
 
diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc
index a0f0fab5566..a29e046eb84 100644
--- a/storage/innobase/fts/fts0fts.cc
+++ b/storage/innobase/fts/fts0fts.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2011, 2017, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -25,6 +25,7 @@ Full Text Search interface
 #include "row0mysql.h"
 #include "row0upd.h"
 #include "dict0types.h"
+#include "dict0stats_bg.h"
 #include "row0sel.h"
 
 #include "fts0fts.h"
@@ -867,18 +868,37 @@ fts_drop_index(
 
 			err = fts_drop_index_tables(trx, index);
 
-			fts_free(table);
-
+			for(;;) {
+				bool retry = false;
+				if (index->index_fts_syncing) {
+					retry = true;
+				}
+				if (!retry){
+					fts_free(table);
+					break;
+				}
+				DICT_BG_YIELD(trx);
+			}
 			return(err);
 		}
 
-		current_doc_id = table->fts->cache->next_doc_id;
-		first_doc_id = table->fts->cache->first_doc_id;
-		fts_cache_clear(table->fts->cache);
-		fts_cache_destroy(table->fts->cache);
-		table->fts->cache = fts_cache_create(table);
-		table->fts->cache->next_doc_id = current_doc_id;
-		table->fts->cache->first_doc_id = first_doc_id;
+		for(;;) {
+			bool retry = false;
+			if (index->index_fts_syncing) {
+				retry = true;
+			}
+			if (!retry){
+				current_doc_id = table->fts->cache->next_doc_id;
+				first_doc_id = table->fts->cache->first_doc_id;
+				fts_cache_clear(table->fts->cache);
+				fts_cache_destroy(table->fts->cache);
+				table->fts->cache = fts_cache_create(table);
+				table->fts->cache->next_doc_id = current_doc_id;
+				table->fts->cache->first_doc_id = first_doc_id;
+				break;
+			}
+			DICT_BG_YIELD(trx);
+		}
 	} else {
 		fts_cache_t*            cache = table->fts->cache;
 		fts_index_cache_t*      index_cache;
@@ -888,9 +908,17 @@ fts_drop_index(
 		index_cache = fts_find_index_cache(cache, index);
 
 		if (index_cache != NULL) {
-			if (index_cache->words) {
-				fts_words_free(index_cache->words);
-				rbt_free(index_cache->words);
+			for(;;) {
+				bool retry = false;
+				if (index->index_fts_syncing) {
+					retry = true;
+				}
+				if (!retry && index_cache->words) {
+					fts_words_free(index_cache->words);
+					rbt_free(index_cache->words);
+					break;
+				}
+				DICT_BG_YIELD(trx);
 			}
 
 			ib_vector_remove(cache->indexes, *(void**) index_cache);
@@ -4611,10 +4639,16 @@ begin_sync:
 		index_cache = static_cast<fts_index_cache_t*>(
 			ib_vector_get(cache->indexes, i));
 
-		if (index_cache->index->to_be_dropped) {
+		if (index_cache->index->to_be_dropped
+		   || index_cache->index->table->to_be_dropped) {
 			continue;
 		}
 
+		index_cache->index->index_fts_syncing = true;
+		DBUG_EXECUTE_IF("fts_instrument_sync_sleep_drop_waits",
+				os_thread_sleep(10000000);
+				);
+
 		error = fts_sync_index(sync, index_cache);
 
 		if (error != DB_SUCCESS && !sync->interrupted) {
@@ -4647,11 +4681,33 @@ begin_sync:
 end_sync:
 	if (error == DB_SUCCESS && !sync->interrupted) {
 		error = fts_sync_commit(sync);
+		if (error == DB_SUCCESS) {
+			for (i = 0; i < ib_vector_size(cache->indexes); ++i) {
+				fts_index_cache_t*      index_cache;
+				index_cache = static_cast<fts_index_cache_t*>(
+					ib_vector_get(cache->indexes, i));
+				if (index_cache->index->index_fts_syncing) {
+					index_cache->index->index_fts_syncing
+								= false;
+				}
+			}
+		}
 	}  else {
 		fts_sync_rollback(sync);
 	}
 
 	rw_lock_x_lock(&cache->lock);
+	/* Clear fts syncing flags of any indexes incase sync is
+	interrupeted */
+	for (i = 0; i < ib_vector_size(cache->indexes); ++i) {
+		fts_index_cache_t*      index_cache;
+		index_cache = static_cast<fts_index_cache_t*>(
+                      ib_vector_get(cache->indexes, i));
+		if (index_cache->index->index_fts_syncing == true) {
+			index_cache->index->index_fts_syncing = false;
+                  }
+	}
+
 	sync->interrupted = false;
 	sync->in_progress = false;
 	os_event_set(sync->event);
diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc
index 19098dc00ef..0a9728abe1d 100644
--- a/storage/innobase/fts/fts0opt.cc
+++ b/storage/innobase/fts/fts0opt.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2007, 2017, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -2970,13 +2970,6 @@ fts_optimize_sync_table(
 {
 	dict_table_t*   table = NULL;
 
-	/* Prevent DROP INDEX etc. from running when we are syncing
-	cache in background. */
-	if (!rw_lock_s_lock_nowait(&dict_operation_lock, __FILE__, __LINE__)) {
-		/* Exit when fail to get dict operation lock. */
-		return;
-	}
-
 	table = dict_table_open_on_id(table_id, FALSE, DICT_TABLE_OP_NORMAL);
 
 	if (table) {
@@ -2986,8 +2979,6 @@ fts_optimize_sync_table(
 
 		dict_table_close(table, FALSE, FALSE);
 	}
-
-	rw_lock_s_unlock(&dict_operation_lock);
 }
 
 /**********************************************************************//**
diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc
index 2c44a21a8f2..fd1199ac434 100644
--- a/storage/innobase/fts/fts0que.cc
+++ b/storage/innobase/fts/fts0que.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2007, 2017, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -3632,6 +3632,11 @@ fts_query_free(
 		fts_doc_ids_free(query->deleted);
 	}
 
+	if (query->intersection) {
+		fts_query_free_doc_ids(query, query->intersection);
+		query->intersection = NULL;
+	}
+
 	if (query->doc_ids) {
 		fts_query_free_doc_ids(query, query->doc_ids);
 	}
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index 0c02c6808ff..142f1b94a1e 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -15834,7 +15834,7 @@ buffer_pool_load_now(
 	const void*			save)	/*!< in: immediate result from
 						check function */
 {
-	if (*(my_bool*) save) {
+	if (*(my_bool*) save && !srv_read_only_mode) {
 		buf_load_start();
 	}
 }
diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc
index e772208ed72..437d38e29de 100644
--- a/storage/innobase/handler/handler0alter.cc
+++ b/storage/innobase/handler/handler0alter.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2005, 2017, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -5607,7 +5607,47 @@ ha_innobase::commit_inplace_alter_table(
 			break;
 		}
 
-		DICT_STATS_BG_YIELD(trx);
+		DICT_BG_YIELD(trx);
+	}
+
+	/* Make a concurrent Drop fts Index to wait until sync of that
+	fts index is happening in the background */
+	for (;;) {
+		bool    retry = false;
+
+		for (inplace_alter_handler_ctx** pctx = ctx_array;
+		    *pctx; pctx++) {
+			int count =0;
+			ha_innobase_inplace_ctx*        ctx
+				= static_cast<ha_innobase_inplace_ctx*>(*pctx);
+			DBUG_ASSERT(new_clustered == ctx->need_rebuild());
+
+			if (dict_fts_index_syncing(ctx->old_table)) {
+				count++;
+				if (count == 100) {
+					fprintf(stderr,
+					"Drop index waiting for background sync"
+					"to finish\n");
+				}
+				retry = true;
+			}
+
+			if (new_clustered && dict_fts_index_syncing(ctx->new_table)) {
+				count++;
+				if (count == 100) {
+					fprintf(stderr,
+                                        "Drop index waiting for background sync"
+                                        "to finish\n");
+				}
+				retry = true;
+			}
+		}
+
+		 if (!retry) {
+			 break;
+		}
+
+		DICT_BG_YIELD(trx);
 	}
 
 	/* Apply the changes to the data dictionary tables, for all
@@ -5923,8 +5963,13 @@ foreign_fail:
 
 		ut_d(dict_table_check_for_dup_indexes(
 			     ctx->new_table, CHECK_ABORTED_OK));
-		ut_a(fts_check_cached_index(ctx->new_table));
 
+#ifdef UNIV_DEBUG
+		if (!(ctx->new_table->fts != NULL
+			&& ctx->new_table->fts->cache->sync->in_progress)) {
+			ut_a(fts_check_cached_index(ctx->new_table));
+		}
+#endif
 		if (new_clustered) {
 			/* Since the table has been rebuilt, we remove
 			all persistent statistics corresponding to the
diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic
index 6fc6098f3d9..9a53dae10ab 100644
--- a/storage/innobase/include/dict0dict.ic
+++ b/storage/innobase/include/dict0dict.ic
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -910,6 +910,27 @@ dict_table_x_lock_indexes(
 }
 
 /*********************************************************************//**
+Returns true if the particular FTS index in the table is still syncing
+in the background, false otherwise.
+@param [in] table      Table containing FTS index
+@return True if sync of fts index is still going in the background  */
+UNIV_INLINE
+bool
+dict_fts_index_syncing(
+	dict_table_t*   table)
+{
+	 dict_index_t*   index;
+
+	for (index = dict_table_get_first_index(table);
+	    index != NULL;
+	    index = dict_table_get_next_index(index)) {
+		if (index->index_fts_syncing) {
+			 return(true);
+		}
+	}
+	return(false);
+}
+/*********************************************************************//**
 Release the exclusive locks on all index tree. */
 UNIV_INLINE
 void
diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h
index 4b49976642d..1f66bc02b5e 100644
--- a/storage/innobase/include/dict0mem.h
+++ b/storage/innobase/include/dict0mem.h
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2012, Facebook Inc.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -606,6 +606,8 @@ struct dict_index_t{
 				dict_sys->mutex. Other changes are
 				protected by index->lock. */
 	dict_field_t*	fields;	/*!< array of field descriptions */
+	bool            index_fts_syncing;/*!< Whether the fts index is
+					still syncing in the background */
 #ifndef UNIV_HOTBACKUP
 	UT_LIST_NODE_T(dict_index_t)
 			indexes;/*!< list of indexes of the table */
diff --git a/storage/innobase/include/dict0stats_bg.h b/storage/innobase/include/dict0stats_bg.h
index 82cd2b468b9..bf77cd125fb 100644
--- a/storage/innobase/include/dict0stats_bg.h
+++ b/storage/innobase/include/dict0stats_bg.h
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2012, 2017, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -59,7 +59,7 @@ dict_stats_recalc_pool_del(
 /** Yield the data dictionary latch when waiting
 for the background thread to stop accessing a table.
 @param trx	transaction holding the data dictionary locks */
-#define DICT_STATS_BG_YIELD(trx)	do {	\
+#define DICT_BG_YIELD(trx)	do {	\
 	row_mysql_unlock_data_dictionary(trx);	\
 	os_thread_sleep(250000);		\
 	row_mysql_lock_data_dictionary(trx);	\
diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc
index 09c20911ec9..8d2d9e1c76b 100644
--- a/storage/innobase/row/row0mysql.cc
+++ b/storage/innobase/row/row0mysql.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -56,6 +56,7 @@ Created 9/17/2000 Heikki Tuuri
 #include "log0log.h"
 #include "btr0sea.h"
 #include "fil0fil.h"
+#include "srv0srv.h"
 #include "ibuf0ibuf.h"
 #include "fts0fts.h"
 #include "fts0types.h"
@@ -3870,6 +3871,16 @@ row_drop_table_for_mysql(
 			ut_ad(!table->fts->add_wq);
 			ut_ad(lock_trx_has_sys_table_locks(trx) == 0);
 
+			for (;;) {
+				bool retry = false;
+				if (dict_fts_index_syncing(table)) {
+					retry = true;
+				}
+				if (!retry) {
+					break;
+				}
+				DICT_BG_YIELD(trx);
+			}
 			row_mysql_unlock_data_dictionary(trx);
 			fts_optimize_remove_table(table);
 			row_mysql_lock_data_dictionary(trx);
diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc
index f321e0ba94a..bd311a99b77 100644
--- a/storage/innobase/row/row0sel.cc
+++ b/storage/innobase/row/row0sel.cc
@@ -2964,8 +2964,8 @@ row_sel_store_mysql_rec(
 	const ulint*	offsets,
 	bool		clust_templ_for_sec)
 {
-	ulint			i;
-	std::vector<ulint>	template_col;
+	ulint				i;
+	std::vector<const dict_col_t*>	template_col;
 
 	ut_ad(rec_clust || index == prebuilt->index);
 	ut_ad(!rec_clust || dict_index_is_clust(index));
@@ -2976,13 +2976,24 @@ row_sel_store_mysql_rec(
 	}
 
 	if (clust_templ_for_sec) {
-		/* Store all clustered index field of
+		/* Store all clustered index column of
 		secondary index record. */
 		for (i = 0; i < dict_index_get_n_fields(
 				prebuilt->index); i++) {
 			ulint   sec_field = dict_index_get_nth_field_pos(
 				index, prebuilt->index, i);
-			template_col.push_back(sec_field);
+
+			if (sec_field == ULINT_UNDEFINED) {
+				template_col.push_back(NULL);
+				continue;
+			}
+
+			const dict_field_t*	field =
+				dict_index_get_nth_field(index, sec_field);
+			const dict_col_t*	col =
+				dict_field_get_col(field);
+
+			template_col.push_back(col);
 		}
 	}
 
@@ -3000,9 +3011,13 @@ row_sel_store_mysql_rec(
 		      == 0);
 
 		if (clust_templ_for_sec) {
-			std::vector<ulint>::iterator	it;
+			std::vector<const dict_col_t*>::iterator it;
+			const dict_field_t*	field =
+					dict_index_get_nth_field(index, field_no);
+			const dict_col_t*	col = dict_field_get_col(
+								field);
 			it = std::find(template_col.begin(),
-				       template_col.end(), field_no);
+				       template_col.end(), col);
 
 			if (it == template_col.end()) {
 				continue;
@@ -4345,6 +4360,10 @@ rec_loop:
 
 	if (page_rec_is_supremum(rec)) {
 
+		DBUG_EXECUTE_IF("compare_end_range",
+				if (end_loop < 100) {
+					end_loop = 100;
+				});
 		/** Compare the last record of the page with end range
 		passed to InnoDB when there is no ICP and number of loops
 		in row_search_for_mysql for rows found but not
@@ -4385,7 +4404,7 @@ rec_loop:
 
 					/** In case of prebuilt->fetch,
 					set the error in prebuilt->end_range. */
-					if (prebuilt->n_fetch_cached > 0) {
+					if (next_buf != NULL) {
 						prebuilt->end_range = true;
 					}