Merge 10.2 into 10.3

6 files changed, 86 insertions, 221 deletions

diff --git a/storage/innobase/fts/fts0ast.cc b/storage/innobase/fts/fts0ast.cc
index 82c4438196b..d6695cec5be 100644
--- a/storage/innobase/fts/fts0ast.cc
+++ b/storage/innobase/fts/fts0ast.cc
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
-Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2007, 2018, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -28,6 +29,7 @@ Created 2007/3/16 Sunny Bains.
 #include "fts0ast.h"
 #include "fts0pars.h"
 #include "fts0fts.h"
+#include "row0sel.h"
 
 /* The FTS ast visit pass. */
 enum fts_ast_visit_pass_t {
@@ -602,6 +604,7 @@ fts_ast_visit(
 	bool			revisit = false;
 	bool			will_be_ignored = false;
 	fts_ast_visit_pass_t	visit_pass = FTS_PASS_FIRST;
+	const trx_t*		trx = node->trx;
 
 	start_node = node->list.head;
 
@@ -700,6 +703,10 @@ fts_ast_visit(
 		}
 	}
 
+	if (trx_is_interrupted(trx)) {
+		return DB_INTERRUPTED;
+	}
+
 	if (revisit) {
 		/* Exist pass processes the skipped FTS_EXIST operation. */
                 for (node = start_node;
diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc
index d874725c374..7c26614b309 100644
--- a/storage/innobase/fts/fts0fts.cc
+++ b/storage/innobase/fts/fts0fts.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2011, 2017, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2011, 2018, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2016, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -202,7 +202,7 @@ FTS auxiliary INDEX table and clear the cache at the end.
 @param[in,out]	sync		sync state
 @param[in]	unlock_cache	whether unlock cache lock when write node
 @param[in]	wait		whether wait when a sync is in progress
-@param[in]      has_dict        whether has dict operation lock
+@param[in]	has_dict	whether has dict operation lock
 @return DB_SUCCESS if all OK */
 static
 dberr_t
@@ -860,37 +860,28 @@ fts_drop_index(
 
 			err = fts_drop_index_tables(trx, index);
 
-			for(;;) {
-				bool retry = false;
-				if (index->index_fts_syncing) {
-					retry = true;
-				}
-				if (!retry){
-					fts_free(table);
-					break;
-				}
+			while (index->index_fts_syncing
+				&& !trx_is_interrupted(trx)) {
 				DICT_BG_YIELD(trx);
 			}
+
+			fts_free(table);
+
 			return(err);
 		}
 
-		for(;;) {
-			bool retry = false;
-			if (index->index_fts_syncing) {
-				retry = true;
-			}
-			if (!retry){
-				current_doc_id = table->fts->cache->next_doc_id;
-				first_doc_id = table->fts->cache->first_doc_id;
-				fts_cache_clear(table->fts->cache);
-				fts_cache_destroy(table->fts->cache);
-				table->fts->cache = fts_cache_create(table);
-				table->fts->cache->next_doc_id = current_doc_id;
-				table->fts->cache->first_doc_id = first_doc_id;
-				break;
-			}
+		while (index->index_fts_syncing
+		       && !trx_is_interrupted(trx)) {
 			DICT_BG_YIELD(trx);
 		}
+
+		current_doc_id = table->fts->cache->next_doc_id;
+		first_doc_id = table->fts->cache->first_doc_id;
+		fts_cache_clear(table->fts->cache);
+		fts_cache_destroy(table->fts->cache);
+		table->fts->cache = fts_cache_create(table);
+		table->fts->cache->next_doc_id = current_doc_id;
+		table->fts->cache->first_doc_id = first_doc_id;
 	} else {
 		fts_cache_t*            cache = table->fts->cache;
 		fts_index_cache_t*      index_cache;
@@ -900,18 +891,14 @@ fts_drop_index(
 		index_cache = fts_find_index_cache(cache, index);
 
 		if (index_cache != NULL) {
-			for(;;) {
-				bool retry = false;
-				if (index->index_fts_syncing) {
-					retry = true;
-				}
-				if (!retry && index_cache->words) {
-					fts_words_free(index_cache->words);
-					rbt_free(index_cache->words);
-					break;
-				}
+			while (index->index_fts_syncing
+			       && !trx_is_interrupted(trx)) {
 				DICT_BG_YIELD(trx);
 			}
+			if (index_cache->words) {
+				fts_words_free(index_cache->words);
+				rbt_free(index_cache->words);
+			}
 
 			ib_vector_remove(cache->indexes, *(void**) index_cache);
 		}
@@ -4048,6 +4035,9 @@ fts_sync_write_words(
 
 		word = rbt_value(fts_tokenizer_word_t, rbt_node);
 
+		DBUG_EXECUTE_IF("fts_instrument_write_words_before_select_index",
+				os_thread_sleep(300000););
+
 		selected = fts_select_index(
 			index_cache->charset, word->text.f_str,
 			word->text.f_len);
@@ -4332,7 +4322,7 @@ FTS auxiliary INDEX table and clear the cache at the end.
 @param[in,out]	sync		sync state
 @param[in]	unlock_cache	whether unlock cache lock when write node
 @param[in]	wait		whether wait when a sync is in progress
-@param[in]      has_dict        whether has dict operation lock
+@param[in]	has_dict	whether has dict operation lock
 @return DB_SUCCESS if all OK */
 static
 dberr_t
@@ -4398,15 +4388,13 @@ begin_sync:
 			continue;
 		}
 
+		DBUG_EXECUTE_IF("fts_instrument_sync_before_syncing",
+				os_thread_sleep(300000););
 		index_cache->index->index_fts_syncing = true;
-		DBUG_EXECUTE_IF("fts_instrument_sync_sleep_drop_waits",
-				os_thread_sleep(10000000);
-				);
 
 		error = fts_sync_index(sync, index_cache);
 
-		if (error != DB_SUCCESS && !sync->interrupted) {
-
+		if (error != DB_SUCCESS) {
 			goto end_sync;
 		}
 	}
@@ -4441,8 +4429,8 @@ end_sync:
 	}
 
 	rw_lock_x_lock(&cache->lock);
-	/* Clear fts syncing flags of any indexes incase sync is
-	interrupeted */
+	/* Clear fts syncing flags of any indexes in case sync is
+	interrupted */
 	for (i = 0; i < ib_vector_size(cache->indexes); ++i) {
 		static_cast<fts_index_cache_t*>(
 			ib_vector_get(cache->indexes, i))
diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc
index 4d93451a40d..77115049903 100644
--- a/storage/innobase/fts/fts0opt.cc
+++ b/storage/innobase/fts/fts0opt.cc
@@ -62,12 +62,6 @@ static const ulint FTS_WORD_NODES_INIT_SIZE = 64;
 /** Last time we did check whether system need a sync */
 static ib_time_t	last_check_sync_time;
 
-#if 0
-/** Check each table in round robin to see whether they'd
-need to be "optimized" */
-static	ulint	fts_optimize_sync_iterator = 0;
-#endif
-
 /** State of a table within the optimization sub system. */
 enum fts_state_t {
 	FTS_STATE_LOADED,
@@ -79,17 +73,11 @@ enum fts_state_t {
 
 /** FTS optimize thread message types. */
 enum fts_msg_type_t {
-	FTS_MSG_START,			/*!< Start optimizing thread */
-
-	FTS_MSG_PAUSE,			/*!< Pause optimizing thread */
-
 	FTS_MSG_STOP,			/*!< Stop optimizing and exit thread */
 
 	FTS_MSG_ADD_TABLE,		/*!< Add table to the optimize thread's
 					work queue */
 
-	FTS_MSG_OPTIMIZE_TABLE,		/*!< Optimize a table */
-
 	FTS_MSG_DEL_TABLE,		/*!< Remove a table from the optimize
 					threads work queue */
 	FTS_MSG_SYNC_TABLE		/*!< Sync fts cache of a table */
@@ -234,7 +222,7 @@ struct fts_msg_t {
 /** The number of words to read and optimize in a single pass. */
 ulong	fts_num_word_optimize;
 
-// FIXME
+/** Whether to enable additional FTS diagnostic printout. */
 char	fts_enable_diag_print;
 
 /** ZLib compressed block size.*/
@@ -2554,12 +2542,9 @@ fts_optimize_create_msg(
 	return(msg);
 }
 
-/**********************************************************************//**
-Add the table to add to the OPTIMIZER's list. */
-void
-fts_optimize_add_table(
-/*===================*/
-	dict_table_t*	table)			/*!< in: table to add */
+/** Add the table to add to the OPTIMIZER's list.
+@param[in]	table	table to add */
+void fts_optimize_add_table(dict_table_t* table)
 {
 	fts_msg_t*	msg;
 
@@ -2575,28 +2560,6 @@ fts_optimize_add_table(
 	ib_wqueue_add(fts_optimize_wq, msg, msg->heap);
 }
 
-#if 0
-/**********************************************************************//**
-Optimize a table. */
-static
-void
-fts_optimize_do_table(
-/*==================*/
-	dict_table_t*	table)			/*!< in: table to optimize */
-{
-	fts_msg_t*	msg;
-
-	/* Optimizer thread could be shutdown */
-	if (!fts_optimize_wq) {
-		return;
-	}
-
-	msg = fts_optimize_create_msg(FTS_MSG_OPTIMIZE_TABLE, table);
-
-	ib_wqueue_add(fts_optimize_wq, msg, msg->heap);
-}
-#endif
-
 /**********************************************************************//**
 Remove the table from the OPTIMIZER's list. We do wait for
 acknowledgement from the consumer of the message. */
@@ -2672,53 +2635,6 @@ fts_optimize_request_sync_table(
 }
 
 /**********************************************************************//**
-Find the slot for a particular table.
-@return slot if found else NULL. */
-static
-fts_slot_t*
-fts_optimize_find_slot(
-/*===================*/
-	ib_vector_t*		tables,		/*!< in: vector of tables */
-	const dict_table_t*	table)		/*!< in: table to add */
-{
-	ulint		i;
-
-	for (i = 0; i < ib_vector_size(tables); ++i) {
-		fts_slot_t*	slot;
-
-		slot = static_cast<fts_slot_t*>(ib_vector_get(tables, i));
-
-		if (slot->table == table) {
-			return(slot);
-		}
-	}
-
-	return(NULL);
-}
-
-/**********************************************************************//**
-Start optimizing table. */
-static
-void
-fts_optimize_start_table(
-/*=====================*/
-	ib_vector_t*		tables,		/*!< in/out: vector of tables */
-	dict_table_t*		table)		/*!< in: table to optimize */
-{
-	fts_slot_t*	slot;
-
-	slot = fts_optimize_find_slot(tables, table);
-
-	if (slot == NULL) {
-		ib::error() << "Table " << table->name << " not registered"
-			" with the optimize thread.";
-	} else {
-		slot->last_run = 0;
-		slot->completed = 0;
-	}
-}
-
-/**********************************************************************//**
 Add the table to the vector if it doesn't already exist. */
 static
 ibool
@@ -2900,57 +2816,6 @@ fts_is_sync_needed(
 	return(false);
 }
 
-#if 0
-/*********************************************************************//**
-Check whether a table needs to be optimized. */
-static
-void
-fts_optimize_need_sync(
-/*===================*/
-	ib_vector_t*	tables)	/*!< in: list of tables */
-{
-	dict_table_t*	table = NULL;
-	fts_slot_t*	slot;
-	ulint		num_table = ib_vector_size(tables);
-
-	if (!num_table) {
-		return;
-	}
-
-	if (fts_optimize_sync_iterator >= num_table) {
-		fts_optimize_sync_iterator = 0;
-	}
-
-	slot = ib_vector_get(tables, fts_optimize_sync_iterator);
-	table = slot->table;
-
-	if (!table) {
-		return;
-	}
-
-	ut_ad(table->fts);
-
-	if (table->fts->cache) {
-		ulint	deleted = table->fts->cache->deleted;
-
-		if (table->fts->cache->added
-		    >= fts_optimize_add_threshold) {
-			fts_sync_table(table);
-		} else if (deleted >= fts_optimize_delete_threshold) {
-			fts_optimize_do_table(table);
-
-			mutex_enter(&table->fts->cache->deleted_lock);
-			table->fts->cache->deleted -= deleted;
-			mutex_exit(&table->fts->cache->deleted_lock);
-		}
-	}
-
-	fts_optimize_sync_iterator++;
-
-	return;
-}
-#endif
-
 /** Sync fts cache of a table
 @param[in]	table_id	table id */
 void
@@ -2963,7 +2828,7 @@ fts_optimize_sync_table(
 
 	if (table) {
 		if (dict_table_has_fts_index(table) && table->fts->cache) {
-			fts_sync_table(table, true, false, true);
+			fts_sync_table(table, true, false, false);
 		}
 
 		dict_table_close(table, FALSE, FALSE);
@@ -3031,8 +2896,7 @@ DECLARE_THREAD(fts_optimize_thread)(
 			fts_msg_t*	msg;
 
 			msg = static_cast<fts_msg_t*>(
-				ib_wqueue_timedwait(wq,
-						    FTS_QUEUE_WAIT_IN_USECS));
+				ib_wqueue_timedwait(wq, FTS_QUEUE_WAIT_IN_USECS));
 
 			/* Timeout ? */
 			if (msg == NULL) {
@@ -3044,12 +2908,6 @@ DECLARE_THREAD(fts_optimize_thread)(
 			}
 
 			switch (msg->type) {
-			case FTS_MSG_START:
-				break;
-
-			case FTS_MSG_PAUSE:
-				break;
-
 			case FTS_MSG_STOP:
 				done = TRUE;
 				break;
@@ -3064,15 +2922,6 @@ DECLARE_THREAD(fts_optimize_thread)(
 				}
 				break;
 
-			case FTS_MSG_OPTIMIZE_TABLE:
-				if (!done) {
-					fts_optimize_start_table(
-						fts_slots,
-						static_cast<dict_table_t*>(
-						msg->ptr));
-				}
-				break;
-
 			case FTS_MSG_DEL_TABLE:
 				if (fts_optimize_del_table(
 					fts_slots, static_cast<fts_msg_del_t*>(
@@ -3087,6 +2936,10 @@ DECLARE_THREAD(fts_optimize_thread)(
 				break;
 
 			case FTS_MSG_SYNC_TABLE:
+				DBUG_EXECUTE_IF(
+					"fts_instrument_msg_sync_sleep",
+					os_thread_sleep(300000););
+
 				fts_optimize_sync_table(
 					*static_cast<table_id_t*>(msg->ptr));
 				break;
@@ -3149,7 +3002,7 @@ fts_optimize_init(void)
 	ut_ad(!srv_read_only_mode);
 
 	/* For now we only support one optimize thread. */
-	ut_a(fts_optimize_wq == NULL);
+	ut_a(!fts_optimize_wq);
 
 	/* Create FTS optimize work queue */
 	fts_optimize_wq = ib_wqueue_create();
diff --git a/storage/innobase/fts/fts0pars.cc b/storage/innobase/fts/fts0pars.cc
index e4d1bba2be6..56cc8d6052c 100644
--- a/storage/innobase/fts/fts0pars.cc
+++ b/storage/innobase/fts/fts0pars.cc
@@ -88,7 +88,7 @@ extern	int fts_lexer(YYSTYPE*, fts_lexer_t*);
 extern	int fts_blexer(YYSTYPE*, yyscan_t);
 extern	int fts_tlexer(YYSTYPE*, yyscan_t);
 
-typedef int (*fts_scan)();
+
 
 extern int ftserror(const char* p);
 
@@ -103,8 +103,8 @@ extern int ftserror(const char* p);
 
 #define YYTOKENFREE(token) fts_ast_string_free((token))
 
-typedef	int	(*fts_scanner_alt)(YYSTYPE* val, yyscan_t yyscanner);
-typedef	int	(*fts_scanner)();
+
+typedef	int	(*fts_scanner)(YYSTYPE* val, yyscan_t yyscanner);
 
 struct fts_lexer_t {
 	fts_scanner	scanner;
@@ -1948,7 +1948,7 @@ fts_lexer_create(
 			reinterpret_cast<const char*>(query),
 			static_cast<int>(query_len),
 			fts_lexer->yyscanner);
-		fts_lexer->scanner = reinterpret_cast<fts_scan>(fts_blexer);
+		fts_lexer->scanner = fts_blexer;
 		/* FIXME: Debugging */
 		/* fts0bset_debug(1 , fts_lexer->yyscanner); */
 	} else {
@@ -1957,7 +1957,7 @@ fts_lexer_create(
 			reinterpret_cast<const char*>(query),
 			static_cast<int>(query_len),
 			fts_lexer->yyscanner);
-		fts_lexer->scanner = reinterpret_cast<fts_scan>(fts_tlexer);
+		fts_lexer->scanner = fts_tlexer;
 	}
 
 	return(fts_lexer);
@@ -1971,7 +1971,7 @@ fts_lexer_free(
 /*===========*/
 	fts_lexer_t*	fts_lexer)
 {
-	if (fts_lexer->scanner == (fts_scan) fts_blexer) {
+	if (fts_lexer->scanner == fts_blexer) {
 		fts0blex_destroy(fts_lexer->yyscanner);
 	} else {
 		fts0tlex_destroy(fts_lexer->yyscanner);
@@ -1988,9 +1988,9 @@ fts_lexer(
 	YYSTYPE*	val,
 	fts_lexer_t*	fts_lexer)
 {
-	fts_scanner_alt func_ptr;
+	fts_scanner func_ptr;
 
-	func_ptr = (fts_scanner_alt) fts_lexer->scanner;
+	func_ptr = fts_lexer->scanner;
 
 	return(func_ptr(val, fts_lexer->yyscanner));
 }
diff --git a/storage/innobase/fts/fts0pars.y b/storage/innobase/fts/fts0pars.y
index 1f4ec9922e3..4db72ca65ad 100644
--- a/storage/innobase/fts/fts0pars.y
+++ b/storage/innobase/fts/fts0pars.y
@@ -36,7 +36,7 @@ extern	int fts_lexer(YYSTYPE*, fts_lexer_t*);
 extern	int fts_blexer(YYSTYPE*, yyscan_t);
 extern	int fts_tlexer(YYSTYPE*, yyscan_t);
 
-typedef int (*fts_scan)();
+
 
 extern int ftserror(const char* p);
 
@@ -49,8 +49,8 @@ extern int ftserror(const char* p);
 #define YYPARSE_PARAM state
 #define YYLEX_PARAM ((fts_ast_state_t*) state)->lexer
 
-typedef	int	(*fts_scanner_alt)(YYSTYPE* val, yyscan_t yyscanner);
-typedef	int	(*fts_scanner)();
+
+typedef	int	(*fts_scanner)(YYSTYPE* val, yyscan_t yyscanner);
 
 struct fts_lexer_struct {
 	fts_scanner	scanner;
@@ -238,13 +238,13 @@ fts_lexer_create(
 	if (boolean_mode) {
 		fts0blex_init(&fts_lexer->yyscanner);
 		fts0b_scan_bytes((char*) query, (int) query_len, fts_lexer->yyscanner);
-		fts_lexer->scanner = (fts_scan) fts_blexer;
+		fts_lexer->scanner = fts_blexer;
 		/* FIXME: Debugging */
 		/* fts0bset_debug(1 , fts_lexer->yyscanner); */
 	} else {
 		fts0tlex_init(&fts_lexer->yyscanner);
 		fts0t_scan_bytes((char*) query, (int) query_len, fts_lexer->yyscanner);
-		fts_lexer->scanner = (fts_scan) fts_tlexer;
+		fts_lexer->scanner = fts_tlexer;
 	}
 
 	return(fts_lexer);
@@ -258,7 +258,7 @@ fts_lexer_free(
 /*===========*/
 	fts_lexer_t*	fts_lexer)
 {
-	if (fts_lexer->scanner == (fts_scan) fts_blexer) {
+	if (fts_lexer->scanner == fts_blexer) {
 		fts0blex_destroy(fts_lexer->yyscanner);
 	} else {
 		fts0tlex_destroy(fts_lexer->yyscanner);
@@ -275,9 +275,9 @@ fts_lexer(
 	YYSTYPE*	val,
 	fts_lexer_t*	fts_lexer)
 {
-	fts_scanner_alt func_ptr;
+	fts_scanner func_ptr;
 
-	func_ptr = (fts_scanner_alt) fts_lexer->scanner;
+	func_ptr = fts_lexer->scanner;
 
 	return(func_ptr(val, fts_lexer->yyscanner));
 }
diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc
index 6a0496bedcd..5ae8e28974e 100644
--- a/storage/innobase/fts/fts0que.cc
+++ b/storage/innobase/fts/fts0que.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2007, 2017, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2007, 2018, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2017, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -3920,6 +3920,7 @@ fts_query_can_optimize(
 }
 
 /** FTS Query entry point.
+@param[in,out]	trx		transaction
 @param[in]	index		fts index to search
 @param[in]	flags		FTS search mode
 @param[in]	query_str	FTS query
@@ -3928,6 +3929,7 @@ fts_query_can_optimize(
 @return DB_SUCCESS if successful otherwise error code */
 dberr_t
 fts_query(
+	trx_t*		trx,
 	dict_index_t*	index,
 	uint		flags,
 	const byte*	query_str,
@@ -3940,7 +3942,7 @@ fts_query(
 	ulint		lc_query_str_len;
 	ulint		result_len;
 	bool		boolean_mode;
-	trx_t*		query_trx;
+	trx_t*		query_trx; /* FIXME: use provided trx */
 	CHARSET_INFO*	charset;
 	ulint		start_time_ms;
 	bool		will_be_ignored = false;
@@ -4055,6 +4057,7 @@ fts_query(
 	/* Parse the input query string. */
 	if (fts_query_parse(&query, lc_query_str, result_len)) {
 		fts_ast_node_t*	ast = query.root;
+		ast->trx = trx;
 
 		/* Optimize query to check if it's a single term */
 		fts_query_can_optimize(&query, flags);
@@ -4068,6 +4071,11 @@ fts_query(
 		query.error = fts_ast_visit(
 			FTS_NONE, ast, fts_query_visitor,
 			&query, &will_be_ignored);
+		if (query.error == DB_INTERRUPTED) {
+			error = DB_INTERRUPTED;
+			ut_free(lc_query_str);
+			goto func_exit;
+		}
 
 		/* If query expansion is requested, extend the search
 		with first search pass result */
@@ -4094,6 +4102,15 @@ fts_query(
 			ut_zalloc_nokey(sizeof(**result)));
 	}
 
+	if (trx_is_interrupted(trx)) {
+		error = DB_INTERRUPTED;
+		ut_free(lc_query_str);
+		if (*result) {
+			fts_query_free_result(*result);
+		}
+		goto func_exit;
+	}
+
 	ut_free(lc_query_str);
 
 	if (fts_enable_diag_print && (*result)) {