Merge 10.2 into 10.3

14 files changed, 139 insertions, 73 deletions

diff --git a/README.md b/README.md
index f46888e4b2f..00587110004 100644
--- a/README.md
+++ b/README.md
@@ -17,10 +17,15 @@ see the Credits appendix.  You can also run 'SHOW authors' to get a
 list of active contributors.
 
 A description of the MariaDB project and a manual can be found at:
+
 https://mariadb.org/
+
 https://mariadb.com/kb/en/
+
 https://mariadb.com/kb/en/mariadb-vs-mysql-features/
+
 https://mariadb.com/kb/en/mariadb-versus-mysql-features/
+
 https://mariadb.com/kb/en/mariadb-versus-mysql-compatibility/
 
 As MariaDB is a full replacement of MySQL, the MySQL manual at
diff --git a/mysql-test/main/cte_nonrecursive.result b/mysql-test/main/cte_nonrecursive.result
index 534a386fe12..150fe13f7fb 100644
--- a/mysql-test/main/cte_nonrecursive.result
+++ b/mysql-test/main/cte_nonrecursive.result
@@ -1462,3 +1462,19 @@ a	b
 4	5
 4	3
 DROP TABLE t1;
+#
+# MDEV-16353: unreferenced CTE specified by query with UNION
+#
+with cte as
+(select 1 union select 2 union select 3)
+select 1 as f;
+f
+1
+create table t1 (a int);
+insert into t1 values (2), (1), (7), (1), (4);
+with cte as
+(select * from t1 where a < 2 union select * from t1 where a > 5)
+select 2 as f;
+f
+2
+drop table t1;
diff --git a/mysql-test/main/cte_nonrecursive.test b/mysql-test/main/cte_nonrecursive.test
index 5e1770496f6..98a77940c99 100644
--- a/mysql-test/main/cte_nonrecursive.test
+++ b/mysql-test/main/cte_nonrecursive.test
@@ -1012,3 +1012,21 @@ SELECT a FROM cte;
 WITH cte(a,b) AS (SELECT 4,5 UNION SELECT 4,3) SELECT a,b FROM cte;
 
 DROP TABLE t1;
+
+--echo #
+--echo # MDEV-16353: unreferenced CTE specified by query with UNION
+--echo #
+
+with cte as
+  (select 1 union select 2 union select 3)
+select 1 as f;
+
+create table t1 (a int);
+insert into t1 values (2), (1), (7), (1), (4);
+
+with cte as
+  (select * from t1 where a < 2 union select * from t1 where a > 5)
+select 2 as f;
+
+drop table t1;
+ 
\ No newline at end of file
diff --git a/mysql-test/suite/gcol/r/innodb_virtual_debug.result b/mysql-test/suite/gcol/r/innodb_virtual_debug.result
index 7774c6c347c..50b714566d9 100644
--- a/mysql-test/suite/gcol/r/innodb_virtual_debug.result
+++ b/mysql-test/suite/gcol/r/innodb_virtual_debug.result
@@ -64,11 +64,19 @@ INSERT INTO t VALUES (18, 1, DEFAULT, 'mm');
 INSERT INTO t VALUES (28, 1, DEFAULT, 'mm');
 INSERT INTO t VALUES (null, null, DEFAULT, 'mm');
 CREATE INDEX idx_1 on t(c);
-SET SESSION debug_dbug="+d,create_index_fail";
-ALTER TABLE t ADD COLUMN x INT GENERATED ALWAYS AS(a+b), ADD INDEX idx (x);
+SET @saved_dbug = @@SESSION.debug_dbug;
+SET debug_dbug = '+d,create_index_fail';
+ALTER TABLE t ADD COLUMN x INT GENERATED ALWAYS AS(a+b), ADD INDEX idx (x),
+ADD INDEX idcx (c,x);
 ERROR 23000: Duplicate entry '' for key '*UNKNOWN*'
-SET SESSION debug_dbug="";
+UPDATE t SET a=a+1;
+affected rows: 3
+info: Rows matched: 4  Changed: 3  Warnings: 0
+ALTER TABLE t ADD INDEX idc(c);
+ERROR 23000: Duplicate entry '' for key '*UNKNOWN*'
+SET debug_dbug = @saved_dbug;
 affected rows: 0
+UPDATE t SET b=b-1;
 SHOW CREATE TABLE t;
 Table	Create Table
 t	CREATE TABLE `t` (
diff --git a/mysql-test/suite/gcol/t/innodb_virtual_debug.test b/mysql-test/suite/gcol/t/innodb_virtual_debug.test
index 3870f84e066..ccdd16c9ebe 100644
--- a/mysql-test/suite/gcol/t/innodb_virtual_debug.test
+++ b/mysql-test/suite/gcol/t/innodb_virtual_debug.test
@@ -119,14 +119,23 @@ INSERT INTO t VALUES (null, null, DEFAULT, 'mm');
 
 CREATE INDEX idx_1 on t(c);
 
-SET SESSION debug_dbug="+d,create_index_fail";
+SET @saved_dbug = @@SESSION.debug_dbug;
+SET debug_dbug = '+d,create_index_fail';
 
 --enable_info
 --error ER_DUP_ENTRY
-ALTER TABLE t ADD COLUMN x INT GENERATED ALWAYS AS(a+b), ADD INDEX idx (x);
-SET SESSION debug_dbug="";
+ALTER TABLE t ADD COLUMN x INT GENERATED ALWAYS AS(a+b), ADD INDEX idx (x),
+ADD INDEX idcx (c,x);
+
+UPDATE t SET a=a+1;
+
+--error ER_DUP_ENTRY
+ALTER TABLE t ADD INDEX idc(c);
+SET debug_dbug = @saved_dbug;
 --disable_info
 
+UPDATE t SET b=b-1;
+
 SHOW CREATE TABLE t;
 
 SELECT c FROM t;
diff --git a/sql/sql_union.cc b/sql/sql_union.cc
index a1963c33a42..266f221ca78 100644
--- a/sql/sql_union.cc
+++ b/sql/sql_union.cc
@@ -1008,7 +1008,7 @@ bool st_select_lex_unit::prepare(TABLE_LIST *derived_arg,
     {
       if (with_element)
       {
-        if (derived_arg->with->rename_columns_of_derived_unit(thd, this))
+        if (with_element->rename_columns_of_derived_unit(thd, this))
           goto err;
         if (check_duplicate_names(thd, sl->item_list, 0))
           goto err;
diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc
index e7acead269c..9415624465f 100644
--- a/storage/innobase/dict/dict0dict.cc
+++ b/storage/innobase/dict/dict0dict.cc
@@ -2612,37 +2612,7 @@ dict_index_remove_from_cache_low(
 	UT_LIST_REMOVE(table->indexes, index);
 
 	/* Remove the index from affected virtual column index list */
-	if (dict_index_has_virtual(index)) {
-		const dict_col_t*	col;
-		const dict_v_col_t*	vcol;
-
-		for (ulint i = 0; i < dict_index_get_n_fields(index); i++) {
-			col =  dict_index_get_nth_col(index, i);
-			if (col->is_virtual()) {
-				vcol = reinterpret_cast<const dict_v_col_t*>(
-					col);
-
-				/* This could be NULL, when we do add virtual
-				column, add index together. We do not need to
-				track this virtual column's index */
-				if (vcol->v_indexes == NULL) {
-					continue;
-				}
-
-				dict_v_idx_list::iterator	it;
-
-				for (it = vcol->v_indexes->begin();
-				     it != vcol->v_indexes->end(); ++it) {
-					dict_v_idx_t	v_index = *it;
-					if (v_index.index == index) {
-						vcol->v_indexes->erase(it);
-						break;
-					}
-				}
-			}
-
-		}
-	}
+	index->detach_columns();
 
 	dict_mem_index_free(index);
 }
diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc
index 769eced242c..08963ceff3e 100644
--- a/storage/innobase/handler/handler0alter.cc
+++ b/storage/innobase/handler/handler0alter.cc
@@ -58,10 +58,6 @@ Smart ALTER TABLE
 #include "ha_innodb.h"
 #include "ut0new.h"
 #include "ut0stage.h"
-#ifdef WITH_WSREP
-//#include "wsrep_api.h"
-#include <sql_acl.h>	// PROCESS_ACL
-#endif
 
 static const char *MSG_UNSUPPORTED_ALTER_ONLINE_ON_VIRTUAL_COLUMN=
 			"INPLACE ADD or DROP of virtual columns cannot be "
@@ -289,6 +285,16 @@ struct ha_innobase_inplace_ctx : public inplace_alter_handler_ctx
 	@return whether the table will be rebuilt */
 	bool need_rebuild () const { return(old_table != new_table); }
 
+	/** Clear uncommmitted added indexes after a failed operation. */
+	void clear_added_indexes()
+	{
+		for (ulint i = 0; i < num_to_add_index; i++) {
+			if (!add_index[i]->is_committed()) {
+				add_index[i]->detach_columns();
+			}
+		}
+	}
+
 	/** Convert table-rebuilding ALTER to instant ALTER. */
 	void prepare_instant()
 	{
@@ -6726,7 +6732,8 @@ check_if_can_drop_indexes:
 		for (dict_index_t* index = dict_table_get_first_index(indexed_table);
 		     index != NULL; index = dict_table_get_next_index(index)) {
 
-			if (!index->to_be_dropped && index->is_corrupted()) {
+			if (!index->to_be_dropped && index->is_committed()
+			    && index->is_corrupted()) {
 				my_error(ER_INDEX_CORRUPT, MYF(0), index->name());
 				goto err_exit;
 			}
@@ -7293,6 +7300,7 @@ oom:
 	that we hold at most a shared lock on the table. */
 	m_prebuilt->trx->error_info = NULL;
 	ctx->trx->error_state = DB_SUCCESS;
+	ctx->clear_added_indexes();
 
 	DBUG_RETURN(true);
 }
diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h
index 5538b3e98ec..66b64f96128 100644
--- a/storage/innobase/include/dict0dict.h
+++ b/storage/innobase/include/dict0dict.h
@@ -731,13 +731,9 @@ dict_index_is_spatial(
 /*==================*/
 	const dict_index_t*	index)	/*!< in: index */
 	MY_ATTRIBUTE((warn_unused_result));
-/** Check whether the index contains a virtual column.
-@param[in]	index	index
-@return	nonzero for index on virtual column, zero for other indexes */
-UNIV_INLINE
-ulint
-dict_index_has_virtual(
-	const dict_index_t*	index);
+
+#define dict_index_has_virtual(index) (index)->has_virtual()
+
 /********************************************************************//**
 Check whether the index is the insert buffer tree.
 @return nonzero for insert buffer, zero for other indexes */
diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic
index 4c6c45a3b1e..db1be6513b1 100644
--- a/storage/innobase/include/dict0dict.ic
+++ b/storage/innobase/include/dict0dict.ic
@@ -314,20 +314,6 @@ dict_index_is_spatial(
 	return ulint(UNIV_EXPECT(index->type & DICT_SPATIAL, 0));
 }
 
-/** Check whether the index contains a virtual column
-@param[in]	index	index
-@return	nonzero for the index has virtual column, zero for other indexes */
-UNIV_INLINE
-ulint
-dict_index_has_virtual(
-	const dict_index_t*	index)
-{
-	ut_ad(index);
-	ut_ad(index->magic_n == DICT_INDEX_MAGIC_N);
-
-	return(index->type & DICT_VIRTUAL);
-}
-
 /********************************************************************//**
 Check whether the index is the insert buffer tree.
 @return nonzero for insert buffer, zero for other indexes */
diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h
index 6c116b9a428..154a503f1b3 100644
--- a/storage/innobase/include/dict0mem.h
+++ b/storage/innobase/include/dict0mem.h
@@ -588,6 +588,10 @@ struct dict_col_t{
 					3072 (REC_VERSION_56_MAX_INDEX_COL_LEN)
 					bytes. */
 
+	/** Detach the column from an index.
+	@param[in]	index	index to be detached from */
+	inline void detach(const dict_index_t& index);
+
 	/** Data for instantly added columns */
 	struct {
 		/** original default value of instantly added column */
@@ -1045,9 +1049,24 @@ struct dict_index_t{
 		return DICT_CLUSTERED == (type & (DICT_CLUSTERED | DICT_IBUF));
 	}
 
+	/** @return whether the index includes virtual columns */
+	bool has_virtual() const { return type & DICT_VIRTUAL; }
+
 	/** @return whether the index is corrupted */
 	inline bool is_corrupted() const;
 
+	/** Detach the columns from the index that is to be freed. */
+	void detach_columns()
+	{
+		if (has_virtual()) {
+			for (unsigned i = 0; i < n_fields; i++) {
+				fields[i].col->detach(*this);
+			}
+
+			n_fields = 0;
+		}
+	}
+
 	/** Determine how many fields of a given prefix can be set NULL.
 	@param[in]	n_prefix	number of fields in the prefix
 	@return	number of fields 0..n_prefix-1 that can be set NULL */
@@ -1112,6 +1131,26 @@ struct dict_index_t{
 	vers_history_row(const rec_t* rec, bool &history_row);
 };
 
+/** Detach a column from an index.
+@param[in]	index	index to be detached from */
+inline void dict_col_t::detach(const dict_index_t& index)
+{
+	if (!is_virtual()) {
+		return;
+	}
+
+	if (dict_v_idx_list* v_indexes = reinterpret_cast<const dict_v_col_t*>
+	    (this)->v_indexes) {
+		for (dict_v_idx_list::iterator i = v_indexes->begin();
+		     i != v_indexes->end(); i++) {
+			if (i->index == &index) {
+				v_indexes->erase(i);
+				return;
+			}
+		}
+	}
+}
+
 /** The status of online index creation */
 enum online_index_status {
 	/** the index is complete and ready for access */
diff --git a/storage/innobase/log/log0crypt.cc b/storage/innobase/log/log0crypt.cc
index 980b26d448c..dff9661c6eb 100644
--- a/storage/innobase/log/log0crypt.cc
+++ b/storage/innobase/log/log0crypt.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (C) 2013, 2015, Google Inc. All Rights Reserved.
-Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved.
+Copyright (C) 2014, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -160,7 +160,7 @@ static bool init_crypt_key(crypt_info_t* info, bool upgrade = false)
 			<< "Obtaining redo log encryption key version "
 			<< info->key_version << " failed (" << rc
 			<< "). Maybe the key or the required encryption "
-			<< " key management plugin was not found.";
+			"key management plugin was not found.";
 		return false;
 	}
 
@@ -280,7 +280,12 @@ log_crypt_101_read_block(byte* buf)
 		}
 	}
 
-	return false;
+	if (infos_used == 0) {
+		return false;
+	}
+	/* MariaDB Server 10.1 would use the first key if it fails to
+	find a key for the current checkpoint. */
+	info = infos;
 found:
 	byte dst[OS_FILE_LOG_BLOCK_SIZE];
 	uint dst_len;
diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc
index 0de383c24b4..b2c66bd389b 100644
--- a/storage/innobase/log/log0recv.cc
+++ b/storage/innobase/log/log0recv.cc
@@ -858,12 +858,11 @@ recv_find_max_checkpoint_0(ulint* max_field)
 
 /** Determine if a pre-MySQL 5.7.9/MariaDB 10.2.2 redo log is clean.
 @param[in]	lsn	checkpoint LSN
+@param[in]	crypt	whether the log might be encrypted
 @return error code
 @retval	DB_SUCCESS	if the redo log is clean
 @retval DB_ERROR	if the redo log is corrupted or dirty */
-static
-dberr_t
-recv_log_format_0_recover(lsn_t lsn)
+static dberr_t recv_log_format_0_recover(lsn_t lsn, bool crypt)
 {
 	log_mutex_enter();
 	const lsn_t	source_offset = log_sys.log.calc_lsn_offset(lsn);
@@ -891,7 +890,13 @@ recv_log_format_0_recover(lsn_t lsn)
 	}
 
 	if (log_block_get_data_len(buf)
-	    != (source_offset & (OS_FILE_LOG_BLOCK_SIZE - 1))) {
+	    == (source_offset & (OS_FILE_LOG_BLOCK_SIZE - 1))) {
+	} else if (crypt) {
+		ib::error() << "Cannot decrypt log for upgrading."
+			" The encrypted log was created"
+			" before MariaDB 10.2.2.";
+		return DB_ERROR;
+	} else {
 		ib::error() << NO_UPGRADE_RECOVERY_MSG << ".";
 		return(DB_ERROR);
 	}
@@ -3153,7 +3158,8 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn)
 	switch (log_sys.log.format) {
 	case 0:
 		log_mutex_exit();
-		return(recv_log_format_0_recover(checkpoint_lsn));
+		return recv_log_format_0_recover(checkpoint_lsn,
+						 buf[20 + 32 * 9] == 2);
 	default:
 		if (end_lsn == 0) {
 			break;
diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc
index e2197f845f3..bca7464bc66 100644
--- a/storage/innobase/row/row0upd.cc
+++ b/storage/innobase/row/row0upd.cc
@@ -581,7 +581,7 @@ row_upd_changes_field_size_or_external(
 		/* We should ignore virtual field if the index is not
 		a virtual index */
 		if (upd_fld_is_virtual_col(upd_field)
-		    && dict_index_has_virtual(index) != DICT_VIRTUAL) {
+		    && !index->has_virtual()) {
 			continue;
 		}