MDEV-8772: Assertion failure in file ha_innodb.cc line 20027 when importing page compressed and encrypted tablespace using incorrect keys

Add error handling to decryp function when decrypt fails during
import.

4 files changed, 35 insertions, 10 deletions

diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc
index e151cb1af38..18f8983d79c 100644
--- a/storage/innobase/fil/fil0crypt.cc
+++ b/storage/innobase/fil/fil0crypt.cc
@@ -714,12 +714,16 @@ fil_space_decrypt(
 	fil_space_crypt_t*	crypt_data,	/*!< in: crypt data */
 	byte*			tmp_frame,	/*!< in: temporary buffer */
 	ulint			page_size,	/*!< in: page size */
-	byte*			src_frame)	/*!< in:out: page buffer */
+	byte*			src_frame,	/*!< in: out: page buffer */
+	dberr_t*		err)		/*!< in: out: DB_SUCCESS or
+						error code */
 {
 	ulint page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE);
 	uint key_version = mach_read_from_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
 	bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
 
+	*err = DB_SUCCESS;
+
 	if (key_version == ENCRYPTION_KEY_NOT_ENCRYPTED) {
 		return false;
 	}
@@ -756,6 +760,12 @@ fil_space_decrypt(
 					   space, offset, lsn);
 
 	if (! ((rc == MY_AES_OK) && ((ulint) dstlen == srclen))) {
+
+		if (rc == -1) {
+			*err = DB_DECRYPTION_FAILED;
+			return false;
+		}
+
 		ib_logf(IB_LOG_LEVEL_FATAL,
 			"Unable to decrypt data-block "
 			" src: %p srclen: %ld buf: %p buflen: %d."
@@ -797,11 +807,14 @@ fil_space_decrypt(
 	ulint		page_size,	/*!< in: page size */
 	byte*		src_frame)	/*!< in/out: page buffer */
 {
+	dberr_t err = DB_SUCCESS;
+
 	bool encrypted = fil_space_decrypt(
 				fil_space_get_crypt_data(space),
 				tmp_frame,
 				page_size,
-				src_frame);
+				src_frame,
+				&err);
 
 	if (encrypted) {
 		/* Copy the decrypted page back to page buffer, not
diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc
index a6f775a4e4d..b3dae5d24d7 100644
--- a/storage/innobase/fil/fil0fil.cc
+++ b/storage/innobase/fil/fil0fil.cc
@@ -6425,6 +6425,7 @@ fil_iterate(
 		if ((iter.crypt_data != NULL && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) ||
 				(srv_encrypt_tables &&
 					iter.crypt_data && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) {
+
 			encrypted = true;
 			readptr = iter.crypt_io_buffer;
 			writeptr = iter.crypt_io_buffer;
@@ -6444,6 +6445,7 @@ fil_iterate(
 
 		for (ulint i = 0; i < n_pages_read; ++i) {
 			ulint size = iter.page_size;
+			dberr_t	err = DB_SUCCESS;
 
 			/* If tablespace is encrypted, we need to decrypt
 			the page. */
@@ -6452,10 +6454,14 @@ fil_iterate(
 							iter.crypt_data,
 							io_buffer + i * size, //dst
 							iter.page_size,
-							readptr + i * size); // src
+							readptr + i * size, // src
+							&err); // src
+
+				if (err != DB_SUCCESS) {
+					return(err);
+				}
 
 				if (decrypted) {
-					/* write back unencrypted page */
 					updated = true;
 				} else {
 					/* TODO: remove unnecessary memcpy's */
@@ -6465,7 +6471,6 @@ fil_iterate(
 
 			buf_block_set_file_page(block, space_id, page_no++);
 
-			dberr_t	err;
 
 			if ((err = callback(page_off, block)) != DB_SUCCESS) {
 
diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h
index a8fd80c6f84..9230ffd3e7c 100644
--- a/storage/innobase/include/fil0crypt.h
+++ b/storage/innobase/include/fil0crypt.h
@@ -211,7 +211,10 @@ fil_space_decrypt(
 	fil_space_crypt_t*	crypt_data,	/*!< in: crypt data */
 	byte*			tmp_frame,	/*!< in: temporary buffer */
 	ulint			page_size,	/*!< in: page size */
-	byte*			src_frame);	/*!< in:out: page buffer */
+	byte*			src_frame,	/*!< in:out: page buffer */
+	dberr_t*		err);		/*!< in: out: DB_SUCCESS or
+						error code */
+
 
 /*********************************************************************
 Encrypt buffer page
diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc
index 4034a228153..ac6380e5a27 100644
--- a/storage/innobase/row/row0import.cc
+++ b/storage/innobase/row/row0import.cc
@@ -1,6 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 2012, 2015, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2015, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -3597,10 +3598,13 @@ row_import_for_mysql(
 		innobase_format_name(
 			table_name, sizeof(table_name), table->name, FALSE);
 
-		ib_errf(trx->mysql_thd, IB_LOG_LEVEL_ERROR,
-			ER_INTERNAL_ERROR,
-			"Cannot reset LSNs in table '%s' : %s",
-			table_name, ut_strerr(err));
+		if (err != DB_DECRYPTION_FAILED) {
+
+			ib_errf(trx->mysql_thd, IB_LOG_LEVEL_ERROR,
+				ER_INTERNAL_ERROR,
+				"Cannot reset LSNs in table '%s' : %s",
+				table_name, ut_strerr(err));
+		}
 
 		return(row_import_cleanup(prebuilt, trx, err));
 	}