MDEV-8173: InnoDB; Failing assertion: crypt_data->type == 1

Make sure that when we publish the crypt_data we access the
memory cache of the tablespace crypt_data. Make sure that
crypt_data is stored whenever it is really needed.

All this is not yet enough in my opinion because:

sql/encryption.cc has DBUG_ASSERT(scheme->type == 1) i.e.
crypt_data->type == CRYPT_SCHEME_1

However, for InnoDB point of view we have global crypt_data
for every tablespace. When we change variables on crypt_data
we take mutex. However, when we use crypt_data for
encryption/decryption we use pointer to this global
structure and no mutex to protect against changes on
crypt_data.

Tablespace encryption starts in fil_crypt_start_encrypting_space
from crypt_data that has crypt_data->type = CRYPT_SCHEME_UNENCRYPTED
and later we write page 0 CRYPT_SCHEME_1 and finally whe publish
that to memory cache.

8 files changed, 32 insertions, 46 deletions

diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc
index b32812d7c81..7249dd7bb5b 100644
--- a/storage/innobase/buf/buf0buf.cc
+++ b/storage/innobase/buf/buf0buf.cc
@@ -4334,7 +4334,7 @@ corrupt:
 				"InnoDB: space %lu file %s read of page %lu.\n"
 				"InnoDB: You may have to recover"
 				" from a backup.\n",
-				bpage->space,
+				(ulint)bpage->space,
 				space ? space->name : "NULL",
 				(ulong) bpage->offset);
 			buf_page_print(frame, buf_page_get_zip_size(bpage),
diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc
index bb790c2e30b..edfa6604dcd 100644
--- a/storage/innobase/fil/fil0crypt.cc
+++ b/storage/innobase/fil/fil0crypt.cc
@@ -266,13 +266,8 @@ fil_space_read_crypt_data(
 	ulint		offset)	/*!< in: offset */
 {
 	if (memcmp(page + offset, EMPTY_PATTERN, MAGIC_SZ) == 0) {
-		/* crypt is not stored but create memory cache for
-		not system tablespace */
-		if (space != 0) {
-			return fil_space_create_crypt_data(FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY);
-		} else {
-			return NULL;
-		}
+		/* Crypt data is not stored. */
+		return NULL;
 	}
 
 	if (memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) {
@@ -288,12 +283,8 @@ fil_space_read_crypt_data(
 			page[offset + 3],
 			page[offset + 4],
 			page[offset + 5]);
-		/* Create memory cache for not system tablespace */
-		if (space != 0) {
-			return fil_space_create_crypt_data(FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY);
-		} else {
-			return NULL;
-		}
+		/* Create data is not stored. */
+		return NULL;
 	}
 
 	ulint type = mach_read_from_1(page + offset + MAGIC_SZ + 0);
@@ -461,12 +452,6 @@ fil_space_write_crypt_data(
 		return;
 	}
 
-	/* If tablespace encryption is disabled and encryption mode is
-	DEFAULT, then do not continue writing crypt data to page 0. */
-	if (!srv_encrypt_tables && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT) {
-		return;
-	}
-
 	fil_space_write_crypt_data_low(crypt_data, crypt_data->type,
 				       page, offset, maxsize, mtr);
 }
@@ -1073,7 +1058,7 @@ fil_crypt_start_encrypting_space(
 	crypt_data->rotate_state.active_threads = 1;
 
 	mutex_enter(&crypt_data->mutex);
-	fil_space_set_crypt_data(space, crypt_data);
+	crypt_data = fil_space_set_crypt_data(space, crypt_data);
 	mutex_exit(&crypt_data->mutex);
 
 	fil_crypt_start_converting = true;
@@ -1108,6 +1093,7 @@ fil_crypt_start_encrypting_space(
 		/* 3 - compute location to store crypt data */
 		byte* frame = buf_block_get_frame(block);
 		ulint maxsize;
+		ut_ad(crypt_data);
 		crypt_data->page0_offset =
 			fsp_header_get_crypt_offset(zip_size, &maxsize);
 
@@ -1160,6 +1146,7 @@ fil_crypt_start_encrypting_space(
 
 		/* 5 - publish crypt data */
 		mutex_enter(&fil_crypt_threads_mutex);
+		ut_ad(crypt_data);
 		mutex_enter(&crypt_data->mutex);
 		crypt_data->type = CRYPT_SCHEME_1;
 		ut_a(crypt_data->rotate_state.active_threads == 1);
@@ -1173,6 +1160,7 @@ fil_crypt_start_encrypting_space(
 		return pending_op;
 	} while (0);
 
+	ut_ad(crypt_data);
 	mutex_enter(&crypt_data->mutex);
 	ut_a(crypt_data->rotate_state.active_threads == 1);
 	crypt_data->rotate_state.active_threads = 0;
diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc
index 629ba0a972c..c193f0e748d 100644
--- a/storage/innobase/fil/fil0fil.cc
+++ b/storage/innobase/fil/fil0fil.cc
@@ -6855,7 +6855,7 @@ fil_space_get_crypt_data(
 /******************************************************************
 Get crypt data for a tablespace */
 UNIV_INTERN
-void
+fil_space_crypt_t*
 fil_space_set_crypt_data(
 /*=====================*/
 	ulint id, 	               /*!< in: space id */
@@ -6863,6 +6863,7 @@ fil_space_set_crypt_data(
 {
 	fil_space_t*	space;
 	fil_space_crypt_t* free_crypt_data = NULL;
+	fil_space_crypt_t* ret_crypt_data = NULL;
 
 	ut_ad(fil_system);
 
@@ -6881,9 +6882,11 @@ fil_space_set_crypt_data(
 			mutex_exit(&fil_system->mutex);
 			fil_space_merge_crypt_data(space->crypt_data,
 						   crypt_data);
+			ret_crypt_data = space->crypt_data;
 			free_crypt_data = crypt_data;
 		} else {
 			space->crypt_data = crypt_data;
+			ret_crypt_data = space->crypt_data;
 			mutex_exit(&fil_system->mutex);
 		}
 	} else {
@@ -6899,4 +6902,6 @@ fil_space_set_crypt_data(
 		*/
 		fil_space_destroy_crypt_data(&free_crypt_data);
 	}
+
+	return ret_crypt_data;
 }
diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h
index ccb2d852572..b633d1100bc 100644
--- a/storage/innobase/include/fil0crypt.h
+++ b/storage/innobase/include/fil0crypt.h
@@ -134,7 +134,7 @@ fil_space_get_crypt_data(
 /*********************************************************************
 Set crypt data for a space*/
 UNIV_INTERN
-void
+fil_space_crypt_t*
 fil_space_set_crypt_data(
 /*=====================*/
 	ulint space,                    /*!< in: tablespace id */
diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc
index 934223fc15c..d0a57472ea1 100644
--- a/storage/xtradb/buf/buf0buf.cc
+++ b/storage/xtradb/buf/buf0buf.cc
@@ -4414,7 +4414,7 @@ corrupt:
 				"InnoDB: space %lu file %s read of page %lu.\n"
 				"InnoDB: You may have to recover"
 				" from a backup.\n",
-				bpage->space,
+				(ulint)bpage->space,
 				space ? space->name : "NULL",
 				(ulong) bpage->offset);
 
diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc
index eb2ef1f21b8..b7d55cec0c9 100644
--- a/storage/xtradb/fil/fil0crypt.cc
+++ b/storage/xtradb/fil/fil0crypt.cc
@@ -266,13 +266,8 @@ fil_space_read_crypt_data(
 	ulint		offset)	/*!< in: offset */
 {
 	if (memcmp(page + offset, EMPTY_PATTERN, MAGIC_SZ) == 0) {
-		/* crypt is not stored but create memory cache for
-		not system tablespace */
-		if (space != 0) {
-			return fil_space_create_crypt_data(FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY);
-		} else {
-			return NULL;
-		}
+		/* Crypt data is not stored. */
+		return NULL;
 	}
 
 	if (memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) {
@@ -288,12 +283,8 @@ fil_space_read_crypt_data(
 			page[offset + 3],
 			page[offset + 4],
 			page[offset + 5]);
-		/* Create memory cache for not system tablespace */
-		if (space != 0) {
-			return fil_space_create_crypt_data(FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY);
-		} else {
-			return NULL;
-		}
+		/* Crypt data is not stored. */
+		return NULL;
 	}
 
 	ulint type = mach_read_from_1(page + offset + MAGIC_SZ + 0);
@@ -461,12 +452,6 @@ fil_space_write_crypt_data(
 		return;
 	}
 
-	/* If tablespace encryption is disabled and encryption mode is
-	DEFAULT, then do not continue writing crypt data to page 0. */
-	if (!srv_encrypt_tables && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT) {
-		return;
-	}
-
 	fil_space_write_crypt_data_low(crypt_data, crypt_data->type,
 				       page, offset, maxsize, mtr);
 }
@@ -1073,7 +1058,7 @@ fil_crypt_start_encrypting_space(
 	crypt_data->rotate_state.active_threads = 1;
 
 	mutex_enter(&crypt_data->mutex);
-	fil_space_set_crypt_data(space, crypt_data);
+	crypt_data = fil_space_set_crypt_data(space, crypt_data);
 	mutex_exit(&crypt_data->mutex);
 
 	fil_crypt_start_converting = true;
@@ -1108,6 +1093,7 @@ fil_crypt_start_encrypting_space(
 		/* 3 - compute location to store crypt data */
 		byte* frame = buf_block_get_frame(block);
 		ulint maxsize;
+		ut_ad(crypt_data);
 		crypt_data->page0_offset =
 			fsp_header_get_crypt_offset(zip_size, &maxsize);
 
@@ -1160,6 +1146,7 @@ fil_crypt_start_encrypting_space(
 
 		/* 5 - publish crypt data */
 		mutex_enter(&fil_crypt_threads_mutex);
+		ut_ad(crypt_data);
 		mutex_enter(&crypt_data->mutex);
 		crypt_data->type = CRYPT_SCHEME_1;
 		ut_a(crypt_data->rotate_state.active_threads == 1);
@@ -1173,6 +1160,7 @@ fil_crypt_start_encrypting_space(
 		return pending_op;
 	} while (0);
 
+	ut_ad(crypt_data);
 	mutex_enter(&crypt_data->mutex);
 	ut_a(crypt_data->rotate_state.active_threads == 1);
 	crypt_data->rotate_state.active_threads = 0;
diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc
index 73e6f514216..9336abf3656 100644
--- a/storage/xtradb/fil/fil0fil.cc
+++ b/storage/xtradb/fil/fil0fil.cc
@@ -6982,7 +6982,7 @@ fil_space_get_crypt_data(
 /******************************************************************
 Get crypt data for a tablespace */
 UNIV_INTERN
-void
+fil_space_crypt_t*
 fil_space_set_crypt_data(
 /*==================*/
 	ulint id, 	               /*!< in: space id */
@@ -6990,6 +6990,7 @@ fil_space_set_crypt_data(
 {
 	fil_space_t*	space;
 	fil_space_crypt_t* free_crypt_data = NULL;
+	fil_space_crypt_t* ret_crypt_data = NULL;
 
 	ut_ad(fil_system);
 
@@ -7008,9 +7009,11 @@ fil_space_set_crypt_data(
 			mutex_exit(&fil_system->mutex);
 			fil_space_merge_crypt_data(space->crypt_data,
 						   crypt_data);
+			ret_crypt_data = space->crypt_data;
 			free_crypt_data = crypt_data;
 		} else {
 			space->crypt_data = crypt_data;
+			ret_crypt_data = space->crypt_data;
 			mutex_exit(&fil_system->mutex);
 		}
 	} else {
@@ -7026,4 +7029,6 @@ fil_space_set_crypt_data(
 		*/
 		fil_space_destroy_crypt_data(&free_crypt_data);
 	}
+
+	return ret_crypt_data;
 }
diff --git a/storage/xtradb/include/fil0crypt.h b/storage/xtradb/include/fil0crypt.h
index 6dedd9c7f65..c6b3a626e8f 100644
--- a/storage/xtradb/include/fil0crypt.h
+++ b/storage/xtradb/include/fil0crypt.h
@@ -134,7 +134,7 @@ fil_space_get_crypt_data(
 /*********************************************************************
 Set crypt data for a space*/
 UNIV_INTERN
-void
+fil_space_crypt_t*
 fil_space_set_crypt_data(
 /*=====================*/
 	ulint space,                    /*!< in: tablespace id */