MDEV-12610: MariaDB start is slow

Problem appears to be that the function fsp_flags_try_adjust()
is being unconditionally invoked on every .ibd file on startup.
Based on performance investigation also the top function
fsp_header_get_crypt_offset() needs to addressed.

Ported implementation of fsp_header_get_encryption_offset()
function from 10.2 to fsp_header_get_crypt_offset().

Introduced a new function fil_crypt_read_crypt_data()
to read page 0 if it is not yet read.

fil_crypt_find_space_to_rotate(): Now that page 0 for every .ibd
file is not read on startup we need to check has page 0 read
from space that we investigate for key rotation, if it is not read
we read it.

fil_space_crypt_get_status(): Now that page 0 for every .ibd
file is not read on startup here also we need to read page 0
if it is not yet read it. This is needed
as tests use IS query to wait until background encryption
or decryption has finished and this function is used to
produce results.

fil_crypt_thread(): Add is_stopping condition for tablespace
so that we do not rotate pages if usage of tablespace should
be stopped. This was needed for failure seen on regression
testing.

fil_space_create: Remove page_0_crypt_read and extra
unnecessary info output.

fil_open_single_table_tablespace(): We call fsp_flags_try_adjust
only when when no errors has happened and server was not started
on read only mode and tablespace validation was requested or
flags contain other table options except low order bits to
FSP_FLAGS_POS_PAGE_SSIZE position.

fil_space_t::page_0_crypt_read removed.

Added test case innodb-first-page-read to test startup when
encryption is on and when encryption is off to check that not
for all tables page 0 is read on startup.

10 files changed, 154 insertions, 90 deletions

diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc
index b4ee3ebc067..334b2fbddb5 100644
--- a/storage/innobase/buf/buf0buf.cc
+++ b/storage/innobase/buf/buf0buf.cc
@@ -4549,6 +4549,7 @@ buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space)
 		!bpage->encrypted &&
 		fil_space_verify_crypt_checksum(dst_frame, zip_size,
 			space, bpage->offset));
+
 	if (!still_encrypted) {
 		/* If traditional checksums match, we assume that page is
 		not anymore encrypted. */
diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc
index 2131a936656..70d8558ede2 100644
--- a/storage/innobase/fil/fil0crypt.cc
+++ b/storage/innobase/fil/fil0crypt.cc
@@ -1115,6 +1115,43 @@ fil_crypt_needs_rotation(
 	return false;
 }
 
+/** Read page 0 and possible crypt data from there.
+@param[in]	space		Tablespace */
+static inline
+void
+fil_crypt_read_crypt_data(fil_space_t* space)
+{
+	mutex_enter(&fil_system->mutex);
+
+	/* If space does not contain crypt data and space size is 0
+	we have not yet read first page of tablespace. We need to
+	read it to find out tablespace current encryption status. */
+	if (!space->crypt_data && space->size == 0) {
+		mtr_t	mtr;
+		mtr_start(&mtr);
+		ulint zip_size = fsp_flags_get_zip_size(space->flags);
+		ulint offset = fsp_header_get_crypt_offset(zip_size);
+		mutex_exit(&fil_system->mutex);
+		if (buf_block_t* block = buf_page_get(space->id, zip_size, 0,
+				RW_X_LATCH, &mtr)) {
+			byte* frame = buf_block_get_frame(block);
+
+			mutex_enter(&fil_system->mutex);
+
+			if (!space->crypt_data) {
+				space->crypt_data = fil_space_read_crypt_data(space->id,
+					frame, offset);
+			}
+
+			mutex_exit(&fil_system->mutex);
+		}
+
+		mtr_commit(&mtr);
+	} else {
+		mutex_exit(&fil_system->mutex);
+	}
+}
+
 /***********************************************************************
 Start encrypting a space
 @param[in,out]		space		Tablespace
@@ -1125,6 +1162,7 @@ fil_crypt_start_encrypting_space(
 	fil_space_t*	space)
 {
 	bool recheck = false;
+
 	mutex_enter(&fil_crypt_threads_mutex);
 
 	fil_space_crypt_t *crypt_data = space->crypt_data;
@@ -1191,8 +1229,6 @@ fil_crypt_start_encrypting_space(
 		byte* frame = buf_block_get_frame(block);
 		crypt_data->type = CRYPT_SCHEME_1;
 		crypt_data->write_page0(frame, &mtr);
-
-
 		mtr_commit(&mtr);
 
 		/* record lsn of update */
@@ -1620,6 +1656,13 @@ fil_crypt_find_space_to_rotate(
 	}
 
 	while (!state->should_shutdown() && state->space) {
+		/* If there is no crypt data and we have not yet read
+		page 0 for this tablespace, we need to read it before
+		we can continue. */
+		if (!state->space->crypt_data) {
+			fil_crypt_read_crypt_data(state->space);
+		}
+
 		if (fil_crypt_space_needs_rotation(state, key_state, recheck)) {
 			ut_ad(key_state->key_id);
 			/* init state->min_key_version_found before
@@ -2314,8 +2357,10 @@ DECLARE_THREAD(fil_crypt_thread)(
 			while (!thr.should_shutdown() &&
 			       fil_crypt_find_page_to_rotate(&new_state, &thr)) {
 
-				/* rotate a (set) of pages */
-				fil_crypt_rotate_pages(&new_state, &thr);
+				if (!thr.space->is_stopping()) {
+					/* rotate a (set) of pages */
+					fil_crypt_rotate_pages(&new_state, &thr);
+				}
 
 				/* If space is marked as stopping, release
 				space and stop rotation. */
@@ -2544,6 +2589,14 @@ fil_space_crypt_get_status(
 	memset(status, 0, sizeof(*status));
 
 	ut_ad(space->n_pending_ops > 0);
+
+	/* If there is no crypt data and we have not yet read
+	page 0 for this tablespace, we need to read it before
+	we can continue. */
+	if (!space->crypt_data) {
+		fil_crypt_read_crypt_data(const_cast<fil_space_t*>(space));
+	}
+
 	fil_space_crypt_t* crypt_data = space->crypt_data;
 	status->space = space->id;
 
diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc
index f88bb2add59..ffb01312fdb 100644
--- a/storage/innobase/fil/fil0fil.cc
+++ b/storage/innobase/fil/fil0fil.cc
@@ -653,12 +653,10 @@ fil_node_open_file(
 
 		/* Try to read crypt_data from page 0 if it is not yet
 		read. */
-		if (!node->space->page_0_crypt_read) {
-			ulint offset = fsp_header_get_crypt_offset(
-				fsp_flags_get_zip_size(flags));
-			ut_ad(node->space->crypt_data == NULL);
+		if (!node->space->crypt_data) {
+			const ulint offset = fsp_header_get_crypt_offset(
+					fsp_flags_get_zip_size(flags));
 			node->space->crypt_data = fil_space_read_crypt_data(space_id, page, offset);
-			node->space->page_0_crypt_read = true;
 		}
 
 		ut_free(buf2);
@@ -1557,22 +1555,6 @@ fil_space_create(
 	space->magic_n = FIL_SPACE_MAGIC_N;
 	space->crypt_data = crypt_data;
 
-	/* In create table we write page 0 so we have already
-	"read" it and for system tablespaces we have read
-	crypt data at startup. */
-	if (create_table || crypt_data != NULL) {
-		space->page_0_crypt_read = true;
-	}
-
-#ifdef UNIV_DEBUG
-	ib_logf(IB_LOG_LEVEL_INFO,
-		"Created tablespace for space %lu name %s key_id %u encryption %d.",
-		space->id,
-		space->name,
-		space->crypt_data ? space->crypt_data->key_id : 0,
-		space->crypt_data ? space->crypt_data->encryption : 0);
-#endif
-
 	rw_lock_create(fil_space_latch_key, &space->latch, SYNC_FSP);
 
 	HASH_INSERT(fil_space_t, hash, fil_system->spaces, id, space);
@@ -2401,8 +2383,8 @@ fil_read_first_page(
 		/* Possible encryption crypt data is also stored only to first page
 		of the first datafile. */
 
-		ulint offset = fsp_header_get_crypt_offset(
-			fsp_flags_get_zip_size(*flags));
+		const ulint offset = fsp_header_get_crypt_offset(
+					fsp_flags_get_zip_size(*flags));
 
 		cdata = fil_space_read_crypt_data(*space_id, page, offset);
 
@@ -4018,6 +4000,7 @@ fsp_flags_try_adjust(ulint space_id, ulint flags)
 					 flags, MLOG_4BYTES, &mtr);
 		}
 	}
+
 	mtr_commit(&mtr);
 }
 
@@ -4437,7 +4420,17 @@ cleanup_and_exit:
 
 	mem_free(def.filepath);
 
-	if (err == DB_SUCCESS && !srv_read_only_mode) {
+	/* We need to check fsp flags when no errors has happened and
+	server was not started on read only mode and tablespace validation
+	was requested or flags contain other table options except
+	low order bits to FSP_FLAGS_POS_PAGE_SSIZE position.
+	Note that flag comparison is pessimistic. Adjust is required
+	only when flags contain buggy MariaDB 10.1.0 -
+	MariaDB 10.1.20 flags.  */
+	if (err == DB_SUCCESS
+	    && !srv_read_only_mode
+	    && (validate
+		|| flags >= (1U << FSP_FLAGS_POS_PAGE_SSIZE))) {
 		fsp_flags_try_adjust(id, flags & ~FSP_FLAGS_MEM_MASK);
 	}
 
diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc
index 4f05549bc1c..878b8d824c7 100644
--- a/storage/innobase/fsp/fsp0fsp.cc
+++ b/storage/innobase/fsp/fsp0fsp.cc
@@ -4124,20 +4124,8 @@ ulint
 fsp_header_get_crypt_offset(
 	const ulint   zip_size)
 {
-	ulint pageno = 0;
-	/* compute first page_no that will have xdes stored on page != 0*/
-	for (ulint i = 0;
-	     (pageno = xdes_calc_descriptor_page(zip_size, i)) == 0; )
-		i++;
-
-	/* use pageno prior to this...i.e last page on page 0 */
-	ut_ad(pageno > 0);
-	pageno--;
-
-	ulint iv_offset = XDES_ARR_OFFSET +
-		XDES_SIZE * (1 + xdes_calc_descriptor_index(zip_size, pageno));
-
-	return FSP_HEADER_OFFSET + iv_offset;
+	return (FSP_HEADER_OFFSET + (XDES_ARR_OFFSET + XDES_SIZE *
+			(zip_size ? zip_size : UNIV_PAGE_SIZE) / FSP_EXTENT_SIZE));
 }
 
 /**********************************************************************//**
diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h
index ba440cb2a1c..e16c7cb102e 100644
--- a/storage/innobase/include/fil0fil.h
+++ b/storage/innobase/include/fil0fil.h
@@ -351,9 +351,6 @@ struct fil_space_t {
 				compression failure */
 	fil_space_crypt_t* crypt_data;
 				/*!< tablespace crypt data or NULL */
-	bool		page_0_crypt_read;
-				/*!< tablespace crypt data has been
-				read */
 	ulint		file_block_size;
 				/*!< file system block size */
 
diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc
index c72900bd082..54f3ac311be 100644
--- a/storage/xtradb/buf/buf0buf.cc
+++ b/storage/xtradb/buf/buf0buf.cc
@@ -4636,6 +4636,7 @@ buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space)
 		!bpage->encrypted &&
 		fil_space_verify_crypt_checksum(dst_frame, zip_size,
 			space, bpage->offset));
+
 	if (!still_encrypted) {
 		/* If traditional checksums match, we assume that page is
 		not anymore encrypted. */
diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc
index 21c1e3b730e..e24278dd102 100644
--- a/storage/xtradb/fil/fil0crypt.cc
+++ b/storage/xtradb/fil/fil0crypt.cc
@@ -1115,6 +1115,43 @@ fil_crypt_needs_rotation(
 	return false;
 }
 
+/** Read page 0 and possible crypt data from there.
+@param[in]	space		Tablespace */
+static inline
+void
+fil_crypt_read_crypt_data(fil_space_t* space)
+{
+	mutex_enter(&fil_system->mutex);
+
+	/* If space does not contain crypt data and space size is 0
+	we have not yet read first page of tablespace. We need to
+	read it to find out tablespace current encryption status. */
+	if (!space->crypt_data && space->size == 0) {
+		mtr_t	mtr;
+		mtr_start(&mtr);
+		ulint zip_size = fsp_flags_get_zip_size(space->flags);
+		ulint offset = fsp_header_get_crypt_offset(zip_size);
+		mutex_exit(&fil_system->mutex);
+		if (buf_block_t* block = buf_page_get(space->id, zip_size, 0,
+				RW_X_LATCH, &mtr)) {
+			byte* frame = buf_block_get_frame(block);
+
+			mutex_enter(&fil_system->mutex);
+
+			if (!space->crypt_data) {
+				space->crypt_data = fil_space_read_crypt_data(space->id,
+					frame, offset);
+			}
+
+			mutex_exit(&fil_system->mutex);
+		}
+
+		mtr_commit(&mtr);
+	} else {
+		mutex_exit(&fil_system->mutex);
+	}
+}
+
 /***********************************************************************
 Start encrypting a space
 @param[in,out]		space		Tablespace
@@ -1125,6 +1162,7 @@ fil_crypt_start_encrypting_space(
 	fil_space_t*	space)
 {
 	bool recheck = false;
+
 	mutex_enter(&fil_crypt_threads_mutex);
 
 	fil_space_crypt_t *crypt_data = space->crypt_data;
@@ -1191,8 +1229,6 @@ fil_crypt_start_encrypting_space(
 		byte* frame = buf_block_get_frame(block);
 		crypt_data->type = CRYPT_SCHEME_1;
 		crypt_data->write_page0(frame, &mtr);
-
-
 		mtr_commit(&mtr);
 
 		/* record lsn of update */
@@ -1620,6 +1656,13 @@ fil_crypt_find_space_to_rotate(
 	}
 
 	while (!state->should_shutdown() && state->space) {
+		/* If there is no crypt data and we have not yet read
+		page 0 for this tablespace, we need to read it before
+		we can continue. */
+		if (!state->space->crypt_data) {
+			fil_crypt_read_crypt_data(state->space);
+		}
+
 		if (fil_crypt_space_needs_rotation(state, key_state, recheck)) {
 			ut_ad(key_state->key_id);
 			/* init state->min_key_version_found before
@@ -2314,8 +2357,10 @@ DECLARE_THREAD(fil_crypt_thread)(
 			while (!thr.should_shutdown() &&
 			       fil_crypt_find_page_to_rotate(&new_state, &thr)) {
 
-				/* rotate a (set) of pages */
-				fil_crypt_rotate_pages(&new_state, &thr);
+				if (!thr.space->is_stopping()) {
+					/* rotate a (set) of pages */
+					fil_crypt_rotate_pages(&new_state, &thr);
+				}
 
 				/* If space is marked as stopping, release
 				space and stop rotation. */
@@ -2545,6 +2590,14 @@ fil_space_crypt_get_status(
 	memset(status, 0, sizeof(*status));
 
 	ut_ad(space->n_pending_ops > 0);
+
+	/* If there is no crypt data and we have not yet read
+	page 0 for this tablespace, we need to read it before
+	we can continue. */
+	if (!space->crypt_data) {
+		fil_crypt_read_crypt_data(const_cast<fil_space_t*>(space));
+	}
+
 	fil_space_crypt_t* crypt_data = space->crypt_data;
 	status->space = space->id;
 
diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc
index d979c05c9a6..12048bc479f 100644
--- a/storage/xtradb/fil/fil0fil.cc
+++ b/storage/xtradb/fil/fil0fil.cc
@@ -661,12 +661,10 @@ fil_node_open_file(
 
 		/* Try to read crypt_data from page 0 if it is not yet
 		read. */
-		if (!node->space->page_0_crypt_read) {
-			ulint offset = fsp_header_get_crypt_offset(
-				fsp_flags_get_zip_size(flags));
-			ut_ad(node->space->crypt_data == NULL);
+		if (!node->space->crypt_data) {
+			const ulint offset = fsp_header_get_crypt_offset(
+					fsp_flags_get_zip_size(flags));
 			node->space->crypt_data = fil_space_read_crypt_data(space_id, page, offset);
-			node->space->page_0_crypt_read = true;
 		}
 
 		ut_free(buf2);
@@ -1600,22 +1598,6 @@ fil_space_create(
 	space->magic_n = FIL_SPACE_MAGIC_N;
 	space->crypt_data = crypt_data;
 
-	/* In create table we write page 0 so we have already
-	"read" it and for system tablespaces we have read
-	crypt data at startup. */
-	if (create_table || crypt_data != NULL) {
-		space->page_0_crypt_read = true;
-	}
-
-#ifdef UNIV_DEBUG
-	ib_logf(IB_LOG_LEVEL_INFO,
-		"Created tablespace for space %lu name %s key_id %u encryption %d.",
-		space->id,
-		space->name,
-		space->crypt_data ? space->crypt_data->key_id : 0,
-		space->crypt_data ? space->crypt_data->encryption : 0);
-#endif
-
 	rw_lock_create(fil_space_latch_key, &space->latch, SYNC_FSP);
 
 	HASH_INSERT(fil_space_t, hash, fil_system->spaces, id, space);
@@ -2463,8 +2445,8 @@ fil_read_first_page(
 		/* Possible encryption crypt data is also stored only to first page
 		of the first datafile. */
 
-		ulint offset = fsp_header_get_crypt_offset(
-			fsp_flags_get_zip_size(*flags));
+		const ulint offset = fsp_header_get_crypt_offset(
+					fsp_flags_get_zip_size(*flags));
 
 		cdata = fil_space_read_crypt_data(*space_id, page, offset);
 
@@ -4211,6 +4193,7 @@ fsp_flags_try_adjust(ulint space_id, ulint flags)
 					 flags, MLOG_4BYTES, &mtr);
 		}
 	}
+
 	mtr_commit(&mtr);
 }
 
@@ -4631,7 +4614,17 @@ cleanup_and_exit:
 
 	mem_free(def.filepath);
 
-	if (err == DB_SUCCESS && !srv_read_only_mode) {
+	/* We need to check fsp flags when no errors has happened and
+	server was not started on read only mode and tablespace validation
+	was requested or flags contain other table options except
+	low order bits to FSP_FLAGS_POS_PAGE_SSIZE position.
+	Note that flag comparison is pessimistic. Adjust is required
+	only when flags contain buggy MariaDB 10.1.0 -
+	MariaDB 10.1.20 flags.  */
+	if (err == DB_SUCCESS
+	    && !srv_read_only_mode
+	    && (validate
+		|| flags >= (1U << FSP_FLAGS_POS_PAGE_SSIZE))) {
 		fsp_flags_try_adjust(id, flags & ~FSP_FLAGS_MEM_MASK);
 	}
 
diff --git a/storage/xtradb/fsp/fsp0fsp.cc b/storage/xtradb/fsp/fsp0fsp.cc
index bd87b88f58d..40a9faa6914 100644
--- a/storage/xtradb/fsp/fsp0fsp.cc
+++ b/storage/xtradb/fsp/fsp0fsp.cc
@@ -4150,20 +4150,8 @@ ulint
 fsp_header_get_crypt_offset(
 	const ulint   zip_size)
 {
-	ulint pageno = 0;
-	/* compute first page_no that will have xdes stored on page != 0*/
-	for (ulint i = 0;
-	     (pageno = xdes_calc_descriptor_page(zip_size, i)) == 0; )
-		i++;
-
-	/* use pageno prior to this...i.e last page on page 0 */
-	ut_ad(pageno > 0);
-	pageno--;
-
-	ulint iv_offset = XDES_ARR_OFFSET +
-		XDES_SIZE * (1 + xdes_calc_descriptor_index(zip_size, pageno));
-
-	return FSP_HEADER_OFFSET + iv_offset;
+	return (FSP_HEADER_OFFSET + (XDES_ARR_OFFSET + XDES_SIZE *
+			(zip_size ? zip_size : UNIV_PAGE_SIZE) / FSP_EXTENT_SIZE));
 }
 
 /**********************************************************************//**
diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h
index b861225f562..a09833c3a73 100644
--- a/storage/xtradb/include/fil0fil.h
+++ b/storage/xtradb/include/fil0fil.h
@@ -350,9 +350,6 @@ struct fil_space_t {
 				compression failure */
 	fil_space_crypt_t* crypt_data;
 				/*!< tablespace crypt data or NULL */
-	bool		page_0_crypt_read;
-				/*!< tablespace crypt data has been
-				read */
 	ulint		file_block_size;
 				/*!< file system block size */