MDEV-11455 shutdown or abort during innodb buffer pool load (from file) causing incomplete save

Merge pull request #622

9 files changed, 310 insertions, 12 deletions

diff --git a/mysql-test/suite/sys_vars/r/innodb_buffer_pool_dump_abort_loads.result b/mysql-test/suite/sys_vars/r/innodb_buffer_pool_dump_abort_loads.result
new file mode 100644
index 00000000000..1e2beea707e
--- /dev/null
+++ b/mysql-test/suite/sys_vars/r/innodb_buffer_pool_dump_abort_loads.result
@@ -0,0 +1,95 @@
+
+# innodb_buffer_pool_load_incomplete defaults 0
+SELECT variable_name, variable_value
+FROM information_schema.global_status
+WHERE LOWER(variable_name) = 'innodb_buffer_pool_load_incomplete';
+variable_name	variable_value
+INNODB_BUFFER_POOL_LOAD_INCOMPLETE	OFF
+
+# populate with data
+CREATE TABLE t1 (
+c01 blob, c02 blob, c03 blob, c04 blob, c05 blob,
+c06 blob, c07 blob, c08 blob, c09 blob, c10 blob,
+c11 blob, c12 blob, c13 blob, c14 blob, c15 blob,
+c16 blob, c17 blob, c18 blob, c19 blob, c20 blob,
+c21 blob, c22 blob, c23 blob, c24 blob, c25 blob,
+c26 blob, c27 blob, c28 blob, c29 blob, c30 blob,
+c31 blob, c32 blob, c33 blob, c34 blob, c35 blob,
+c36 blob, c37 blob, c38 blob, c39 blob, c40 blob,
+c41 blob, c42 blob, c43 blob, c44 blob, c45 blob,
+c46 blob, c47 blob, c48 blob, c49 blob, c50 blob,
+c51 blob, c52 blob, c53 blob, c54 blob, c55 blob,
+c56 blob, c57 blob, c58 blob, c59 blob, c60 blob,
+c61 blob, c62 blob, c63 blob, c64 blob
+) ROW_FORMAT=dynamic;
+SET @a = repeat('a', 16 * 1024);
+INSERT INTO t1 VALUES (@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a
+);
+SET GLOBAL innodb_buffer_pool_dump_now=1;
+
+# Restart server
+
+# Abort after 16 pages
+SET GLOBAL innodb_buffer_pool_load_pages_abort=16,
+GLOBAL innodb_buffer_pool_load_now=1,
+GLOBAL innodb_buffer_pool_dump_at_shutdown=1;
+SELECT variable_name, SUBSTR(variable_value, 1, 38) as VALUE
+FROM information_schema.global_status
+WHERE LOWER(variable_name) IN ('innodb_buffer_pool_load_incomplete','innodb_buffer_pool_load_status')
+ORDER BY variable_name;
+variable_name	VALUE
+INNODB_BUFFER_POOL_LOAD_INCOMPLETE	ON
+INNODB_BUFFER_POOL_LOAD_STATUS	Buffer pool(s) load aborted on request
+
+# Restart server
+
+# Load buffer pool
+SET GLOBAL innodb_buffer_pool_load_now=1;
+
+# Should be more than previous as we didn't overwrite our save file
+select count(*) > Previous_loaded as Loaded_more from information_schema.INNODB_BUFFER_PAGE WHERE PAGE_TYPE='BLOB' group by PAGE_TYPE;;
+Loaded_more
+1
+
+# Successful, so innodb_buffer_pool_load_incomplete should be FALSE
+SELECT variable_name, SUBSTR(variable_value, 1, 33) as VALUE
+FROM information_schema.global_status
+WHERE LOWER(variable_name) IN ('innodb_buffer_pool_load_incomplete','innodb_buffer_pool_load_status')
+ORDER BY variable_name;
+variable_name	VALUE
+INNODB_BUFFER_POOL_LOAD_INCOMPLETE	OFF
+INNODB_BUFFER_POOL_LOAD_STATUS	Buffer pool(s) load completed at 
+
+# innodb_buffer_pool_dump_now=1 should reset the innodb_buffer_pool_load_incomplete status
+SET GLOBAL innodb_buffer_pool_dump_now=1;
+SELECT variable_name, SUBSTR(variable_value, 1, 33) as VALUE
+FROM information_schema.global_status
+WHERE LOWER(variable_name) IN ('innodb_buffer_pool_load_incomplete', 'innodb_buffer_pool_dump_status');
+variable_name	VALUE
+INNODB_BUFFER_POOL_DUMP_STATUS	Buffer pool(s) dump completed at 
+INNODB_BUFFER_POOL_LOAD_INCOMPLETE	OFF
+
+# Restart server
+
+# Load buffer pool
+SET GLOBAL innodb_buffer_pool_load_now=1;
+
+# Should be same amount
+select abs(Previously_dumped - count(*)) <= 2 as Loaded_about_same_size from information_schema.INNODB_BUFFER_PAGE WHERE PAGE_TYPE='BLOB' group by PAGE_TYPE;;
+Loaded_about_same_size
+1
+
+# Clean up
+DROP TABLE t1;
diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result
index ac0d4f27aef..ab66efb13d4 100644
--- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result
+++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result
@@ -4,7 +4,8 @@ variable_name not in (
 'innodb_disallow_writes',           # only available WITH_WSREP
 'innodb_numa_interleave',           # only available WITH_NUMA
 'innodb_sched_priority_cleaner',    # linux only
-'innodb_use_native_aio')            # default value depends on OS
+'innodb_use_native_aio',            # default value depends on OS
+'innodb_buffer_pool_load_pages_abort')            # debug build only, and is only for testing
 order by variable_name;
 VARIABLE_NAME	INNODB_ADAPTIVE_FLUSHING
 SESSION_VALUE	NULL
diff --git a/mysql-test/suite/sys_vars/t/innodb_buffer_pool_dump_abort_loads.opt b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_dump_abort_loads.opt
new file mode 100644
index 00000000000..787f6ce532d
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_dump_abort_loads.opt
@@ -0,0 +1,5 @@
+--loose-default-storage-engine=innodb
+--loose-innodb_buffer_pool_load_at_startup=0
+--loose-innodb_buffer_pool_dump_at_shutdown=0
+--loose-innodb-buffer-pool-size=8M
+--loose-innodb-page-size=16k
diff --git a/mysql-test/suite/sys_vars/t/innodb_buffer_pool_dump_abort_loads.test b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_dump_abort_loads.test
new file mode 100644
index 00000000000..4a4411101e2
--- /dev/null
+++ b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_dump_abort_loads.test
@@ -0,0 +1,149 @@
+#
+# MDEV-11455 - add status variable innodb_buffer_pool_load_abort
+#
+--source include/have_innodb.inc
+--source include/have_debug.inc
+
+--echo
+--echo # innodb_buffer_pool_load_incomplete defaults 0
+SELECT variable_name, variable_value
+    FROM information_schema.global_status
+    WHERE LOWER(variable_name) = 'innodb_buffer_pool_load_incomplete';
+
+--echo
+--echo # populate with data
+
+CREATE TABLE t1 (
+c01 blob, c02 blob, c03 blob, c04 blob, c05 blob,
+c06 blob, c07 blob, c08 blob, c09 blob, c10 blob,
+c11 blob, c12 blob, c13 blob, c14 blob, c15 blob,
+c16 blob, c17 blob, c18 blob, c19 blob, c20 blob,
+c21 blob, c22 blob, c23 blob, c24 blob, c25 blob,
+c26 blob, c27 blob, c28 blob, c29 blob, c30 blob,
+c31 blob, c32 blob, c33 blob, c34 blob, c35 blob,
+c36 blob, c37 blob, c38 blob, c39 blob, c40 blob,
+c41 blob, c42 blob, c43 blob, c44 blob, c45 blob,
+c46 blob, c47 blob, c48 blob, c49 blob, c50 blob,
+c51 blob, c52 blob, c53 blob, c54 blob, c55 blob,
+c56 blob, c57 blob, c58 blob, c59 blob, c60 blob,
+c61 blob, c62 blob, c63 blob, c64 blob
+) ROW_FORMAT=dynamic;
+
+SET @a = repeat('a', 16 * 1024);
+INSERT INTO t1 VALUES (@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a,@a,
+@a,@a,@a,@a
+);
+
+SET GLOBAL innodb_buffer_pool_dump_now=1;
+# Wait for for the dump to complete
+let $wait_condition =
+  SELECT SUBSTR(variable_value, 1, 33) = 'Buffer pool(s) dump completed at '
+    FROM information_schema.global_status
+    WHERE LOWER(variable_name) = 'innodb_buffer_pool_dump_status';
+-- source include/wait_condition.inc
+
+--echo
+--echo # Restart server
+--source include/restart_mysqld.inc
+
+--echo
+--echo # Abort after 16 pages
+SET GLOBAL innodb_buffer_pool_load_pages_abort=16,
+    GLOBAL innodb_buffer_pool_load_now=1,
+    GLOBAL innodb_buffer_pool_dump_at_shutdown=1;
+
+let $wait_condition =
+  SELECT SUBSTR(variable_value, 1, 19) = 'Buffer pool(s) load'
+    FROM information_schema.global_status
+    WHERE LOWER(variable_name) = 'innodb_buffer_pool_load_status';
+-- source include/wait_condition.inc
+
+SELECT variable_name, SUBSTR(variable_value, 1, 38) as VALUE
+    FROM information_schema.global_status
+    WHERE LOWER(variable_name) IN ('innodb_buffer_pool_load_incomplete','innodb_buffer_pool_load_status')
+    ORDER BY variable_name;
+
+--let $incomplete=`select count(*) as BLOB_PAGES from information_schema.INNODB_BUFFER_PAGE WHERE PAGE_TYPE='BLOB' group by PAGE_TYPE`
+
+# Shouldn't dump at shutdown due to innodb_buffer_pool_load_incomplete
+
+--echo
+--echo # Restart server
+--source include/restart_mysqld.inc
+
+--echo
+--echo # Load buffer pool
+SET GLOBAL innodb_buffer_pool_load_now=1;
+
+# Wait for for the load to complete
+let $wait_condition =
+  SELECT SUBSTR(variable_value, 1, 33) = 'Buffer pool(s) load completed at '
+    FROM information_schema.global_status
+    WHERE LOWER(variable_name) = 'innodb_buffer_pool_load_status';
+-- source include/wait_condition.inc
+
+--echo
+--echo # Should be more than previous as we didn't overwrite our save file
+--replace_result $incomplete Previous_loaded
+--eval select count(*) > $incomplete as Loaded_more from information_schema.INNODB_BUFFER_PAGE WHERE PAGE_TYPE='BLOB' group by PAGE_TYPE;
+
+--echo
+--echo # Successful, so innodb_buffer_pool_load_incomplete should be FALSE
+SELECT variable_name, SUBSTR(variable_value, 1, 33) as VALUE
+    FROM information_schema.global_status
+    WHERE LOWER(variable_name) IN ('innodb_buffer_pool_load_incomplete','innodb_buffer_pool_load_status')
+    ORDER BY variable_name;
+
+--echo
+--echo # innodb_buffer_pool_dump_now=1 should reset the innodb_buffer_pool_load_incomplete status
+
+SET GLOBAL innodb_buffer_pool_dump_now=1;
+# Wait for for the dump to complete
+let $wait_condition =
+  SELECT SUBSTR(variable_value, 1, 33) = 'Buffer pool(s) dump completed at '
+    FROM information_schema.global_status
+    WHERE LOWER(variable_name) = 'innodb_buffer_pool_dump_status';
+-- source include/wait_condition.inc
+
+SELECT variable_name, SUBSTR(variable_value, 1, 33) as VALUE
+    FROM information_schema.global_status
+    WHERE LOWER(variable_name) IN ('innodb_buffer_pool_load_incomplete', 'innodb_buffer_pool_dump_status');
+
+--let $fulldump=`select count(*) as BLOB_PAGES from information_schema.INNODB_BUFFER_PAGE WHERE PAGE_TYPE='BLOB' group by PAGE_TYPE`
+
+--echo
+--echo # Restart server
+--source include/restart_mysqld.inc
+
+--echo
+--echo # Load buffer pool
+SET GLOBAL innodb_buffer_pool_load_now=1;
+
+# Wait for for the load to complete
+let $wait_condition =
+  SELECT SUBSTR(variable_value, 1, 33) = 'Buffer pool(s) load completed at '
+    FROM information_schema.global_status
+    WHERE LOWER(variable_name) = 'innodb_buffer_pool_load_status';
+-- source include/wait_condition.inc
+
+--echo
+--echo # Should be same amount
+--replace_result $fulldump Previously_dumped
+--eval select abs($fulldump - count(*)) <= 2 as Loaded_about_same_size from information_schema.INNODB_BUFFER_PAGE WHERE PAGE_TYPE='BLOB' group by PAGE_TYPE;
+
+--echo
+--echo # Clean up
+
+--remove_file $MYSQLTEST_VARDIR/mysqld.1/data/ib_buffer_pool
+DROP TABLE t1;
diff --git a/mysql-test/suite/sys_vars/t/sysvars_innodb.test b/mysql-test/suite/sys_vars/t/sysvars_innodb.test
index 2abafda34c6..fe91835eac5 100644
--- a/mysql-test/suite/sys_vars/t/sysvars_innodb.test
+++ b/mysql-test/suite/sys_vars/t/sysvars_innodb.test
@@ -16,5 +16,6 @@ select * from information_schema.system_variables
     'innodb_disallow_writes',           # only available WITH_WSREP
     'innodb_numa_interleave',           # only available WITH_NUMA
     'innodb_sched_priority_cleaner',    # linux only
-    'innodb_use_native_aio')            # default value depends on OS
+    'innodb_use_native_aio',            # default value depends on OS
+    'innodb_buffer_pool_load_pages_abort')            # debug build only, and is only for testing
   order by variable_name;
diff --git a/storage/innobase/buf/buf0dump.cc b/storage/innobase/buf/buf0dump.cc
index 6cd8111e902..ea660ad3a50 100644
--- a/storage/innobase/buf/buf0dump.cc
+++ b/storage/innobase/buf/buf0dump.cc
@@ -413,6 +413,11 @@ buf_dump(
 
 	buf_dump_status(STATUS_INFO,
 			"Buffer pool(s) dump completed at %s", now);
+
+	/* Though dumping doesn't related to an incomplete load,
+	 we reset this to 0 here to indicate that a shutdown can also perform
+	 a dump */
+	export_vars.innodb_buffer_pool_load_incomplete = 0;
 }
 
 /*****************************************************************//**
@@ -576,6 +581,8 @@ buf_load()
 
 	rewind(f);
 
+	export_vars.innodb_buffer_pool_load_incomplete = 1;
+
 	for (i = 0; i < dump_n && !SHUTTING_DOWN(); i++) {
 		fscanf_ret = fscanf(f, ULINTPF "," ULINTPF,
 				    &space_id, &page_no);
@@ -624,7 +631,7 @@ buf_load()
 		ut_sprintf_timestamp(now);
 		buf_load_status(STATUS_INFO,
 				"Buffer pool(s) load completed at %s"
-				" (%s was empty)", now, full_filename);
+				" (%s was empty or had errors)", now, full_filename);
 		return;
 	}
 
@@ -713,6 +720,12 @@ buf_load()
 
 		buf_load_throttle_if_needed(
 			&last_check_time, &last_activity_cnt, i);
+
+#ifdef UNIV_DEBUG
+		if ((i+1) >= srv_buf_pool_load_pages_abort) {
+			buf_load_abort_flag = 1;
+		}
+#endif
 	}
 
 	if (space != NULL) {
@@ -723,8 +736,23 @@ buf_load()
 
 	ut_sprintf_timestamp(now);
 
-	buf_load_status(STATUS_INFO,
+	if (i == dump_n) {
+		buf_load_status(STATUS_INFO,
 			"Buffer pool(s) load completed at %s", now);
+		export_vars.innodb_buffer_pool_load_incomplete = 0;
+	} else if (!buf_load_abort_flag) {
+		buf_load_status(STATUS_INFO,
+			"Buffer pool(s) load aborted due to user instigated abort at %s",
+			now);
+		/* intentionally don't reset innodb_buffer_pool_load_incomplete
+                   as we don't want a shutdown to save the buffer pool */
+	} else {
+		buf_load_status(STATUS_INFO,
+			"Buffer pool(s) load aborted due to shutdown at %s",
+			now);
+		/* intentionally don't reset innodb_buffer_pool_load_incomplete
+                   as we want to abort without saving the buffer pool */
+	}
 
 	/* Make sure that estimated = completed when we end. */
 	/* mysql_stage_set_work_completed(pfs_stage_progress, dump_n); */
@@ -793,15 +821,16 @@ DECLARE_THREAD(buf_dump_thread)(void*)
 	}
 
 	if (srv_buffer_pool_dump_at_shutdown && srv_fast_shutdown != 2) {
+		if (export_vars.innodb_buffer_pool_load_incomplete) {
+			buf_dump_status(STATUS_INFO,
+				"Dumping of buffer pool not started"
+				" as load was incomplete");
 #ifdef WITH_WSREP
-		if (!wsrep_recovery) {
+		} else if (wsrep_recovery) {
 #endif /* WITH_WSREP */
-
-		buf_dump(FALSE /* ignore shutdown down flag,
-		keep going even if we are in a shutdown state */);
-#ifdef WITH_WSREP
+		} else {
+			buf_dump(FALSE/* do complete dump at shutdown */);
 		}
-#endif /* WITH_WSREP */
 	}
 
 	srv_buf_dump_thread_active = false;
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index 699e2899f57..e14463ca909 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -930,6 +930,8 @@ static SHOW_VAR innodb_status_variables[]= {
   (char*) &export_vars.innodb_buffer_pool_load_status,	  SHOW_CHAR},
   {"buffer_pool_resize_status",
   (char*) &export_vars.innodb_buffer_pool_resize_status,  SHOW_CHAR},
+  {"buffer_pool_load_incomplete",
+  &export_vars.innodb_buffer_pool_load_incomplete,        SHOW_BOOL},
   {"buffer_pool_pages_data",
   (char*) &export_vars.innodb_buffer_pool_pages_data,	  SHOW_LONG},
   {"buffer_pool_bytes_data",
@@ -20157,6 +20159,12 @@ static MYSQL_SYSVAR_ULONG(buffer_pool_dump_pct, srv_buf_pool_dump_pct,
   NULL, NULL, 25, 1, 100, 0);
 
 #ifdef UNIV_DEBUG
+/* Added to test the innodb_buffer_pool_load_incomplete status variable. */
+static MYSQL_SYSVAR_ULONG(buffer_pool_load_pages_abort, srv_buf_pool_load_pages_abort,
+  PLUGIN_VAR_RQCMDARG,
+  "Number of pages during a buffer pool load to process before signaling innodb_buffer_pool_load_abort=1",
+  NULL, NULL, LONG_MAX, 1, LONG_MAX, 0);
+
 static MYSQL_SYSVAR_STR(buffer_pool_evict, srv_buffer_pool_evict,
   PLUGIN_VAR_RQCMDARG,
   "Evict pages from the buffer pool",
@@ -20901,6 +20909,9 @@ static struct st_mysql_sys_var* innobase_system_variables[]= {
 #endif /* UNIV_DEBUG */
   MYSQL_SYSVAR(buffer_pool_load_now),
   MYSQL_SYSVAR(buffer_pool_load_abort),
+#ifdef UNIV_DEBUG
+  MYSQL_SYSVAR(buffer_pool_load_pages_abort),
+#endif /* UNIV_DEBUG */
   MYSQL_SYSVAR(buffer_pool_load_at_startup),
   MYSQL_SYSVAR(defragment),
   MYSQL_SYSVAR(defragment_n_pages),
diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h
index 9f37b78ac5d..be29b184387 100644
--- a/storage/innobase/include/srv0srv.h
+++ b/storage/innobase/include/srv0srv.h
@@ -381,8 +381,6 @@ extern ulong	srv_n_page_hash_locks;
 /** Scan depth for LRU flush batch i.e.: number of blocks scanned*/
 extern ulong	srv_LRU_scan_depth;
 /** Whether or not to flush neighbors of a block */
-extern ulong	srv_buf_pool_dump_pct;	/*!< dump that may % of each buffer
-					pool during BP dump */
 extern ulong	srv_flush_neighbors;
 /** Previously requested size */
 extern ulint	srv_buf_pool_old_size;
@@ -392,6 +390,10 @@ extern ulint	srv_buf_pool_base_size;
 extern ulint	srv_buf_pool_curr_size;
 /** Dump this % of each buffer pool during BP dump */
 extern ulong	srv_buf_pool_dump_pct;
+#ifdef UNIV_DEBUG
+/** Abort load after this amount of pages */
+extern ulong srv_buf_pool_load_pages_abort;
+#endif
 /** Lock table size in bytes */
 extern ulint	srv_lock_table_size;
 
@@ -946,6 +948,7 @@ struct export_var_t{
 	char  innodb_buffer_pool_dump_status[OS_FILE_MAX_PATH + 128];/*!< Buf pool dump status */
 	char  innodb_buffer_pool_load_status[OS_FILE_MAX_PATH + 128];/*!< Buf pool load status */
 	char  innodb_buffer_pool_resize_status[512];/*!< Buf pool resize status */
+	my_bool innodb_buffer_pool_load_incomplete;/*!< Buf pool load incomplete */
 	ulint innodb_buffer_pool_pages_total;	/*!< Buffer pool size */
 	ulint innodb_buffer_pool_pages_data;	/*!< Data pages */
 	ulint innodb_buffer_pool_bytes_data;	/*!< File bytes used */
diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc
index cd3fad2c678..2b32deebb21 100644
--- a/storage/innobase/srv/srv0srv.cc
+++ b/storage/innobase/srv/srv0srv.cc
@@ -246,6 +246,10 @@ ulint	srv_buf_pool_base_size;
 ulint	srv_buf_pool_curr_size;
 /** Dump this % of each buffer pool during BP dump */
 ulong	srv_buf_pool_dump_pct;
+/** Abort load after this amount of pages */
+#ifdef UNIV_DEBUG
+ulong srv_buf_pool_load_pages_abort = LONG_MAX;
+#endif
 /** Lock table size in bytes */
 ulint	srv_lock_table_size	= ULINT_MAX;