MDEV-20060: Failing assertion: srv_log_file_size <= 512ULL << 30 while preparing backup

The general reason why innodb redo log file is limited by 512G is that
log_block_convert_lsn_to_no() returns value limited by 1G. But there is no
need to have unique log block numbers in log group. The fix removes 512G
limit and limits log group size by
(uint32_t maximum value) * (minimum page size), which, in turns, can be
removed if fil_io() is no longer used for innodb redo log io.

10 files changed, 147 insertions, 16 deletions

diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc
index 524aed6cd5a..bc7149a87fd 100644
--- a/extra/mariabackup/xtrabackup.cc
+++ b/extra/mariabackup/xtrabackup.cc
@@ -1298,7 +1298,7 @@ struct my_option xb_server_options[] =
   {"innodb_log_file_size", OPT_INNODB_LOG_FILE_SIZE,
    "Ignored for mysqld option compatibility",
    (G_PTR*) &srv_log_file_size, (G_PTR*) &srv_log_file_size, 0,
-   GET_ULL, REQUIRED_ARG, 48 << 20, 1 << 20, 512ULL << 30, 0,
+   GET_ULL, REQUIRED_ARG, 48 << 20, 1 << 20, log_group_max_size, 0,
    UNIV_PAGE_SIZE_MAX, 0},
   {"innodb_log_files_in_group", OPT_INNODB_LOG_FILES_IN_GROUP,
    "Ignored for mysqld option compatibility",
diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl
index a29ae0b67ae..47f697ae0de 100755
--- a/mysql-test/mysql-test-run.pl
+++ b/mysql-test/mysql-test-run.pl
@@ -3333,6 +3333,7 @@ sub mysql_install_db {
   # Create the bootstrap.sql file
   # ----------------------------------------------------------------------
   my $bootstrap_sql_file= "$opt_vardir/log/bootstrap.sql";
+  $ENV{'MYSQL_BOOTSTRAP_SQL_FILE'}= $bootstrap_sql_file;
 
   if (! -e $bootstrap_sql_file)
   {
diff --git a/mysql-test/suite/mariabackup/big_innodb_log.result b/mysql-test/suite/mariabackup/big_innodb_log.result
new file mode 100644
index 00000000000..6577c0c9aae
--- /dev/null
+++ b/mysql-test/suite/mariabackup/big_innodb_log.result
@@ -0,0 +1,30 @@
+# Kill the server
+CREATE TABLE t(i INT) ENGINE InnoDB;
+INSERT INTO t VALUES
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9);
+# xtrabackup backup, execute the following query after test.t is copied:
+# BEGIN NOT ATOMIC INSERT INTO test.t SELECT * FROM test.t; UPDATE test.t SET i = 10 WHERE i = 0; DELETE FROM test.t WHERE i = 1; END
+SELECT count(*) FROM t WHERE i = 0;
+count(*)
+0
+# xtrabackup prepare
+# shutdown server
+# remove datadir
+# xtrabackup move back
+# restart server
+SELECT count(*) FROM t WHERE i = 0;
+count(*)
+0
+Ok
+Ok
+DROP TABLE t;
+# Kill the server
diff --git a/mysql-test/suite/mariabackup/big_innodb_log.test b/mysql-test/suite/mariabackup/big_innodb_log.test
new file mode 100644
index 00000000000..05dc3aa39c5
--- /dev/null
+++ b/mysql-test/suite/mariabackup/big_innodb_log.test
@@ -0,0 +1,87 @@
+# The general reason why innodb redo log file is limited by 512G is that
+# log_block_convert_lsn_to_no() returns value limited by 1G. But there is no
+# need to have unique log block numbers in log group. This test forces innodb
+# to generate redo log files with non-unique log block numbers and tests
+# recovery process with such numbers.
+--source include/have_innodb.inc
+--source include/have_debug.inc
+
+--let MYSQLD_DATADIR= `select @@datadir`
+let $MYSQLD_BOOTSTRAP_CMD= $MYSQLD_BOOTSTRAP_CMD --datadir=$MYSQLD_DATADIR --debug-dbug=+d,innodb_small_log_block_no_limit;
+
+--source include/kill_mysqld.inc
+--rmdir $MYSQLD_DATADIR
+--mkdir $MYSQLD_DATADIR
+--mkdir $MYSQLD_DATADIR/mysql
+--mkdir $MYSQLD_DATADIR/test
+--exec $MYSQLD_BOOTSTRAP_CMD < $MYSQL_BOOTSTRAP_SQL_FILE >> $MYSQLTEST_VARDIR/tmp/bootstrap.log 2>&1
+let $old_restart_parameters=$restart_parameters;
+let $restart_parameters= $old_restart_parameters --debug-dbug=+d,innodb_small_log_block_no_limit;
+--source include/start_mysqld.inc
+
+CREATE TABLE t(i INT) ENGINE InnoDB;
+INSERT INTO t VALUES
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9);
+
+--let after_copy_test_t=BEGIN NOT ATOMIC INSERT INTO test.t SELECT * FROM test.t; UPDATE test.t SET i = 10 WHERE i = 0; DELETE FROM test.t WHERE i = 1; END
+
+--echo # xtrabackup backup, execute the following query after test.t is copied:
+--echo # $after_copy_test_t
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --dbug=+d,mariabackup_events,innodb_small_log_block_no_limit;
+--enable_result_log
+
+--let $total_before=`SELECT count(*) FROM t`
+SELECT count(*) FROM t WHERE i = 0;
+--let $updated_before=`SELECT count(*) FROM t WHERE i = 10`
+
+echo # xtrabackup prepare;
+--disable_result_log
+exec $XTRABACKUP  --prepare --target-dir=$targetdir --dbug=+d,innodb_small_log_block_no_limit;
+--source include/restart_and_restore.inc
+--enable_result_log
+
+--let $total_after=`SELECT count(*) FROM t`
+SELECT count(*) FROM t WHERE i = 0;
+--let $updated_after=`SELECT count(*) FROM t WHERE i = 10`
+
+if ($total_before == $total_after) {
+--echo Ok
+}
+if ($total_before != $total_after) {
+--echo Failed
+}
+if ($updated_before == $updated_after) {
+--echo Ok
+}
+if ($updated_before != $updated_after) {
+--echo Failed
+}
+
+DROP TABLE t;
+rmdir $targetdir;
+--source include/kill_mysqld.inc
+--rmdir $MYSQLD_DATADIR
+
+perl;
+use lib "lib";
+use My::File::Path;
+my $install_db_dir = ($ENV{MTR_PARALLEL} == 1) ?
+  "$ENV{'MYSQLTEST_VARDIR'}/install.db" :
+  "$ENV{'MYSQLTEST_VARDIR'}/../install.db";
+copytree($install_db_dir, $ENV{'MYSQLD_DATADIR'});
+EOF
+
+--let $restart_parameters= $old_restart_parameters
+--source include/start_mysqld.inc
diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result
index 497366ebf3d..20f2db8ba70 100644
--- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result
+++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result
@@ -1793,7 +1793,7 @@ VARIABLE_SCOPE	GLOBAL
 VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	Size of each log file in a log group.
 NUMERIC_MIN_VALUE	1048576
-NUMERIC_MAX_VALUE	549755813888
+NUMERIC_MAX_VALUE	17592186044415
 NUMERIC_BLOCK_SIZE	65536
 ENUM_VALUE_LIST	NULL
 READ_ONLY	YES
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index 3a3688e35df..db0fa4001c6 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -20453,7 +20453,7 @@ static MYSQL_SYSVAR_LONG(log_buffer_size, innobase_log_buffer_size,
 static MYSQL_SYSVAR_ULONGLONG(log_file_size, srv_log_file_size,
   PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
   "Size of each log file in a log group.",
-  NULL, NULL, 48 << 20, 1 << 20, 512ULL << 30, UNIV_PAGE_SIZE_MAX);
+  NULL, NULL, 48 << 20, 1 << 20, log_group_max_size, UNIV_PAGE_SIZE_MAX);
 /* OS_FILE_LOG_BLOCK_SIZE would be more appropriate than UNIV_PAGE_SIZE_MAX,
 but fil_space_t is being used for the redo log, and it uses data pages. */
 
diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h
index 4864041ef51..8a6705359db 100644
--- a/storage/innobase/include/log0log.h
+++ b/storage/innobase/include/log0log.h
@@ -40,6 +40,10 @@ Created 12/9/1995 Heikki Tuuri
 #include "os0event.h"
 #include "os0file.h"
 
+#ifndef UINT32_MAX
+#define UINT32_MAX             (4294967295U)
+#endif
+
 /** Redo log group */
 struct log_group_t;
 
@@ -527,6 +531,12 @@ MariaDB 10.2.18 and later will use the 10.3 format, but LOG_HEADER_SUBFORMAT
 					header */
 #define LOG_FILE_HDR_SIZE	(4 * OS_FILE_LOG_BLOCK_SIZE)
 
+/* As long as fil_io() is used to handle log io, log group max size is limited
+by (maximum page number) * (minimum page size). Page number type is uint32_t.
+Remove this limitation if page number is no longer used for log file io. */
+static const ulonglong log_group_max_size =
+	((ulonglong(UINT32_MAX) + 1) * UNIV_PAGE_SIZE_MIN - 1);
+
 /** The state of a log group */
 enum log_group_state_t {
 	/** No corruption detected */
diff --git a/storage/innobase/include/log0log.ic b/storage/innobase/include/log0log.ic
index c6bfc9560a2..36caaedfaa2 100644
--- a/storage/innobase/include/log0log.ic
+++ b/storage/innobase/include/log0log.ic
@@ -185,7 +185,9 @@ log_block_convert_lsn_to_no(
 /*========================*/
 	lsn_t	lsn)	/*!< in: lsn of a byte within the block */
 {
-	return(((ulint) (lsn / OS_FILE_LOG_BLOCK_SIZE) & 0x3FFFFFFFUL) + 1);
+	return(((ulint) (lsn / OS_FILE_LOG_BLOCK_SIZE) &
+		DBUG_EVALUATE_IF("innodb_small_log_block_no_limit",
+			0xFUL, 0x3FFFFFFFUL)) + 1);
 }
 
 /************************************************************//**
diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc
index 1d98f92bff0..682a79409f9 100644
--- a/storage/innobase/log/log0log.cc
+++ b/storage/innobase/log/log0log.cc
@@ -942,11 +942,14 @@ loop:
 	the trailer fields of the log blocks */
 
 	for (i = 0; i < write_len / OS_FILE_LOG_BLOCK_SIZE; i++) {
+#ifdef UNIV_DEBUG
+		ulint hdr_no_2 = log_block_get_hdr_no(buf) + i;
+		DBUG_EXECUTE_IF("innodb_small_log_block_no_limit",
+				hdr_no_2 = ((hdr_no_2 - 1) & 0xFUL) + 1;);
+#endif
 		ut_ad(pad_len >= len
-		      || i * OS_FILE_LOG_BLOCK_SIZE >= len - pad_len
-		      || log_block_get_hdr_no(
-			      buf + i * OS_FILE_LOG_BLOCK_SIZE)
-			 == log_block_get_hdr_no(buf) + i);
+			|| i * OS_FILE_LOG_BLOCK_SIZE >= len - pad_len
+			|| log_block_get_hdr_no(buf + i * OS_FILE_LOG_BLOCK_SIZE) == hdr_no_2);
 		log_block_store_checksum(buf + i * OS_FILE_LOG_BLOCK_SIZE);
 	}
 
diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc
index 3cd5f289991..1b4a478639c 100644
--- a/storage/innobase/srv/srv0start.cc
+++ b/storage/innobase/srv/srv0start.cc
@@ -1852,13 +1852,11 @@ innobase_start_or_create_for_mysql()
 		srv_start_state_set(SRV_START_STATE_IO);
 	}
 
-	if (srv_n_log_files * srv_log_file_size >= 512ULL << 30) {
-		/* log_block_convert_lsn_to_no() limits the returned block
-		number to 1G and given that OS_FILE_LOG_BLOCK_SIZE is 512
-		bytes, then we have a limit of 512 GB. If that limit is to
-		be raised, then log_block_convert_lsn_to_no() must be
-		modified. */
-		ib::error() << "Combined size of log files must be < 512 GB";
+	if (srv_n_log_files * srv_log_file_size >= log_group_max_size) {
+		/* Log group size is limited by the size of page number. Remove this
+		limitation when fil_io() is not used for recovery log io. */
+		ib::error() << "Combined size of log files must be < "
+			<< log_group_max_size << " GB";
 
 		return(srv_init_abort(DB_ERROR));
 	}
@@ -2070,7 +2068,7 @@ innobase_start_or_create_for_mysql()
 		ut_a(fil_validate());
 		ut_a(log_space);
 
-		ut_a(srv_log_file_size <= 512ULL << 30);
+		ut_a(srv_log_file_size <= log_group_max_size);
 
 		const ulint size = 1 + ulint((srv_log_file_size - 1)
 					     >> srv_page_size_shift);