Merge 10.4 into 10.5

29 files changed, 265 insertions, 145 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index c52d6621bb1..80fac4b2ccc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -197,6 +197,7 @@ ENDIF()
 OPTION(NOT_FOR_DISTRIBUTION "Allow linking with GPLv2-incompatible system libraries. Only set it you never plan to distribute the resulting binaries" OFF)
 
 INCLUDE(check_compiler_flag)
+INCLUDE(check_linker_flag)
 
 OPTION(WITH_ASAN "Enable address sanitizer" OFF)
 
@@ -250,7 +251,7 @@ OPTION(SECURITY_HARDENED "Use security-enhancing compiler features (stack protec
 IF(SECURITY_HARDENED AND NOT WITH_ASAN AND NOT WITH_UBSAN AND NOT WITH_TSAN)
   # security-enhancing flags
   MY_CHECK_AND_SET_COMPILER_FLAG("-pie -fPIC")
-  MY_CHECK_AND_SET_COMPILER_FLAG("-Wl,-z,relro,-z,now")
+  MY_CHECK_AND_SET_LINKER_FLAG("-Wl,-z,relro,-z,now")
   MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-protector --param=ssp-buffer-size=4")
   MY_CHECK_AND_SET_COMPILER_FLAG("-D_FORTIFY_SOURCE=2" RELEASE RELWITHDEBINFO)
 ENDIF()
diff --git a/cmake/check_linker_flag.cmake b/cmake/check_linker_flag.cmake
new file mode 100644
index 00000000000..ff4b91e89f6
--- /dev/null
+++ b/cmake/check_linker_flag.cmake
@@ -0,0 +1,27 @@
+include(CheckCXXSourceCompiles)
+
+FUNCTION(MY_CHECK_AND_SET_LINKER_FLAG flag_to_set)
+  # Let's avoid expensive compiler tests on Windows:
+  IF(WIN32)
+    RETURN()
+  ENDIF()
+  STRING(REGEX REPLACE "[-,= +]" "_" result "HAVE_LINK_FLAG_${flag_to_set}")
+  SET(SAVE_CMAKE_REQUIRED_LINK_OPTIONS "${CMAKE_REQUIRED_LINK_OPTIONS}")
+  STRING(REGEX REPLACE "^-Wno-" "-W" flag_to_check ${flag_to_set})
+  SET(CMAKE_REQUIRED_LINK_OPTIONS ${CMAKE_REQUIRED_LINK_OPTIONS} ${flag_to_check})
+  CHECK_CXX_SOURCE_COMPILES("int main(void) { return 0; }" ${result})
+  SET(CMAKE_REQUIRED_LINK_OPTIONS "${SAVE_CMAKE_REQUIRED_LINK_OPTIONS}")
+  IF (${result})
+    FOREACH(linktype SHARED MODULE EXE)
+        IF(ARGN)
+          FOREACH(type ${ARGN})
+            SET(CMAKE_${linktype}_LINKER_FLAGS_${type}
+              "${CMAKE_${linktype}_LINKER_FLAGS_${type}} ${flag_to_set}" PARENT_SCOPE)
+          ENDFOREACH()
+        ELSE()
+          SET(CMAKE_${linktype}_LINKER_FLAGS
+            "${CMAKE_${linktype}_LINKER_FLAGS} ${flag_to_set}" PARENT_SCOPE)
+        ENDIF()
+    ENDFOREACH()
+  ENDIF()
+ENDFUNCTION()
diff --git a/mysql-test/main/partition_explicit_prune.result b/mysql-test/main/partition_explicit_prune.result
index 21741bdff73..5b3049c146f 100644
--- a/mysql-test/main/partition_explicit_prune.result
+++ b/mysql-test/main/partition_explicit_prune.result
@@ -1897,6 +1897,24 @@ ALTER TABLE t1 EXCHANGE PARTITION p0 WITH TABLE t2;
 SELECT * FROM t1 PARTITION (p0);
 i
 UNLOCK TABLES;
+DROP TABLE t1;
+#
+# MDEV-18371 Server crashes in ha_innobase::cmp_ref upon UPDATE with PARTITION clause.
+#
+CREATE TABLE t1 (a INT, b INT, KEY (a)) ENGINE=InnoDB PARTITION BY KEY(b) PARTITIONS 4;
+INSERT INTO t1 VALUES (3,0),(8,2),(7,8),(3,4),(2,4),(0,7),(4,3),(3,6);
+FLUSH TABLES;
+UPDATE t1 PARTITION (p3,p1) SET a = 2 WHERE a = 3;
+SELECT * FROM t1;
+a	b
+2	0
+7	8
+2	4
+2	4
+0	7
+4	3
+8	2
+2	6
 DROP TABLE t1, t2;
 #
 # MDEV-18982: INSERT using explicit patition pruning with column list
diff --git a/mysql-test/main/partition_explicit_prune.test b/mysql-test/main/partition_explicit_prune.test
index a516527c055..fdbbcadd8cb 100644
--- a/mysql-test/main/partition_explicit_prune.test
+++ b/mysql-test/main/partition_explicit_prune.test
@@ -874,6 +874,18 @@ SELECT * FROM t1 PARTITION (p0);
 ALTER TABLE t1 EXCHANGE PARTITION p0 WITH TABLE t2;
 SELECT * FROM t1 PARTITION (p0);
 UNLOCK TABLES;
+DROP TABLE t1;
+
+--echo #
+--echo # MDEV-18371 Server crashes in ha_innobase::cmp_ref upon UPDATE with PARTITION clause.
+--echo #
+
+CREATE TABLE t1 (a INT, b INT, KEY (a)) ENGINE=InnoDB PARTITION BY KEY(b) PARTITIONS 4;
+INSERT INTO t1 VALUES (3,0),(8,2),(7,8),(3,4),(2,4),(0,7),(4,3),(3,6);
+FLUSH TABLES;
+UPDATE t1 PARTITION (p3,p1) SET a = 2 WHERE a = 3;
+SELECT * FROM t1;
+
 
 # Cleanup
 DROP TABLE t1, t2;
diff --git a/mysql-test/main/stat_tables_innodb.result b/mysql-test/main/stat_tables_innodb.result
index a5c702213c3..e937d1ffe0f 100644
--- a/mysql-test/main/stat_tables_innodb.result
+++ b/mysql-test/main/stat_tables_innodb.result
@@ -926,4 +926,28 @@ set global innodb_stats_persistent= @innodb_stats_persistent_save;
 set global innodb_stats_persistent_sample_pages=
 @innodb_stats_persistent_sample_pages_save;
 set optimizer_switch=@save_optimizer_switch_for_stat_tables_test;
+#
+# MDEV-22851: Engine independent index statistics are incorrect for large tables on Windows.
+#
+CREATE TABLE t1(a INT) ENGINE=INNODB;
+INSERT INTO t1 SELECT 1 FROM seq_1_to_60000;
+SET @save_use_stat_tables= @@use_stat_tables;
+SET use_stat_tables= preferably;
+SELECT count(*) FROM t1;
+count(*)
+60000
+CREATE INDEX idx ON t1(a);
+ANALYZE TABLE t1 PERSISTENT FOR COLUMNS (a) INDEXES (idx);
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+SELECT * FROM mysql.index_stats where table_name='t1';
+db_name	table_name	index_name	prefix_arity	avg_frequency
+test	t1	idx	1	60000.0000
+SELECT * FROM mysql.column_stats where table_name='t1';
+db_name	table_name	column_name	min_value	max_value	nulls_ratio	avg_length	avg_frequency	hist_size	hist_type	histogram
+test	t1	a	1	1	0.0000	4.0000	60000.0000	0	NULL	NULL
+SET use_stat_tables= @save_use_stat_tables;
+DROP TABLE t1;
+# end of 10.1 tests
 SET SESSION DEFAULT_STORAGE_ENGINE=DEFAULT;
diff --git a/mysql-test/main/stat_tables_innodb.test b/mysql-test/main/stat_tables_innodb.test
index 533f56387a2..f439ec52315 100644
--- a/mysql-test/main/stat_tables_innodb.test
+++ b/mysql-test/main/stat_tables_innodb.test
@@ -1,4 +1,5 @@
 --source include/have_innodb.inc
+--source include/have_sequence.inc
 
 SET SESSION DEFAULT_STORAGE_ENGINE='InnoDB';
 
@@ -18,4 +19,23 @@ set global innodb_stats_persistent_sample_pages=
 
 set optimizer_switch=@save_optimizer_switch_for_stat_tables_test;
 
+--echo #
+--echo # MDEV-22851: Engine independent index statistics are incorrect for large tables on Windows.
+--echo #
+
+CREATE TABLE t1(a INT) ENGINE=INNODB;
+INSERT INTO t1 SELECT 1 FROM seq_1_to_60000;
+
+SET @save_use_stat_tables= @@use_stat_tables;
+SET use_stat_tables= preferably;
+SELECT count(*) FROM t1;
+CREATE INDEX idx ON t1(a);
+ANALYZE TABLE t1 PERSISTENT FOR COLUMNS (a) INDEXES (idx);
+SELECT * FROM mysql.index_stats where table_name='t1';
+SELECT * FROM mysql.column_stats where table_name='t1';
+SET use_stat_tables= @save_use_stat_tables;
+DROP TABLE t1;
+
+--echo # end of 10.1 tests
+
 SET SESSION DEFAULT_STORAGE_ENGINE=DEFAULT;
diff --git a/mysql-test/suite/innodb/r/instant_alter_extend,utf8.rdiff b/mysql-test/suite/innodb/r/instant_alter_extend,utf8.rdiff
index 596dfe43ab8..08f9afa696a 100644
--- a/mysql-test/suite/innodb/r/instant_alter_extend,utf8.rdiff
+++ b/mysql-test/suite/innodb/r/instant_alter_extend,utf8.rdiff
@@ -1,29 +1,38 @@
 --- instant_alter_convert.result
 +++ instant_alter_convert,utf8.result
-@@ -37,7 +37,7 @@
- test.t	check	status	OK
+@@ -38,7 +38,7 @@
+ best.t	check	status	OK
  call check_table('t');
  name	mtype	prtype	len
 -a	2	800FE	200
 +a	13	2100FE	600
  # CHAR enlargement
- alter table t modify a char(220), algorithm=instant;
- select count(a) from t where a = @bigval;
-@@ -51,7 +51,7 @@
- test.t	check	status	OK
+ alter table t modify a char(220);
+ affected rows: 2
+@@ -54,7 +54,7 @@
+ best.t	check	status	OK
  call check_table('t');
  name	mtype	prtype	len
 -a	2	800FE	220
 +a	13	2100FE	660
+ ALTER TABLE t CHANGE COLUMN a a CHAR(230) BINARY;
+ affected rows: 2
+ info: Records: 2  Duplicates: 0  Warnings: 0
+@@ -69,7 +69,7 @@
+ best.t	check	status	OK
+ call check_table('t');
+ name	mtype	prtype	len
+-a	13	2F00FE	230
++a	13	5300FE	690
  # Convert from VARCHAR to a bigger CHAR
- alter table t modify a varchar(200), algorithm=instant;
- ERROR 0A000: ALGORITHM=INSTANT is not supported. Reason: Cannot change column type. Try ALGORITHM=COPY
-@@ -72,7 +72,7 @@
- test.t	check	status	OK
+ alter table t modify a varchar(200);
+ affected rows: 2
+@@ -92,7 +92,7 @@
+ best.t	check	status	OK
  call check_table('t');
  name	mtype	prtype	len
 -a	2	800FE	255
 +a	13	2100FE	765
  # BINARY/VARBINARY test
  create or replace table t (a varbinary(300));
- alter table t modify a binary(255), algorithm=instant;
+ insert into t values(NULL);
diff --git a/mysql-test/suite/innodb/r/instant_alter_extend.result b/mysql-test/suite/innodb/r/instant_alter_extend.result
index fb03ef9a182..33a5f57c7b6 100644
--- a/mysql-test/suite/innodb/r/instant_alter_extend.result
+++ b/mysql-test/suite/innodb/r/instant_alter_extend.result
diff --git a/mysql-test/suite/innodb/t/instant_alter_extend.test b/mysql-test/suite/innodb/t/instant_alter_extend.test
index 4320d9bae05..7258ba6d238 100644
--- a/mysql-test/suite/innodb/t/instant_alter_extend.test
+++ b/mysql-test/suite/innodb/t/instant_alter_extend.test
@@ -64,6 +64,15 @@ select a, length(a) from t where a = 'z';
 check table t extended;
 call check_table('t');
 
+--enable_info
+ALTER TABLE t CHANGE COLUMN a a CHAR(230) BINARY;
+ALTER TABLE t ADD COLUMN b INT FIRST;
+ALTER TABLE t DROP b;
+--disable_info
+
+check table t extended;
+call check_table('t');
+
 --echo # Convert from VARCHAR to a bigger CHAR
 --enable_info
 alter table t modify a varchar(200);
diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc
index f5eab028755..307f7ff24af 100644
--- a/sql/ha_partition.cc
+++ b/sql/ha_partition.cc
@@ -5712,8 +5712,9 @@ extern "C" int cmp_key_rowid_part_id(void *ptr, uchar *ref1, uchar *ref2)
   {
     return res;
   }
-  if ((res= file->m_file[0]->cmp_ref(ref1 + PARTITION_BYTES_IN_POS + file->m_rec_length,
-                                     ref2 + PARTITION_BYTES_IN_POS + file->m_rec_length)))
+  if ((res= file->get_open_file_sample()->cmp_ref(ref1 +
+          PARTITION_BYTES_IN_POS + file->m_rec_length,
+          ref2 + PARTITION_BYTES_IN_POS + file->m_rec_length)))
   {
     return res;
   }
@@ -9744,7 +9745,7 @@ uint8 ha_partition::table_cache_type()
 {
   DBUG_ENTER("ha_partition::table_cache_type");
 
-  DBUG_RETURN(m_file[0]->table_cache_type());
+  DBUG_RETURN(get_open_file_sample()->table_cache_type());
 }
 
 
diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index 95d1c84a811..da86bbc3a37 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -4106,8 +4106,10 @@ static int init_common_variables()
     get corrupted if accesses with names of different case.
   */
   DBUG_PRINT("info", ("lower_case_table_names: %d", lower_case_table_names));
+  if(mysql_real_data_home_ptr == NULL || *mysql_real_data_home_ptr == 0)
+    mysql_real_data_home_ptr= mysql_real_data_home;
   SYSVAR_AUTOSIZE(lower_case_file_system,
-                  test_if_case_insensitive(mysql_real_data_home));
+                  test_if_case_insensitive(mysql_real_data_home_ptr));
   if (!lower_case_table_names && lower_case_file_system == 1)
   {
     if (lower_case_table_names_used)
@@ -4124,7 +4126,7 @@ static int init_common_variables()
     {
       if (global_system_variables.log_warnings)
 	sql_print_warning("Setting lower_case_table_names=2 because file "
-                  "system for %s is case insensitive", mysql_real_data_home);
+                  "system for %s is case insensitive", mysql_real_data_home_ptr);
       SYSVAR_AUTOSIZE(lower_case_table_names, 2);
     }
   }
@@ -4135,7 +4137,7 @@ static int init_common_variables()
       sql_print_warning("lower_case_table_names was set to 2, even though your "
                         "the file system '%s' is case sensitive.  Now setting "
                         "lower_case_table_names to 0 to avoid future problems.",
-			mysql_real_data_home);
+			mysql_real_data_home_ptr);
     SYSVAR_AUTOSIZE(lower_case_table_names, 0);
   }
   else
diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc
index 879955af723..2636299e330 100644
--- a/sql/sql_statistics.cc
+++ b/sql/sql_statistics.cc
@@ -2123,8 +2123,8 @@ int alloc_statistics_for_table(THD* thd, TABLE *table)
                                     sizeof(Index_statistics) * keys);
 
   uint key_parts= table->s->ext_key_parts;
-  ulong *idx_avg_frequency= (ulong*) alloc_root(&table->mem_root,
-                                                sizeof(ulong) * key_parts);
+  ulonglong *idx_avg_frequency= (ulonglong*) alloc_root(&table->mem_root,
+                                               sizeof(ulonglong) * key_parts);
 
   uint columns= 0;
   for (field_ptr= table->field; *field_ptr; field_ptr++)
@@ -2169,7 +2169,7 @@ int alloc_statistics_for_table(THD* thd, TABLE *table)
     }
   }
 
-  memset(idx_avg_frequency, 0, sizeof(ulong) * key_parts);
+  memset(idx_avg_frequency, 0, sizeof(ulonglong) * key_parts);
 
   KEY *key_info, *end;
   for (key_info= table->key_info, end= key_info + table->s->keys;
@@ -2285,14 +2285,14 @@ static int alloc_statistics_for_table_share(THD* thd, TABLE_SHARE *table_share)
   }
 
   uint key_parts= table_share->ext_key_parts;
-  ulong *idx_avg_frequency=  table_stats->idx_avg_frequency;
+  ulonglong *idx_avg_frequency=  table_stats->idx_avg_frequency;
   if (!idx_avg_frequency)
   {
-    idx_avg_frequency= (ulong*) alloc_root(&stats_cb->mem_root,
-                                           sizeof(ulong) * key_parts);
+    idx_avg_frequency= (ulonglong*) alloc_root(&stats_cb->mem_root,
+                                               sizeof(ulonglong) * key_parts);
     if (idx_avg_frequency)
     {
-      memset(idx_avg_frequency, 0, sizeof(ulong) * key_parts);
+      memset(idx_avg_frequency, 0, sizeof(ulonglong) * key_parts);
       table_stats->idx_avg_frequency= idx_avg_frequency;
       for (key_info= table_share->key_info, end= key_info + keys;
            key_info < end; 
diff --git a/sql/sql_statistics.h b/sql/sql_statistics.h
index b90c614b74f..20ecf06bfee 100644
--- a/sql/sql_statistics.h
+++ b/sql/sql_statistics.h
@@ -294,7 +294,9 @@ public:
   uchar *min_max_record_buffers;    /* Record buffers for min/max values  */
   Column_statistics *column_stats;  /* Array of statistical data for columns */
   Index_statistics *index_stats;    /* Array of statistical data for indexes */
-  ulong *idx_avg_frequency;   /* Array of records per key for index prefixes */
+
+  /* Array of records per key for index prefixes */
+  ulonglong *idx_avg_frequency;
   uchar *histograms;                /* Sequence of histograms       */                    
 };
 
@@ -346,7 +348,7 @@ private:
     CHAR values are stripped of trailing spaces.
     Flexible values are stripped of their length prefixes.
   */
-  ulong avg_length;
+  ulonglong avg_length;
 
   /*
     The ratio N/D multiplied by the scale factor Scale_factor_avg_frequency,
@@ -354,7 +356,7 @@ private:
        N is the number of rows with not null value in the column,
        D the number of distinct values among them
   */
-  ulong avg_frequency;
+  ulonglong avg_frequency;
 
 public:
 
@@ -404,12 +406,12 @@ public:
 
   void set_avg_length (double val)
   {
-    avg_length= (ulong) (val * Scale_factor_avg_length);
+    avg_length= (ulonglong) (val * Scale_factor_avg_length);
   }
 
   void set_avg_frequency (double val)
   {
-    avg_frequency= (ulong) (val * Scale_factor_avg_frequency);
+    avg_frequency= (ulonglong) (val * Scale_factor_avg_frequency);
   }
 
   bool min_max_values_are_provided()
@@ -448,11 +450,11 @@ private:
     in the first k components, and D is the number of distinct
     k-component prefixes among them 
   */
-  ulong *avg_frequency;
+  ulonglong *avg_frequency;
 
 public:
 
-  void init_avg_frequency(ulong *ptr) { avg_frequency= ptr; }
+  void init_avg_frequency(ulonglong *ptr) { avg_frequency= ptr; }
 
   bool avg_frequency_is_inited() { return avg_frequency != NULL; }
 
@@ -463,7 +465,7 @@ public:
 
   void set_avg_frequency(uint i, double val)
   {
-    avg_frequency[i]= (ulong) (val * Scale_factor_avg_frequency);
+    avg_frequency[i]= (ulonglong) (val * Scale_factor_avg_frequency);
   }
 
 };
diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc
index 22699f2c968..c1bba8ad8a8 100644
--- a/storage/innobase/buf/buf0dblwr.cc
+++ b/storage/innobase/buf/buf0dblwr.cc
@@ -513,15 +513,17 @@ buf_dblwr_process()
 		const ulint		page_no	= page_get_page_no(page);
 		const page_id_t		page_id(space_id, page_no);
 
-		if (page_no >= space->size) {
+		if (UNIV_UNLIKELY(page_no >= space->size)) {
 
 			/* Do not report the warning for undo
 			tablespaces, because they can be truncated in place. */
 			if (!srv_is_undo_tablespace(space_id)) {
-				ib::warn() << "A copy of page " << page_id
+				ib::warn() << "A copy of page " << page_no
 					<< " in the doublewrite buffer slot "
 					<< page_no_dblwr
-					<< " is not within space bounds";
+					<< " is beyond the end of tablespace "
+					<< space->name
+					<< " (" << space->size << " pages)";
 			}
 			continue;
 		}
diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc
index af9302aa113..13f120414f3 100644
--- a/storage/innobase/buf/buf0flu.cc
+++ b/storage/innobase/buf/buf0flu.cc
@@ -1262,7 +1262,7 @@ static page_id_t buf_flush_check_neighbors(const fil_space_t &space,
   page_id_t low= id - (id.page_no() % buf_flush_area);
   page_id_t high= low + buf_flush_area;
   high.set_page_no(std::min(high.page_no(),
-                            static_cast<uint32_t>(space.size - 1)));
+                            static_cast<uint32_t>(space.committed_size - 1)));
 
   /* Determine the contiguous dirty area around id. */
   const ulint id_fold= id.fold();
diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc
index 0f6e0e84e84..19f99333b75 100644
--- a/storage/innobase/buf/buf0rea.cc
+++ b/storage/innobase/buf/buf0rea.cc
@@ -415,7 +415,7 @@ buf_read_ahead_random(const page_id_t page_id, ulint zip_size, bool ibuf)
   const page_id_t low= page_id - (page_id.page_no() % buf_read_ahead_area);
   page_id_t high= low + buf_read_ahead_area;
   high.set_page_no(std::min(high.page_no(),
-                            static_cast<uint32_t>(space->size - 1)));
+                            static_cast<uint32_t>(space->committed_size - 1)));
 
   /* Count how many blocks in the area have been recently accessed,
   that is, reside near the start of the LRU list. */
@@ -600,7 +600,7 @@ buf_read_ahead_linear(const page_id_t page_id, ulint zip_size, bool ibuf)
   fil_space_t *space= fil_space_acquire(page_id.space());
   if (!space)
     return 0;
-  if (high_1.page_no() >= space->size)
+  if (high_1.page_no() >= space->committed_size)
   {
     /* The area is not whole. */
     space->release();
@@ -663,7 +663,7 @@ hard_fail:
       if (id != new_low && id != new_high_1)
         /* This is not a border page of the area: return */
         goto hard_fail;
-      if (new_high_1.page_no() >= space->size)
+      if (new_high_1.page_no() >= space->committed_size)
         /* The area is not whole */
         goto hard_fail;
     }
diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc
index 87a72367031..aaca87be52e 100644
--- a/storage/innobase/fil/fil0fil.cc
+++ b/storage/innobase/fil/fil0fil.cc
@@ -284,34 +284,6 @@ fil_space_get(
 	return(space);
 }
 
-/** Returns the latch of a file space.
-@param[in]	id	space id
-@param[out]	flags	tablespace flags
-@return latch protecting storage allocation */
-rw_lock_t*
-fil_space_get_latch(
-	ulint	id,
-	ulint*	flags)
-{
-	fil_space_t*	space;
-
-	ut_ad(fil_system.is_initialised());
-
-	mutex_enter(&fil_system.mutex);
-
-	space = fil_space_get_by_id(id);
-
-	ut_a(space);
-
-	if (flags) {
-		*flags = space->flags;
-	}
-
-	mutex_exit(&fil_system.mutex);
-
-	return(&(space->latch));
-}
-
 /**********************************************************************//**
 Checks if all the file nodes in a space are flushed.
 @return true if all are flushed */
@@ -879,6 +851,9 @@ fil_mutex_enter_and_prepare_for_io(
 			ut_a(success);
 			/* InnoDB data files cannot shrink. */
 			ut_a(space->size >= size);
+			if (size > space->committed_size) {
+				space->committed_size = size;
+			}
 
 			/* There could be multiple concurrent I/O requests for
 			this tablespace (multiple threads trying to extend
diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc
index 94a11778beb..a2a6335a17a 100644
--- a/storage/innobase/fsp/fsp0fsp.cc
+++ b/storage/innobase/fsp/fsp0fsp.cc
@@ -319,7 +319,7 @@ xdes_get_descriptor_with_space_hdr(
 	ulint	limit;
 	ulint	size;
 	ulint	descr_page_no;
-	ut_ad(mtr->memo_contains(space->latch, MTR_MEMO_X_LOCK));
+	ut_ad(mtr->memo_contains(*space));
 	ut_ad(mtr->memo_contains_flagged(header, MTR_MEMO_PAGE_SX_FIX));
 	/* Read free limit and space size */
 	limit = mach_read_from_4(FSP_HEADER_OFFSET + FSP_FREE_LIMIT
@@ -444,7 +444,7 @@ xdes_lst_get_descriptor(
 	buf_block_t**		block,
 	mtr_t*			mtr)
 {
-	ut_ad(mtr->memo_contains(space->latch, MTR_MEMO_X_LOCK));
+	ut_ad(mtr->memo_contains(*space));
 	return fut_get_ptr(space->id, space->zip_size(),
 			   lst_node, RW_SX_LATCH, mtr, block)
 		- XDES_FLST_NODE;
@@ -887,30 +887,20 @@ fsp_fill_free_list(
 					      FIL_PAGE_TYPE_XDES);
 			}
 
-			/* Initialize the ibuf bitmap page in a separate
-			mini-transaction because it is low in the latching
-			order, and we must be able to release its latch.
-			Note: Insert-Buffering is disabled for tables that
-			reside in the temp-tablespace. */
 			if (space->purpose != FIL_TYPE_TEMPORARY) {
-				mtr_t	ibuf_mtr;
-
-				ibuf_mtr.start();
-				ibuf_mtr.set_named_space(space);
-
+				const auto savepoint = mtr->get_savepoint();
 				block = buf_page_create(
 					space,
 					static_cast<uint32_t>(
 						i + FSP_IBUF_BITMAP_OFFSET),
-					zip_size, &ibuf_mtr);
-				ibuf_mtr.sx_latch_at_savepoint(0, block);
+					zip_size, mtr);
+				mtr->sx_latch_at_savepoint(savepoint, block);
 				buf_block_dbg_add_level(block, SYNC_FSP_PAGE);
 
-				fsp_init_file_page(space, block, &ibuf_mtr);
-				ibuf_mtr.write<2>(*block,
-						  block->frame + FIL_PAGE_TYPE,
-						  FIL_PAGE_IBUF_BITMAP);
-				ibuf_mtr.commit();
+				fsp_init_file_page(space, block, mtr);
+				mtr->write<2>(*block,
+					      block->frame + FIL_PAGE_TYPE,
+					      FIL_PAGE_IBUF_BITMAP);
 			}
 		}
 
@@ -1296,7 +1286,7 @@ static void fsp_free_page(fil_space_t* space, page_no_t offset, mtr_t* mtr)
 @param[in,out]  mtr     mini-transaction */
 static void fsp_free_extent(fil_space_t* space, page_no_t offset, mtr_t* mtr)
 {
-  ut_ad(mtr->memo_contains(space->latch, MTR_MEMO_X_LOCK));
+  ut_ad(mtr->memo_contains(*space));
 
   buf_block_t *block= fsp_get_header(space, mtr);
   buf_block_t *xdes= 0;
@@ -1308,8 +1298,8 @@ static void fsp_free_extent(fil_space_t* space, page_no_t offset, mtr_t* mtr)
   xdes_init(*xdes, descr, mtr);
 
   flst_add_last(block, FSP_HEADER_OFFSET + FSP_FREE,
-                xdes, static_cast<uint16_t>(descr - xdes->frame
-					    + XDES_FLST_NODE), mtr);
+                xdes, static_cast<uint16_t>(descr - xdes->frame +
+                                            XDES_FLST_NODE), mtr);
   space->free_len++;
 }
 
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index 4e84873fa23..b03900d76fa 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -21062,43 +21062,35 @@ is_part_of_a_primary_key(const Field* field)
 	       && field->part_of_key.is_set(s->primary_key);
 }
 
-bool
-ha_innobase::can_convert_string(const Field_string* field,
-				const Column_definition& new_type) const
+bool ha_innobase::can_convert_string(const Field_string *field,
+                                     const Column_definition &new_type) const
 {
-	DBUG_ASSERT(!field->compression_method());
-	if (new_type.type_handler() != field->type_handler()) {
-		return false;
-	}
+  DBUG_ASSERT(!field->compression_method());
+  if (new_type.type_handler() != field->type_handler())
+    return false;
 
-	if (new_type.char_length < field->char_length()) {
-		return false;
-	}
+  if (new_type.char_length != field->char_length())
+    return false;
 
-	if (new_type.charset != field->charset()) {
-		if (new_type.length != field->max_display_length()
-		    && !m_prebuilt->table->not_redundant()) {
-			return IS_EQUAL_NO;
-		}
+  const Charset field_cs(field->charset());
 
-		Charset field_cs(field->charset());
-		if (!field_cs.encoding_allows_reinterpret_as(
-			new_type.charset)) {
-			return false;
-		}
+  if (new_type.length != field->max_display_length() &&
+      (!m_prebuilt->table->not_redundant() ||
+       field_cs.mbminlen() == field_cs.mbmaxlen()))
+    return false;
 
-		if (!field_cs.eq_collation_specific_names(new_type.charset)) {
-			return !is_part_of_a_primary_key(field);
-		}
+  if (new_type.charset != field->charset())
+  {
+    if (!field_cs.encoding_allows_reinterpret_as(new_type.charset))
+      return false;
 
-		return true;
-	}
+    if (!field_cs.eq_collation_specific_names(new_type.charset))
+      return !is_part_of_a_primary_key(field);
 
-	if (new_type.length != field->max_display_length()) {
-		return false;
-	}
+    return true;
+  }
 
-	return true;
+  return true;
 }
 
 static bool
diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h
index cb12c91d29a..24d587519eb 100644
--- a/storage/innobase/include/fil0fil.h
+++ b/storage/innobase/include/fil0fil.h
@@ -326,6 +326,8 @@ struct fil_space_t
 				/*!< recovered tablespace size in pages;
 				0 if no size change was read from the redo log,
 				or if the size change was implemented */
+  /** the committed size of the tablespace in pages */
+  Atomic_relaxed<ulint> committed_size;
 	ulint		n_reserved_extents;
 				/*!< number of reserved free extents for
 				ongoing operations like B-tree page split */
@@ -386,6 +388,15 @@ struct fil_space_t
 	/** @return whether the tablespace is about to be dropped */
 	bool is_stopping() const { return stop_new_ops;	}
 
+  /** Clamp a page number for batched I/O, such as read-ahead.
+  @param offset   page number limit
+  @return offset clamped to the tablespace size */
+  ulint max_page_number_for_io(ulint offset) const
+  {
+    const ulint limit= committed_size;
+    return limit > offset ? offset : limit;
+  }
+
 	/** @return whether doublewrite buffering is needed */
 	bool use_doublewrite() const
 	{
@@ -1327,15 +1338,6 @@ inline void fil_node_t::complete_io(bool write)
 
 #include "fil0crypt.h"
 
-/** Returns the latch of a file space.
-@param[in]	id	space id
-@param[out]	flags	tablespace flags
-@return latch protecting storage allocation */
-rw_lock_t*
-fil_space_get_latch(
-	ulint	id,
-	ulint*	flags);
-
 /** Create a space memory object and put it to the fil_system hash table.
 Error messages are issued to the server log.
 @param[in]	name		tablespace name
diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h
index 6058d0daaf6..f1e517adfbc 100644
--- a/storage/innobase/include/mtr0mtr.h
+++ b/storage/innobase/include/mtr0mtr.h
@@ -264,7 +264,8 @@ struct mtr_t {
 		ut_ad(space->purpose == FIL_TYPE_TEMPORARY
 		      || space->purpose == FIL_TYPE_IMPORT
 		      || space->purpose == FIL_TYPE_TABLESPACE);
-		x_lock(&space->latch, file, line);
+		memo_push(space, MTR_MEMO_SPACE_X_LOCK);
+		rw_lock_x_lock_inline(&space->latch, 0, file, line);
 	}
 
 	/** Release an object in the memo stack.
@@ -337,6 +338,12 @@ public:
   @return whether (lock,type) is contained */
   bool memo_contains(const rw_lock_t &lock, mtr_memo_type_t type)
     MY_ATTRIBUTE((warn_unused_result));
+  /** Check if we are holding exclusive tablespace latch
+  @param space  tablespace to search for
+  @return whether space.latch is being held */
+  bool memo_contains(const fil_space_t& space)
+    MY_ATTRIBUTE((warn_unused_result));
+
 
 	/** Check if memo contains the given item.
 	@param object		object to search
diff --git a/storage/innobase/include/mtr0mtr.ic b/storage/innobase/include/mtr0mtr.ic
index 16777da1044..940d9952d84 100644
--- a/storage/innobase/include/mtr0mtr.ic
+++ b/storage/innobase/include/mtr0mtr.ic
@@ -43,7 +43,7 @@ mtr_t::memo_push(void* object, mtr_memo_type_t type)
 	ut_ad(is_active());
 	ut_ad(object != NULL);
 	ut_ad(type >= MTR_MEMO_PAGE_S_FIX);
-	ut_ad(type <= MTR_MEMO_SX_LOCK);
+	ut_ad(type <= MTR_MEMO_SPACE_X_LOCK);
 	ut_ad(ut_is_2pow(type));
 
 	/* If this mtr has x-fixed a clean page then we set
diff --git a/storage/innobase/include/mtr0types.h b/storage/innobase/include/mtr0types.h
index f110e45aad8..fe259c43e9f 100644
--- a/storage/innobase/include/mtr0types.h
+++ b/storage/innobase/include/mtr0types.h
@@ -337,7 +337,10 @@ enum mtr_memo_type_t {
 
 	MTR_MEMO_X_LOCK = RW_X_LATCH << 5,
 
-	MTR_MEMO_SX_LOCK = RW_SX_LATCH << 5
+	MTR_MEMO_SX_LOCK = RW_SX_LATCH << 5,
+
+	/** acquire X-latch on fil_space_t::latch */
+	MTR_MEMO_SPACE_X_LOCK = MTR_MEMO_SX_LOCK << 1
 };
 #endif /* !UNIV_CHECKSUM */
 
diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc
index 940429b2cd6..8a542a9f842 100644
--- a/storage/innobase/mtr/mtr0mtr.cc
+++ b/storage/innobase/mtr/mtr0mtr.cc
@@ -218,6 +218,13 @@ static void memo_slot_release(mtr_memo_slot_t *slot)
   case MTR_MEMO_SX_LOCK:
     rw_lock_sx_unlock(reinterpret_cast<rw_lock_t*>(slot->object));
     break;
+  case MTR_MEMO_SPACE_X_LOCK:
+    {
+      fil_space_t *space= static_cast<fil_space_t*>(slot->object);
+      space->committed_size= space->size;
+      rw_lock_x_unlock(&space->latch);
+    }
+    break;
   case MTR_MEMO_X_LOCK:
     rw_lock_x_unlock(reinterpret_cast<rw_lock_t*>(slot->object));
     break;
@@ -251,6 +258,13 @@ struct ReleaseLatches {
     case MTR_MEMO_S_LOCK:
       rw_lock_s_unlock(reinterpret_cast<rw_lock_t*>(slot->object));
       break;
+    case MTR_MEMO_SPACE_X_LOCK:
+      {
+        fil_space_t *space= static_cast<fil_space_t*>(slot->object);
+        space->committed_size= space->size;
+        rw_lock_x_unlock(&space->latch);
+      }
+      break;
     case MTR_MEMO_X_LOCK:
       rw_lock_x_unlock(reinterpret_cast<rw_lock_t*>(slot->object));
       break;
@@ -427,7 +441,7 @@ void mtr_t::commit()
         freed_space= fil_system.sys_space;
       }
 
-      ut_ad(memo_contains(freed_space->latch, MTR_MEMO_X_LOCK));
+      ut_ad(memo_contains(*freed_space));
       /* Update the last freed lsn */
       freed_space->update_last_freed_lsn(m_commit_lsn);
 
@@ -719,6 +733,18 @@ bool mtr_t::memo_contains(const rw_lock_t &lock, mtr_memo_type_t type)
   return true;
 }
 
+/** Check if we are holding exclusive tablespace latch
+@param space  tablespace to search for
+@return whether space.latch is being held */
+bool mtr_t::memo_contains(const fil_space_t& space)
+{
+  Iterate<Find> iteration(Find(&space, MTR_MEMO_SPACE_X_LOCK));
+  if (m_memo.for_each_block_in_reverse(iteration))
+    return false;
+  ut_ad(rw_lock_own(const_cast<rw_lock_t*>(&space.latch), RW_LOCK_X));
+  return true;
+}
+
 /** Debug check for flags */
 struct FlaggedCheck {
 	FlaggedCheck(const void* ptr, ulint flags)
diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc
index ae51a6d1bad..f30e42adeae 100644
--- a/storage/innobase/os/os0file.cc
+++ b/storage/innobase/os/os0file.cc
@@ -4569,13 +4569,17 @@ invalid:
 		space->flags = (space->flags & FSP_FLAGS_MEM_MASK) | flags;
 
 		this->size = ulint(size_bytes / psize);
-		space->size += this->size;
+		space->committed_size = space->size += this->size;
 	} else if (space->id != TRX_SYS_SPACE || space->size_in_header) {
 		/* If this is not the first-time open, do nothing.
 		For the system tablespace, we always get invoked as
 		first=false, so we detect the true first-time-open based
-		on size_in_header and proceed to initiailze the data. */
+		on size_in_header and proceed to initialize the data. */
 		return true;
+	} else {
+		/* Initialize the size of predefined tablespaces
+		to FSP_SIZE. */
+		space->committed_size = size;
 	}
 
 	ut_ad(space->free_limit == 0 || space->free_limit == free_limit);
diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc
index 9612d95690b..6f46479291b 100644
--- a/storage/innobase/row/row0import.cc
+++ b/storage/innobase/row/row0import.cc
@@ -560,14 +560,6 @@ protected:
 	/** Space id of the file being iterated over. */
 	ulint			m_space;
 
-	/** Minimum page number for which the free list has not been
-	initialized: the pages >= this limit are, by definition, free;
-	note that in a single-table tablespace where size < 64 pages,
-	this number is 64, i.e., we have initialized the space about
-	the first extent, but have not physically allocted those pages
-	to the file. @see FSP_LIMIT. */
-	ulint			m_free_limit;
-
 	/** Current size of the space in pages */
 	ulint			m_size;
 
@@ -627,7 +619,6 @@ AbstractCallback::init(
 	}
 
 	m_size  = mach_read_from_4(page + FSP_SIZE);
-	m_free_limit = mach_read_from_4(page + FSP_FREE_LIMIT);
 	if (m_space == ULINT_UNDEFINED) {
 		m_space = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_ID
 					   + page);
diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc
index 5dbf01c6b49..9f7f039bde1 100644
--- a/storage/innobase/srv/srv0start.cc
+++ b/storage/innobase/srv/srv0start.cc
@@ -583,6 +583,7 @@ err_exit:
   {
     space->size= file->size= ulint(size >> srv_page_size_shift);
     space->size_in_header= SRV_UNDO_TABLESPACE_SIZE_IN_PAGES;
+    space->committed_size= SRV_UNDO_TABLESPACE_SIZE_IN_PAGES;
   }
   else
   {
@@ -1590,6 +1591,8 @@ file_checked:
 			if (sum_of_new_sizes > 0) {
 				/* New data file(s) were added */
 				mtr.start();
+				mtr.x_lock_space(fil_system.sys_space,
+						 __FILE__, __LINE__);
 				buf_block_t* block = buf_page_get(
 					page_id_t(0, 0), 0,
 					RW_SX_LATCH, &mtr);
diff --git a/storage/innobase/trx/trx0rseg.cc b/storage/innobase/trx/trx0rseg.cc
index 28268ed6bf0..b90ad9b73c8 100644
--- a/storage/innobase/trx/trx0rseg.cc
+++ b/storage/innobase/trx/trx0rseg.cc
@@ -312,7 +312,7 @@ trx_rseg_header_create(
 {
 	buf_block_t*	block;
 
-	ut_ad(mtr->memo_contains(space->latch, MTR_MEMO_X_LOCK));
+	ut_ad(mtr->memo_contains(*space));
 	ut_ad(!sys_header == (space == fil_system.temp_space));
 
 	/* Allocate a new file segment for the rollback segment */
diff --git a/storage/innobase/trx/trx0sys.cc b/storage/innobase/trx/trx0sys.cc
index 52b344498b1..7144ccff6f5 100644
--- a/storage/innobase/trx/trx0sys.cc
+++ b/storage/innobase/trx/trx0sys.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, 2019, MariaDB Corporation.
+Copyright (c) 2017, 2020, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software