From b112c9dfaacbcb7c3548414c6f402114663223dc Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Sat, 11 Sep 2021 18:32:07 +0200
Subject: Fix Connect build with MSVC+Ninja

---
 storage/connect/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt
index 0100f80415d..5c1bdef4c5c 100644
--- a/storage/connect/CMakeLists.txt
+++ b/storage/connect/CMakeLists.txt
@@ -425,6 +425,7 @@ IF(MSVC)
   # Temporarily disable "conversion from size_t .."
   IF(CMAKE_SIZEOF_VOID_P EQUAL 8)
     SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4267")
+    SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4267")
   ENDIF()
   SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4996")
   string(REPLACE "/permissive-" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
-- 
cgit v1.2.1


From 696de6d06c0eeaf7b20d5f89278ed7d62a9f204f Mon Sep 17 00:00:00 2001
From: Thirunarayanan Balathandayuthapani <thiru@mariadb.com>
Date: Mon, 13 Sep 2021 12:37:07 +0530
Subject: MDEV-25702 Auxiliary FTS table evicts during optimize table

InnoDB could evict the fts auxiliary table in
row_fts_merge_insert(). So bulk insert could be
dealing with garbage FTS auxiliary table.Patch
should delay closing the table in row_fts_merge_insert().
---
 storage/innobase/row/row0ftsort.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc
index 29e126290cf..c274c63182e 100644
--- a/storage/innobase/row/row0ftsort.cc
+++ b/storage/innobase/row/row0ftsort.cc
@@ -1664,7 +1664,6 @@ row_fts_merge_insert(
 	aux_table = dict_table_open_on_name(aux_table_name, FALSE, FALSE,
 					    DICT_ERR_IGNORE_NONE);
 	ut_ad(aux_table != NULL);
-	dict_table_close(aux_table, FALSE, FALSE);
 	aux_index = dict_table_get_first_index(aux_table);
 
 	/* Create bulk load instance */
@@ -1789,6 +1788,8 @@ row_fts_merge_insert(
 	}
 
 exit:
+	dict_table_close(aux_table, FALSE, FALSE);
+
 	fts_sql_commit(trx);
 
 	trx->op_info = "";
-- 
cgit v1.2.1


From 8937762ead9623f4bcd23d445319c08e07d6c321 Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Wed, 15 Sep 2021 14:19:24 +0200
Subject: MDEV-26573 : A static analyzer warning about ds_archive.cc

This file had not been compiled for long time.
Remove this from the tree.
---
 extra/mariabackup/datasink.cc   |   8 --
 extra/mariabackup/datasink.h    |   1 -
 extra/mariabackup/ds_archive.cc | 281 ----------------------------------------
 extra/mariabackup/ds_archive.h  |  28 ----
 4 files changed, 318 deletions(-)
 delete mode 100644 extra/mariabackup/ds_archive.cc
 delete mode 100644 extra/mariabackup/ds_archive.h

diff --git a/extra/mariabackup/datasink.cc b/extra/mariabackup/datasink.cc
index 4235a46ea00..ecd60a7df95 100644
--- a/extra/mariabackup/datasink.cc
+++ b/extra/mariabackup/datasink.cc
@@ -22,7 +22,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335  USA
 #include "common.h"
 #include "datasink.h"
 #include "ds_compress.h"
-#include "ds_archive.h"
 #include "ds_xbstream.h"
 #include "ds_local.h"
 #include "ds_stdout.h"
@@ -44,13 +43,6 @@ ds_create(const char *root, ds_type_t type)
 	case DS_TYPE_LOCAL:
 		ds = &datasink_local;
 		break;
-	case DS_TYPE_ARCHIVE:
-#ifdef HAVE_LIBARCHIVE
-		ds = &datasink_archive;
-#else
-		die("mariabackup was built without libarchive support");
-#endif
-		break;
 	case DS_TYPE_XBSTREAM:
 		ds = &datasink_xbstream;
 		break;
diff --git a/extra/mariabackup/datasink.h b/extra/mariabackup/datasink.h
index 5c82556b9ba..4bede4ec9e7 100644
--- a/extra/mariabackup/datasink.h
+++ b/extra/mariabackup/datasink.h
@@ -63,7 +63,6 @@ static inline int dummy_remove(const char *) {
 typedef enum {
 	DS_TYPE_STDOUT,
 	DS_TYPE_LOCAL,
-	DS_TYPE_ARCHIVE,
 	DS_TYPE_XBSTREAM,
 	DS_TYPE_COMPRESS,
 	DS_TYPE_ENCRYPT,
diff --git a/extra/mariabackup/ds_archive.cc b/extra/mariabackup/ds_archive.cc
deleted file mode 100644
index 18f13fbcf3a..00000000000
--- a/extra/mariabackup/ds_archive.cc
+++ /dev/null
@@ -1,281 +0,0 @@
-/******************************************************
-Copyright (c) 2013 Percona LLC and/or its affiliates.
-
-Streaming implementation for XtraBackup.
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335  USA
-
-*******************************************************/
-
-#include <my_base.h>
-#include <archive.h>
-#include <archive_entry.h>
-#include "common.h"
-#include "datasink.h"
-
-#if ARCHIVE_VERSION_NUMBER < 3000000
-#define archive_write_add_filter_none(X) archive_write_set_compression_none(X)
-#define archive_write_free(X) archive_write_finish(X)
-#endif
-
-typedef struct {
-	struct archive	*archive;
-	ds_file_t	*dest_file;
-	pthread_mutex_t	mutex;
-} ds_archive_ctxt_t;
-
-typedef struct {
-	struct archive_entry	*entry;
-	ds_archive_ctxt_t	*archive_ctxt;
-} ds_archive_file_t;
-
-
-/***********************************************************************
-General archive interface */
-
-static ds_ctxt_t *archive_init(const char *root);
-static ds_file_t *archive_open(ds_ctxt_t *ctxt, const char *path,
-			      MY_STAT *mystat);
-static int archive_write(ds_file_t *file, const void *buf, size_t len);
-static int archive_close(ds_file_t *file);
-static void archive_deinit(ds_ctxt_t *ctxt);
-
-datasink_t datasink_archive = {
-	&archive_init,
-	&archive_open,
-	&archive_write,
-	&archive_close,
-	&dummy_remove,
-	&archive_deinit
-};
-
-static
-int
-my_archive_open_callback(struct archive *a __attribute__((unused)),
-		      void *data __attribute__((unused)))
-{
-	return ARCHIVE_OK;
-}
-
-static
-ssize_t
-my_archive_write_callback(struct archive *a __attribute__((unused)),
-		       void *data, const void *buffer, size_t length)
-{
-	ds_archive_ctxt_t	*archive_ctxt;
-
-	archive_ctxt = (ds_archive_ctxt_t *) data;
-
-	xb_ad(archive_ctxt != NULL);
-	xb_ad(archive_ctxt->dest_file != NULL);
-
-	if (!ds_write(archive_ctxt->dest_file, buffer, length)) {
-		return length;
-	}
-	return -1;
-}
-
-static
-int
-my_archive_close_callback(struct archive *a __attribute__((unused)),
-			  void *data __attribute__((unused)))
-{
-	return ARCHIVE_OK;
-}
-
-static
-ds_ctxt_t *
-archive_init(const char *root __attribute__((unused)))
-{
-	ds_ctxt_t		*ctxt;
-	ds_archive_ctxt_t	*archive_ctxt;
-	struct archive		*a;
-
-	ctxt = my_malloc(sizeof(ds_ctxt_t) + sizeof(ds_archive_ctxt_t),
-			 MYF(MY_FAE));
-	archive_ctxt = (ds_archive_ctxt_t *)(ctxt + 1);
-
-	if (pthread_mutex_init(&archive_ctxt->mutex, NULL)) {
-		msg("archive_init: pthread_mutex_init() failed.\n");
-		goto err;
-	}
-
-	a = archive_write_new();
-	if (a == NULL) {
-		msg("archive_write_new() failed.\n");
-		goto err;
-	}
-
-	archive_ctxt->archive = a;
-	archive_ctxt->dest_file = NULL;
-
-        if(archive_write_add_filter_none(a) != ARCHIVE_OK ||
-	    archive_write_set_format_pax_restricted(a) != ARCHIVE_OK ||
-	    /* disable internal buffering so we don't have to flush the
-	    output in xtrabackup */
-	    archive_write_set_bytes_per_block(a, 0) != ARCHIVE_OK) {
-		msg("failed to set libarchive archive options: %s\n",
-		    archive_error_string(a));
-                archive_write_free(a);
-		goto err;
-	}
-
-	if (archive_write_open(a, archive_ctxt, my_archive_open_callback,
-			       my_archive_write_callback,
-			       my_archive_close_callback) != ARCHIVE_OK) {
-		msg("cannot open output archive.\n");
-		return NULL;
-	}
-
-	ctxt->ptr = archive_ctxt;
-
-	return ctxt;
-
-err:
-	my_free(ctxt);
-	return NULL;
-}
-
-static
-ds_file_t *
-archive_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
-{
-	ds_archive_ctxt_t	*archive_ctxt;
-	ds_ctxt_t		*dest_ctxt;
-	ds_file_t		*file;
-	ds_archive_file_t	*archive_file;
-
-	struct archive		*a;
-	struct archive_entry	*entry;
-
-	xb_ad(ctxt->pipe_ctxt != NULL);
-	dest_ctxt = ctxt->pipe_ctxt;
-
-	archive_ctxt = (ds_archive_ctxt_t *) ctxt->ptr;
-
-	pthread_mutex_lock(&archive_ctxt->mutex);
-	if (archive_ctxt->dest_file == NULL) {
-		archive_ctxt->dest_file = ds_open(dest_ctxt, path, mystat);
-		if (archive_ctxt->dest_file == NULL) {
-			return NULL;
-		}
-	}
-	pthread_mutex_unlock(&archive_ctxt->mutex);
-
-	file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
-				       sizeof(ds_archive_file_t),
-				       MYF(MY_FAE));
-
-	archive_file = (ds_archive_file_t *) (file + 1);
-
-	a = archive_ctxt->archive;
-
-	entry = archive_entry_new();
-	if (entry == NULL) {
-		msg("archive_entry_new() failed.\n");
-		goto err;
-	}
-
-	archive_entry_set_size(entry, mystat->st_size);
-	archive_entry_set_mode(entry, 0660);
-	archive_entry_set_filetype(entry, AE_IFREG);
-	archive_entry_set_pathname(entry, path);
-	archive_entry_set_mtime(entry, mystat->st_mtime, 0);
-
-	archive_file->entry = entry;
-	archive_file->archive_ctxt = archive_ctxt;
-
-	if (archive_write_header(a, entry) != ARCHIVE_OK) {
-		msg("archive_write_header() failed.\n");
-		archive_entry_free(entry);
-		goto err;
-	}
-
-	file->ptr = archive_file;
-	file->path = archive_ctxt->dest_file->path;
-
-	return file;
-
-err:
-	if (archive_ctxt->dest_file) {
-		ds_close(archive_ctxt->dest_file);
-		archive_ctxt->dest_file = NULL;
-	}
-	my_free(file);
-
-	return NULL;
-}
-
-static
-int
-archive_write(ds_file_t *file, const void *buf, size_t len)
-{
-	ds_archive_file_t	*archive_file;
-	struct archive		*a;
-
-	archive_file = (ds_archive_file_t *) file->ptr;
-
-	a = archive_file->archive_ctxt->archive;
-
-	xb_ad(archive_file->archive_ctxt->dest_file != NULL);
-	if (archive_write_data(a, buf, len) < 0) {
-		msg("archive_write_data() failed: %s (errno = %d)\n",
-		    archive_error_string(a), archive_errno(a));
-		return 1;
-	}
-
-	return 0;
-}
-
-static
-int
-archive_close(ds_file_t *file)
-{
-	ds_archive_file_t	*archive_file;
-	int			rc = 0;
-
-	archive_file = (ds_archive_file_t *)file->ptr;
-
-	archive_entry_free(archive_file->entry);
-
-	my_free(file);
-
-	return rc;
-}
-
-static
-void
-archive_deinit(ds_ctxt_t *ctxt)
-{
-	struct archive *a;
-	ds_archive_ctxt_t	*archive_ctxt;
-
-	archive_ctxt = (ds_archive_ctxt_t *) ctxt->ptr;
-
-	a = archive_ctxt->archive;
-
-	if (archive_write_close(a) != ARCHIVE_OK) {
-		msg("archive_write_close() failed.\n");
-	}
-	archive_write_free(a);
-
-	if (archive_ctxt->dest_file) {
-		ds_close(archive_ctxt->dest_file);
-		archive_ctxt->dest_file = NULL;
-	}
-
-	pthread_mutex_destroy(&archive_ctxt->mutex);
-
-	my_free(ctxt);
-}
diff --git a/extra/mariabackup/ds_archive.h b/extra/mariabackup/ds_archive.h
deleted file mode 100644
index f419fca0c9f..00000000000
--- a/extra/mariabackup/ds_archive.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/******************************************************
-Copyright (c) 2013 Percona LLC and/or its affiliates.
-
-Streaming interface for XtraBackup.
-
-This program is free software; you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation; version 2 of the License.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1335  USA
-
-*******************************************************/
-
-#ifndef DS_ARCHIVE_H
-#define DS_ARCHIVE_H
-
-#include "datasink.h"
-
-extern datasink_t datasink_archive;
-
-#endif
-- 
cgit v1.2.1


From b1351c15946349f9daa7e5297fb2ac6f3139e4a8 Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Wed, 15 Sep 2021 14:55:45 +0200
Subject: MDEV-26574 An improper locking bug due to unreleased lock in the
 ds_xbstream.cc

release lock in all as cases n xbstream_open, also fix the case where malloc would return NULL.
---
 extra/mariabackup/ds_xbstream.cc | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/extra/mariabackup/ds_xbstream.cc b/extra/mariabackup/ds_xbstream.cc
index 3b60456f8ed..798d7da6da4 100644
--- a/extra/mariabackup/ds_xbstream.cc
+++ b/extra/mariabackup/ds_xbstream.cc
@@ -126,15 +126,19 @@ xbstream_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat)
 	pthread_mutex_lock(&stream_ctxt->mutex);
 	if (stream_ctxt->dest_file == NULL) {
 		stream_ctxt->dest_file = ds_open(dest_ctxt, path, mystat);
-		if (stream_ctxt->dest_file == NULL) {
-			return NULL;
-		}
 	}
 	pthread_mutex_unlock(&stream_ctxt->mutex);
+	if (stream_ctxt->dest_file == NULL) {
+		return NULL;
+	}
 
 	file = (ds_file_t *) my_malloc(sizeof(ds_file_t) +
 				       sizeof(ds_stream_file_t),
 				       MYF(MY_FAE));
+	if (!file) {
+		msg("my_malloc() failed.");
+		goto err;
+	}
 	stream_file = (ds_stream_file_t *) (file + 1);
 
 	xbstream = stream_ctxt->xbstream;
-- 
cgit v1.2.1


From dabc3329c30d3be53449d49f501acaf0b0788dbf Mon Sep 17 00:00:00 2001
From: Daniel Black <daniel@mariadb.org>
Date: Thu, 16 Sep 2021 20:42:26 +1000
Subject: MDEV-26622: man mysqldump - insert-ignore not insert-into

Thanks Karl Levik
---
 man/mysqldump.1 | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/man/mysqldump.1 b/man/mysqldump.1
index 2cfe308836a..907eeda4d38 100644
--- a/man/mysqldump.1
+++ b/man/mysqldump.1
@@ -2261,7 +2261,7 @@ servers \- remote (federated) servers as \fBCREATE SERVER\fR\&.
 .sp -1
 .IP \(bu 2.3
 .\}
-stats \- statistics tables, InnoDB and Engine Independent Table Statistics (EITS), are dumped as \fBREPLACE INTO\fR (or \fBINSERT IGNORE\fR if \fB\-\-insert\-into\fR is specified) statements without (re)creating tables\&.
+stats \- statistics tables, InnoDB and Engine Independent Table Statistics (EITS), are dumped as \fBREPLACE INTO\fR (or \fBINSERT IGNORE\fR if \fB\-\-insert\-ignore\fR is specified) statements without (re)creating tables\&.
 .RE
 .RS 4
 .ie n \{\
@@ -2271,17 +2271,17 @@ stats \- statistics tables, InnoDB and Engine Independent Table Statistics (EITS
 .sp -1
 .IP \(bu 2.3
 .\}
-timezones \- timezone related system tables dumped as \fBREPLACE INTO\fR (or \fBINSERT IGNORE\fR if \fB\-\-insert\-into\fR is specified) statements without (re)creating tables\&.
+timezones \- timezone related system tables dumped as \fBREPLACE INTO\fR (or \fBINSERT IGNORE\fR if \fB\-\-insert\-ignore\fR is specified) statements without (re)creating tables\&.
 .RE
 .sp
-The format of the output is affected by \fB\-\-replace\fR and \fB\-\-insert\-into\fR\&. The \fB\-\-replace\fR option will output \fBCREATE OR REPLACE\fR
+The format of the output is affected by \fB\-\-replace\fR and \fB\-\-insert\-ignore\fR\&. The \fB\-\-replace\fR option will output \fBCREATE OR REPLACE\fR
 forms of SQL, and also \fBDROP IF EXISTS\fR prior to \fBCREATE\fR, if a \fBCREATE OR REPLACE\fR option isn't available.
 .sp
 With \fB\-\-system=user\fR (or \fBall\fR), and \fB\-\-replace\fR, SQL is generated to generate an error if attempting to import the dump with a connection user that is being replaced within the dump\&.
 .sp
-The \fB\-\-insert\-into\fR option will cause \fBCREATE IF NOT EXIST\fR forms of SQL to generated if available.
+The \fB\-\-insert\-ignore\fR option will cause \fBCREATE IF NOT EXIST\fR forms of SQL to generated if available.
 .sp
-For stats, and timezones, \fB\-\-replace\fR and \fB\-\-insert\-into\fR have the usual effects.
+For stats, and timezones, \fB\-\-replace\fR and \fB\-\-insert\-ignore\fR have the usual effects.
 .sp
 Enabling specific options here will cause the relevant tables in the mysql database to be ignored when dumping the mysql database or \fB\-\-all\-databases\fR\&.
 .sp
-- 
cgit v1.2.1


From dce490e9d4d91a7e00111bd515db558cadc19d06 Mon Sep 17 00:00:00 2001
From: Oleksandr Byelkin <sanja@mariadb.com>
Date: Wed, 15 Sep 2021 16:37:54 +0200
Subject: Give less memory to get reliable error.

---
 mysql-test/r/truncate_notembedded.result | 4 ++--
 mysql-test/t/truncate_notembedded.test   | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/mysql-test/r/truncate_notembedded.result b/mysql-test/r/truncate_notembedded.result
index 18a01c684e7..dabd5474141 100644
--- a/mysql-test/r/truncate_notembedded.result
+++ b/mysql-test/r/truncate_notembedded.result
@@ -5,7 +5,7 @@
 CREATE TABLE t1 (a INT) ENGINE=MyISAM;
 LOCK TABLE t1 READ;
 connect  con1,localhost,root,,test;
-SET SESSION max_session_mem_used= 65536;
+SET SESSION max_session_mem_used= 45500;
 LOCK TABLE t1 WRITE;
 connection default;
 SELECT * FROM t1;
@@ -13,7 +13,7 @@ a
 UNLOCK TABLES;
 connection con1;
 TRUNCATE TABLE t1;
-ERROR HY000: The MariaDB server is running with the --max-thread-mem-used=65536 option so it cannot execute this statement
+ERROR HY000: The MariaDB server is running with the --max-thread-mem-used=45500 option so it cannot execute this statement
 disconnect con1;
 connection default;
 DROP TABLE t1;
diff --git a/mysql-test/t/truncate_notembedded.test b/mysql-test/t/truncate_notembedded.test
index 7e186c8f7d5..3addad2d975 100644
--- a/mysql-test/t/truncate_notembedded.test
+++ b/mysql-test/t/truncate_notembedded.test
@@ -9,7 +9,7 @@ CREATE TABLE t1 (a INT) ENGINE=MyISAM;
 LOCK TABLE t1 READ;
 
 --connect (con1,localhost,root,,test)
-SET SESSION max_session_mem_used= 65536;
+SET SESSION max_session_mem_used= 45500;
 --send
   LOCK TABLE t1 WRITE;
 
-- 
cgit v1.2.1


From 496d3dded4559ef8cb64ee4fc68eb4d461d78bd5 Mon Sep 17 00:00:00 2001
From: Thirunarayanan Balathandayuthapani <thiru@mariadb.com>
Date: Fri, 17 Sep 2021 19:37:29 +0530
Subject: MDEV-15675 encryption.innodb_encryption failed in buildbot with
 timeout

Test case fail to include undo tablespace while waiting for the
encryption thread to encrypt all existing tablespace
---
 mysql-test/suite/encryption/r/innodb_encryption.result | 2 +-
 mysql-test/suite/encryption/t/innodb_encryption.test   | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/mysql-test/suite/encryption/r/innodb_encryption.result b/mysql-test/suite/encryption/r/innodb_encryption.result
index 75614c6b88c..22b4c3a5ad6 100644
--- a/mysql-test/suite/encryption/r/innodb_encryption.result
+++ b/mysql-test/suite/encryption/r/innodb_encryption.result
@@ -19,7 +19,7 @@ innodb_system
 # Success!
 # Now turn off encryption and wait for threads to decrypt everything
 SET GLOBAL innodb_encrypt_tables = off;
-# Wait max 10 min for key encryption threads to encrypt all spaces
+# Wait max 10 min for key encryption threads to decrypt all spaces
 SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0
 AND NAME NOT LIKE 'innodb_undo%' AND NAME NOT LIKE 'mysql/innodb_%_stats';
 NAME
diff --git a/mysql-test/suite/encryption/t/innodb_encryption.test b/mysql-test/suite/encryption/t/innodb_encryption.test
index cd844af0649..303e74fdba4 100644
--- a/mysql-test/suite/encryption/t/innodb_encryption.test
+++ b/mysql-test/suite/encryption/t/innodb_encryption.test
@@ -14,7 +14,9 @@ SHOW VARIABLES LIKE 'innodb_encrypt%';
 
 SET GLOBAL innodb_encrypt_tables = ON;
 
---let $tables_count= `select count(*) + 1 from information_schema.tables where engine = 'InnoDB'`
+let $undo_count= `select @@global.innodb_undo_tablespaces`;
+
+--let $tables_count= `select count(*) + 1 + $undo_count from information_schema.tables where engine = 'InnoDB'`
 
 --echo # Wait max 10 min for key encryption threads to encrypt all spaces
 --let $wait_timeout= 600
@@ -33,7 +35,7 @@ AND NAME NOT LIKE 'innodb_undo%' AND NAME NOT LIKE 'mysql/innodb_%_stats';
 --echo # Now turn off encryption and wait for threads to decrypt everything
 SET GLOBAL innodb_encrypt_tables = off;
 
---echo # Wait max 10 min for key encryption threads to encrypt all spaces
+--echo # Wait max 10 min for key encryption threads to decrypt all spaces
 --let $wait_timeout= 600
 --let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0;
 --source include/wait_condition.inc
-- 
cgit v1.2.1


From 56843d62f935d3859bf45b5ea5876a7b97fc118d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Sat, 18 Sep 2021 14:19:55 +0300
Subject: MDEV-24258 fixup: Correct a condition

dict_stats_process_entry_from_defrag_pool(): Restore a condition as it was
before commit 82b7c561b7919fa24e3d24b3f04a16046e24374f.
---
 storage/innobase/dict/dict0defrag_bg.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/storage/innobase/dict/dict0defrag_bg.cc b/storage/innobase/dict/dict0defrag_bg.cc
index 8dd1a8601aa..396b66941c5 100644
--- a/storage/innobase/dict/dict0defrag_bg.cc
+++ b/storage/innobase/dict/dict0defrag_bg.cc
@@ -190,8 +190,8 @@ static void dict_stats_process_entry_from_defrag_pool()
   /* If the table is no longer cached, we've already lost the in
   memory stats so there's nothing really to write to disk. */
   dict_table_t *table= dict_sys.find_table(table_id);
-  dict_index_t *index= table && table->corrupted
-    ? nullptr : dict_table_find_index_on_id(table, index_id);
+  dict_index_t *index= table && !table->corrupted
+    ? dict_table_find_index_on_id(table, index_id) : nullptr;
   const bool save= index && !index->is_corrupted();
   if (save)
     table->acquire();
-- 
cgit v1.2.1


From b740b2356d539a27f8a7a9e9b6455f31bc6c9196 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Sat, 18 Sep 2021 14:39:32 +0300
Subject: MDEV-25919 fixup: Acquire MDL also in defragmentation

dict_stats_process_entry_from_defrag_pool(): Acquire MDL on the table
for which we are invoking dict_stats_save_defrag_stats(), to avoid
any race condition with DROP TABLE or similar operations.
---
 storage/innobase/dict/dict0defrag_bg.cc   | 31 ++++++++++++++-----------------
 storage/innobase/dict/dict0stats_bg.cc    |  2 +-
 storage/innobase/include/dict0defrag_bg.h |  8 +++-----
 3 files changed, 18 insertions(+), 23 deletions(-)

diff --git a/storage/innobase/dict/dict0defrag_bg.cc b/storage/innobase/dict/dict0defrag_bg.cc
index 396b66941c5..35ffee7e0d0 100644
--- a/storage/innobase/dict/dict0defrag_bg.cc
+++ b/storage/innobase/dict/dict0defrag_bg.cc
@@ -173,10 +173,10 @@ dict_stats_defrag_pool_del(
 /*****************************************************************//**
 Get the first index that has been added for updating persistent defrag
 stats and eventually save its stats. */
-static void dict_stats_process_entry_from_defrag_pool()
+static void dict_stats_process_entry_from_defrag_pool(THD *thd)
 {
-  table_id_t	table_id;
-  index_id_t	index_id;
+  table_id_t table_id;
+  index_id_t index_id;
 
   ut_ad(!srv_read_only_mode);
 
@@ -185,31 +185,28 @@ static void dict_stats_process_entry_from_defrag_pool()
     /* no index in defrag pool */
     return;
 
-  dict_sys.freeze(SRW_LOCK_CALL);
-
   /* If the table is no longer cached, we've already lost the in
   memory stats so there's nothing really to write to disk. */
-  dict_table_t *table= dict_sys.find_table(table_id);
-  dict_index_t *index= table && !table->corrupted
-    ? dict_table_find_index_on_id(table, index_id) : nullptr;
-  const bool save= index && !index->is_corrupted();
-  if (save)
-    table->acquire();
-  dict_sys.unfreeze();
-  if (save)
+  MDL_ticket *mdl= nullptr;
+  if (dict_table_t *table=
+      dict_table_open_on_id(table_id, false, DICT_TABLE_OP_OPEN_ONLY_IF_CACHED,
+                            thd, &mdl))
   {
-    dict_stats_save_defrag_stats(index);
-    table->release();
+    if (dict_index_t *index= !table->corrupted
+        ? dict_table_find_index_on_id(table, index_id) : nullptr)
+      if (!index->is_corrupted())
+        dict_stats_save_defrag_stats(index);
+    dict_table_close(table, false, thd, mdl);
   }
 }
 
 /**
 Get the first index that has been added for updating persistent defrag
 stats and eventually save its stats. */
-void dict_defrag_process_entries_from_defrag_pool()
+void dict_defrag_process_entries_from_defrag_pool(THD *thd)
 {
   while (!defrag_pool.empty())
-    dict_stats_process_entry_from_defrag_pool();
+    dict_stats_process_entry_from_defrag_pool(thd);
 }
 
 /*********************************************************************//**
diff --git a/storage/innobase/dict/dict0stats_bg.cc b/storage/innobase/dict/dict0stats_bg.cc
index 91c18c95851..676656122a3 100644
--- a/storage/innobase/dict/dict0stats_bg.cc
+++ b/storage/innobase/dict/dict0stats_bg.cc
@@ -391,7 +391,7 @@ static void dict_stats_func(void*)
   THD *thd= innobase_create_background_thd("InnoDB statistics");
   set_current_thd(thd);
   while (dict_stats_process_entry_from_recalc_pool(thd)) {}
-  dict_defrag_process_entries_from_defrag_pool();
+  dict_defrag_process_entries_from_defrag_pool(thd);
   set_current_thd(nullptr);
   innobase_destroy_background_thd(thd);
 }
diff --git a/storage/innobase/include/dict0defrag_bg.h b/storage/innobase/include/dict0defrag_bg.h
index 0edc6304788..679484ad64e 100644
--- a/storage/innobase/include/dict0defrag_bg.h
+++ b/storage/innobase/include/dict0defrag_bg.h
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2016, 2020, MariaDB Corporation.
+Copyright (c) 2016, 2021, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -80,12 +80,10 @@ dict_stats_defrag_pool_del(
 					all entries for the table */
 	const dict_index_t*	index);	/*!< in: index to remove */
 
-/*****************************************************************//**
+/**
 Get the first index that has been added for updating persistent defrag
 stats and eventually save its stats. */
-void
-dict_defrag_process_entries_from_defrag_pool();
-/*===========================================*/
+void dict_defrag_process_entries_from_defrag_pool(THD *thd);
 
 /*********************************************************************//**
 Save defragmentation result.
-- 
cgit v1.2.1


From 3209bc667fd3aeb7b3c3709ca927c2e77a5bf273 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Sat, 18 Sep 2021 15:47:52 +0300
Subject: MDEV-26636: InnoDB defragmentation statistics cause races on
 TEMPORARY TABLE

btr_defragment_save_defrag_stats_if_needed(): Do not save
defragmentation statistics for temporary tables.
They are exempt of defragmentation anyway
(ha_innobase::optimize() never invokes defragmentation for them),
and the user-visible names are not available inside InnoDB.

Furthermore, InnoDB assumes that temporary tables are never accessed
by other threads than the one that handles the session with which
the temporary table is associated with.

Furthermore, we simplify the test innodb.innodb_defrag_stats
and include a test case that demonstrates that defragmentation
statistics are no longer being saved for temporary tables.
---
 .../suite/innodb/r/innodb_defrag_stats.result      | 103 +++++++--------------
 mysql-test/suite/innodb/t/innodb_defrag_stats.test |  72 +++++---------
 storage/innobase/btr/btr0defragment.cc             |   3 +-
 3 files changed, 62 insertions(+), 116 deletions(-)

diff --git a/mysql-test/suite/innodb/r/innodb_defrag_stats.result b/mysql-test/suite/innodb/r/innodb_defrag_stats.result
index 7092688f07b..6c5fe1817e2 100644
--- a/mysql-test/suite/innodb/r/innodb_defrag_stats.result
+++ b/mysql-test/suite/innodb/r/innodb_defrag_stats.result
@@ -1,22 +1,8 @@
-DROP TABLE if exists t1;
-select @@global.innodb_stats_persistent;
-@@global.innodb_stats_persistent
-0
-set global innodb_defragment_stats_accuracy = 20;
+SET GLOBAL innodb_defragment_stats_accuracy = 20;
+DELETE FROM mysql.innodb_index_stats;
 # Create table.
 CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARCHAR(256), KEY SECOND(a, b)) ENGINE=INNODB;
-# Populate data
-INSERT INTO t1 VALUES(1, REPEAT('A', 256));
-INSERT INTO t1 (b) SELECT b from t1;
-INSERT INTO t1 (b) SELECT b from t1;
-INSERT INTO t1 (b) SELECT b from t1;
-INSERT INTO t1 (b) SELECT b from t1;
-INSERT INTO t1 (b) SELECT b from t1;
-INSERT INTO t1 (b) SELECT b from t1;
-INSERT INTO t1 (b) SELECT b from t1;
-INSERT INTO t1 (b) SELECT b from t1;
-INSERT INTO t1 (b) SELECT b from t1;
-INSERT INTO t1 (b) SELECT b from t1;
+INSERT INTO t1 SELECT seq, REPEAT('A', 256) FROM seq_1_to_1024;
 # Not enough page splits to trigger persistent stats write yet.
 select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
 count(stat_value) = 0
@@ -27,7 +13,7 @@ count(stat_value) = 0
 select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
 count(stat_value) = 0
 1
-INSERT INTO t1 (b) SELECT b from t1;
+INSERT INTO t1 SELECT seq, REPEAT('A', 256) FROM seq_1025_to_2048;
 # Persistent stats recorded.
 select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
 count(stat_value) > 0
@@ -39,6 +25,7 @@ select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like
 count(stat_value) > 0
 1
 # Delete some rows.
+BEGIN;
 delete from t1 where a between 100 * 20 and 100 * 20 + 30;
 delete from t1 where a between 100 * 19 and 100 * 19 + 30;
 delete from t1 where a between 100 * 18 and 100 * 18 + 30;
@@ -59,8 +46,7 @@ delete from t1 where a between 100 * 4 and 100 * 4 + 30;
 delete from t1 where a between 100 * 3 and 100 * 3 + 30;
 delete from t1 where a between 100 * 2 and 100 * 2 + 30;
 delete from t1 where a between 100 * 1 and 100 * 1 + 30;
-# Server Restarted
-# Confirm persistent stats still there after restart.
+COMMIT;
 select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
 count(stat_value) > 0
 1
@@ -73,9 +59,6 @@ count(stat_value) > 0
 optimize table t1;
 Table	Op	Msg_type	Msg_text
 test.t1	optimize	status	OK
-select sleep(2);
-sleep(2)
-0
 select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
 count(stat_value) > 0
 1
@@ -108,9 +91,6 @@ count(stat_value) > 0
 1
 # Table rename should cause stats rename.
 rename table t1 to t2;
-select sleep(1);
-sleep(1)
-0
 select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
 count(stat_value) = 0
 1
@@ -129,47 +109,36 @@ count(stat_value) > 0
 select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_leaf_pages_defrag');
 count(stat_value) > 0
 1
-# Drop index should cause stats drop.
+# Drop index should cause stats drop, but will not.
 drop index SECOND on t2;
-select sleep(3);
-sleep(3)
-0
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and index_name = 'SECOND' and  stat_name in ('n_page_split');
-count(stat_value) > 0
-1
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and index_name = 'SECOND' and  stat_name in ('n_pages_freed');
-count(stat_value) > 0
-1
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and index_name = 'SECOND' and stat_name in ('n_leaf_pages_defrag');
-count(stat_value) > 0
-1
-Server Restarted
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
-count(stat_value) = 0
-1
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
-count(stat_value) = 0
-1
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
-count(stat_value) = 0
-1
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_page_split');
-count(stat_value) > 0
-1
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_pages_freed');
-count(stat_value) > 0
-1
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_leaf_pages_defrag');
-count(stat_value) > 0
-1
+SELECT stat_name, stat_value>0 FROM mysql.innodb_index_stats
+WHERE table_name like '%t2%' AND index_name='SECOND';
+stat_name	stat_value>0
+n_leaf_pages_defrag	1
+n_leaf_pages_reserved	1
+n_page_split	1
+n_pages_freed	1
+#
+# MDEV-26636: Statistics must not be written for temporary tables
+#
+SET GLOBAL innodb_defragment_stats_accuracy = 1;
+CREATE TEMPORARY TABLE t (a INT PRIMARY KEY, c CHAR(255) NOT NULL)
+ENGINE=InnoDB;
+INSERT INTO t SELECT seq, '' FROM seq_1_to_100;
+SELECT * FROM mysql.innodb_index_stats where table_name like '%t1%';
+database_name	table_name	index_name	last_update	stat_name	stat_value	sample_size	stat_description
+SELECT table_name, index_name, stat_name, stat_value>0
+FROM mysql.innodb_index_stats;
+table_name	index_name	stat_name	stat_value>0
+t2	PRIMARY	n_leaf_pages_defrag	1
+t2	PRIMARY	n_leaf_pages_reserved	1
+t2	PRIMARY	n_page_split	1
+t2	PRIMARY	n_pages_freed	1
+t2	SECOND	n_leaf_pages_defrag	1
+t2	SECOND	n_leaf_pages_reserved	1
+t2	SECOND	n_page_split	1
+t2	SECOND	n_pages_freed	1
 # Clean up
 DROP TABLE t2;
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_page_split');
-count(stat_value) = 0
-1
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_pages_freed');
-count(stat_value) = 0
-1
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_leaf_pages_defrag');
-count(stat_value) = 0
-1
+SELECT * FROM mysql.innodb_index_stats;
+database_name	table_name	index_name	last_update	stat_name	stat_value	sample_size	stat_description
diff --git a/mysql-test/suite/innodb/t/innodb_defrag_stats.test b/mysql-test/suite/innodb/t/innodb_defrag_stats.test
index 2a5026a68e5..e1e88a07477 100644
--- a/mysql-test/suite/innodb/t/innodb_defrag_stats.test
+++ b/mysql-test/suite/innodb/t/innodb_defrag_stats.test
@@ -1,41 +1,23 @@
 --source include/have_innodb.inc
---source include/big_test.inc
 --source include/not_valgrind.inc
 --source include/not_embedded.inc
+--source include/have_sequence.inc
 
---disable_warnings
-DROP TABLE if exists t1;
---enable_warnings
+SET GLOBAL innodb_defragment_stats_accuracy = 20;
 
---disable_query_log
-let $innodb_defragment_stats_accuracy_orig=`select @@innodb_defragment_stats_accuracy`;
---enable_query_log
-
-select @@global.innodb_stats_persistent;
-set global innodb_defragment_stats_accuracy = 20;
+DELETE FROM mysql.innodb_index_stats;
 
 --echo # Create table.
 CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARCHAR(256), KEY SECOND(a, b)) ENGINE=INNODB;
 
---echo # Populate data
-INSERT INTO t1 VALUES(1, REPEAT('A', 256));
-INSERT INTO t1 (b) SELECT b from t1;
-INSERT INTO t1 (b) SELECT b from t1;
-INSERT INTO t1 (b) SELECT b from t1;
-INSERT INTO t1 (b) SELECT b from t1;
-INSERT INTO t1 (b) SELECT b from t1;
-INSERT INTO t1 (b) SELECT b from t1;
-INSERT INTO t1 (b) SELECT b from t1;
-INSERT INTO t1 (b) SELECT b from t1;
-INSERT INTO t1 (b) SELECT b from t1;
-INSERT INTO t1 (b) SELECT b from t1;
+INSERT INTO t1 SELECT seq, REPEAT('A', 256) FROM seq_1_to_1024;
 
 --echo # Not enough page splits to trigger persistent stats write yet.
 select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
 select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
 select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
 
-INSERT INTO t1 (b) SELECT b from t1;
+INSERT INTO t1 SELECT seq, REPEAT('A', 256) FROM seq_1025_to_2048;
 
 --echo # Persistent stats recorded.
 select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
@@ -43,6 +25,7 @@ select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like
 select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
 
 --echo # Delete some rows.
+BEGIN;
 let $num_delete = 20;
 while ($num_delete)
 {
@@ -50,17 +33,13 @@ while ($num_delete)
     eval delete from t1 where a between $j and $j + 30;
     dec $num_delete;
 }
+COMMIT;
 
---source include/restart_mysqld.inc
---echo # Server Restarted
-
---echo # Confirm persistent stats still there after restart.
 select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
 select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
 select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
 
 optimize table t1;
-select sleep(2);
 
 select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
 select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
@@ -84,7 +63,6 @@ select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like
 
 --echo # Table rename should cause stats rename.
 rename table t1 to t2;
-select sleep(1);
 
 select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
 select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
@@ -94,32 +72,30 @@ select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like
 select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_pages_freed');
 select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_leaf_pages_defrag');
 
---echo # Drop index should cause stats drop.
+--echo # Drop index should cause stats drop, but will not.
 drop index SECOND on t2;
-select sleep(3);
 
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and index_name = 'SECOND' and  stat_name in ('n_page_split');
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and index_name = 'SECOND' and  stat_name in ('n_pages_freed');
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and index_name = 'SECOND' and stat_name in ('n_leaf_pages_defrag');
+--sorted_result
+SELECT stat_name, stat_value>0 FROM mysql.innodb_index_stats
+WHERE table_name like '%t2%' AND index_name='SECOND';
+
+--echo #
+--echo # MDEV-26636: Statistics must not be written for temporary tables
+--echo #
+SET GLOBAL innodb_defragment_stats_accuracy = 1;
+CREATE TEMPORARY TABLE t (a INT PRIMARY KEY, c CHAR(255) NOT NULL)
+ENGINE=InnoDB;
+INSERT INTO t SELECT seq, '' FROM seq_1_to_100;
 
 --source include/restart_mysqld.inc
---echo Server Restarted
 
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_page_split');
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_pages_freed');
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and  stat_name in ('n_leaf_pages_defrag');
+SELECT * FROM mysql.innodb_index_stats where table_name like '%t1%';
 
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_page_split');
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_pages_freed');
-select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_leaf_pages_defrag');
+--sorted_result
+SELECT table_name, index_name, stat_name, stat_value>0
+FROM mysql.innodb_index_stats;
 
 --echo # Clean up
 DROP TABLE t2;
 
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_page_split');
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_pages_freed');
-select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t2%' and  stat_name in ('n_leaf_pages_defrag');
-
---disable_query_log
-EVAL SET GLOBAL innodb_defragment_stats_accuracy = $innodb_defragment_stats_accuracy_orig;
---enable_query_log
+SELECT * FROM mysql.innodb_index_stats;
diff --git a/storage/innobase/btr/btr0defragment.cc b/storage/innobase/btr/btr0defragment.cc
index 38ef1db9cec..36dd9ab8470 100644
--- a/storage/innobase/btr/btr0defragment.cc
+++ b/storage/innobase/btr/btr0defragment.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (C) 2012, 2014 Facebook, Inc. All Rights Reserved.
-Copyright (C) 2014, 2019, MariaDB Corporation.
+Copyright (C) 2014, 2021, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -312,6 +312,7 @@ btr_defragment_save_defrag_stats_if_needed(
 {
 	if (srv_defragment_stats_accuracy != 0 // stats tracking disabled
 	    && dict_index_get_space(index) != 0 // do not track system tables
+	    && !index->table->is_temporary()
 	    && index->stat_defrag_modified_counter
 	       >= srv_defragment_stats_accuracy) {
 		dict_stats_defrag_pool_add(index);
-- 
cgit v1.2.1


From f4d6d01782ad4c885f1a4041514ba58f7b7d4ad2 Mon Sep 17 00:00:00 2001
From: Julius Goryavsky <julius.goryavsky@mariadb.com>
Date: Mon, 20 Sep 2021 12:10:53 +0200
Subject: MDEV-26441: Linux-dependent construct in SST scripts

SST scripts currently use Linux-specific construction
to create a temporary directory if the path prefix for
that directory is specified by the user. This does not
work with FreeBSD. This commit adds support for FreeBSD.

No separate test required.
---
 scripts/wsrep_sst_mariabackup.sh   | 4 +++-
 scripts/wsrep_sst_rsync.sh         | 4 +++-
 scripts/wsrep_sst_xtrabackup-v2.sh | 4 +++-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh
index 562b9b929f2..54632e5f79b 100644
--- a/scripts/wsrep_sst_mariabackup.sh
+++ b/scripts/wsrep_sst_mariabackup.sh
@@ -949,8 +949,10 @@ then
         tmpdir=$(parse_cnf "$encgroups" 'tmpdir')
         if [ -z "$tmpdir" ]; then
             xtmpdir="$(mktemp -d)"
-        else
+        elif [ "$OS" = 'Linux' ]; then
             xtmpdir=$(mktemp '-d' "--tmpdir=$tmpdir")
+        else
+            xtmpdir=$(TMPDIR="$tmpdir"; mktemp '-d')
         fi
 
         wsrep_log_info "Using '$xtmpdir' as mariabackup temporary directory"
diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh
index d90e87b68f2..e16ed75cb16 100644
--- a/scripts/wsrep_sst_rsync.sh
+++ b/scripts/wsrep_sst_rsync.sh
@@ -725,8 +725,10 @@ EOF
             tmpdir=$(parse_cnf '--mysqld|sst' 'tmpdir')
             if [ -z "$tmpdir" ]; then
                tmpfile="$(mktemp)"
-            else
+            elif [ "$OS" = 'Linux' ]; then
                tmpfile=$(mktemp "--tmpdir=$tmpdir")
+            else
+               tmpfile=$(TMPDIR="$tmpdir"; mktemp '-d')
             fi
 
             wsrep_log_info "Extracting binlog files:"
diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh
index 13a4a1d25c1..999c6c649ba 100644
--- a/scripts/wsrep_sst_xtrabackup-v2.sh
+++ b/scripts/wsrep_sst_xtrabackup-v2.sh
@@ -973,8 +973,10 @@ then
         tmpdir=$(parse_cnf "$encgroups" 'tmpdir')
         if [ -z "$tmpdir" ]; then
             xtmpdir="$(mktemp -d)"
-        else
+        elif [ "$OS" = 'Linux' ]; then
             xtmpdir=$(mktemp '-d' "--tmpdir=$tmpdir")
+        else
+            xtmpdir=$(TMPDIR="$tmpdir"; mktemp '-d')
         fi
 
         wsrep_log_info "Using '$xtmpdir' as xtrabackup temporary directory"
-- 
cgit v1.2.1


From 25d6bbcd5172eb9af0a68be11fb662457a7d572b Mon Sep 17 00:00:00 2001
From: Alexey Bychko <alexey.bychko@mariadb.com>
Date: Tue, 21 Sep 2021 14:44:39 +0700
Subject: MDEV-23506 mariadb-connector-c-devel package from standard RHEL 8
 repo conflicts with MariaDB's packages

added alternative name for MariaDB-devel package to replace
mariadb-connector-c-devel from RHEL 8 distribution.
this patch is for 10.3+ on RHEL/Centos 8
---
 cmake/cpack_rpm.cmake | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake
index 1be5d26d7e0..6abe4e8b223 100644
--- a/cmake/cpack_rpm.cmake
+++ b/cmake/cpack_rpm.cmake
@@ -239,6 +239,7 @@ ELSEIF(RPM MATCHES "(rhel|centos)8")
   ALTERNATIVE_NAME("server" "mariadb-server-utils")
   ALTERNATIVE_NAME("shared" "mariadb-connector-c" ${MARIADB_CONNECTOR_C_VERSION}-1)
   ALTERNATIVE_NAME("shared" "mariadb-connector-c-config" ${MARIADB_CONNECTOR_C_VERSION}-1)
+  ALTERNATIVE_NAME("devel" "mariadb-connector-c-devel" ${MARIADB_CONNECTOR_C_VERSION}-1)
   SETA(CPACK_RPM_client_PACKAGE_PROVIDES "mariadb-galera = 3:%{version}-%{release}")
   SETA(CPACK_RPM_common_PACKAGE_PROVIDES "mariadb-galera-common = 3:%{version}-%{release}")
   SETA(CPACK_RPM_common_PACKAGE_REQUIRES "MariaDB-shared")
-- 
cgit v1.2.1


From a584117c0c6ce58e665dc84d8005d25403d0d22e Mon Sep 17 00:00:00 2001
From: Monty <monty@mariadb.org>
Date: Tue, 21 Sep 2021 15:00:35 +0300
Subject: Fixed max length calculation for embedded server

The old code did not take into account unsigned numbers when calculating
max_lengths of fields.
---
 libmysqld/libmysql.c | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/libmysqld/libmysql.c b/libmysqld/libmysql.c
index 52e6a08d2f8..1ebcae0d8d5 100644
--- a/libmysqld/libmysql.c
+++ b/libmysqld/libmysql.c
@@ -3925,6 +3925,7 @@ static my_bool is_binary_compatible(enum enum_field_types type1,
 
 static my_bool setup_one_fetch_function(MYSQL_BIND *param, MYSQL_FIELD *field)
 {
+  my_bool field_is_unsigned;
   DBUG_ENTER("setup_one_fetch_function");
 
   /* Setup data copy functions for the different supported types */
@@ -4001,6 +4002,7 @@ static my_bool setup_one_fetch_function(MYSQL_BIND *param, MYSQL_FIELD *field)
 
   /* Setup skip_result functions (to calculate max_length) */
   param->skip_result= skip_result_fixed;
+  field_is_unsigned= MY_TEST(field->flags & UNSIGNED_FLAG);
   switch (field->type) {
   case MYSQL_TYPE_NULL: /* for dummy binds */
     param->pack_length= 0;
@@ -4008,23 +4010,23 @@ static my_bool setup_one_fetch_function(MYSQL_BIND *param, MYSQL_FIELD *field)
     break;
   case MYSQL_TYPE_TINY:
     param->pack_length= 1;
-    field->max_length= 4;                     /* as in '-127' */
+    field->max_length= field_is_unsigned ? 3 : 4; /* '255' and '-127' */
     break;
   case MYSQL_TYPE_YEAR:
   case MYSQL_TYPE_SHORT:
     param->pack_length= 2;
-    field->max_length= 6;                     /* as in '-32767' */
+    field->max_length= field_is_unsigned ? 5 : 6; /* 65536 and '-32767' */
     break;
   case MYSQL_TYPE_INT24:
-    field->max_length= 9;  /* as in '16777216' or in '-8388607' */
+    field->max_length=  8;  /* '16777216' or in '-8388607' */
     param->pack_length= 4;
     break;
   case MYSQL_TYPE_LONG:
-    field->max_length= 11;                    /* '-2147483647' */
+    field->max_length= field_is_unsigned ? 10 : 11; /* '4294967295' and '-2147483647' */
     param->pack_length= 4;
     break;
   case MYSQL_TYPE_LONGLONG:
-    field->max_length= 21;                    /* '18446744073709551616' */
+    field->max_length= 20; /* '18446744073709551616' or -9223372036854775808 */
     param->pack_length= 8;
     break;
   case MYSQL_TYPE_FLOAT:
-- 
cgit v1.2.1


From d6bddfca229c702fba94b5ed5a864d7df130ca3a Mon Sep 17 00:00:00 2001
From: Monty <monty@mariadb.org>
Date: Tue, 21 Sep 2021 14:03:24 +0300
Subject: Updated main.alias test to fix max_length

The test will work after libmariadb has been updated to return correct
max_length for prepared statements
---
 mysql-test/main/alias.result | 12 ++++++------
 mysql-test/main/alias.test   |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/mysql-test/main/alias.result b/mysql-test/main/alias.result
index 0d14607f613..266991afbec 100644
--- a/mysql-test/main/alias.result
+++ b/mysql-test/main/alias.result
@@ -223,16 +223,16 @@ disconnect c1;
 #
 create or replace table t1 (a int);
 create or replace table t2 (b int);
-insert into t1 values(1<<30),(1<<29);
+insert into t1 values(111111111),(-2147483648);
 insert into t2 values(1),(2);
 select t1.a as a1 from t1 as t1,t2 order by t2.b,t1.a;
 Catalog	Database	Table	Table_alias	Column	Column_alias	Type	Length	Max length	Is_null	Flags	Decimals	Charsetnr
-def	test	t1	t1	a	a1	3	11	10	Y	32768	0	63
+def	test	t1	t1	a	a1	3	11	11	Y	32768	0	63
 a1
-536870912
-1073741824
-536870912
-1073741824
+-2147483648
+111111111
+-2147483648
+111111111
 drop table t1,t2;
 #
 # End of 10.4 tests
diff --git a/mysql-test/main/alias.test b/mysql-test/main/alias.test
index 2408509ba10..881cc34d6c6 100644
--- a/mysql-test/main/alias.test
+++ b/mysql-test/main/alias.test
@@ -231,7 +231,7 @@ disconnect c1;
 --echo #
 create or replace table t1 (a int);
 create or replace table t2 (b int);
-insert into t1 values(1<<30),(1<<29);
+insert into t1 values(111111111),(-2147483648);
 insert into t2 values(1),(2);
 --enable_metadata
 select t1.a as a1 from t1 as t1,t2 order by t2.b,t1.a;
-- 
cgit v1.2.1


From f2021b5ee4ef8c53bedbab309bad2feda4c73f46 Mon Sep 17 00:00:00 2001
From: Monty <monty@mariadb.org>
Date: Tue, 21 Sep 2021 15:03:04 +0300
Subject: Fixed random failure of main.truncate_notembedded

The test depends on how the server allocates memory and may fail randomly.
Fixed by accepting that TRUNCATE may work in some cases (happened to me)
---
 mysql-test/main/truncate_notembedded.result | 1 -
 mysql-test/main/truncate_notembedded.test   | 5 ++++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/mysql-test/main/truncate_notembedded.result b/mysql-test/main/truncate_notembedded.result
index 18a01c684e7..6280a995149 100644
--- a/mysql-test/main/truncate_notembedded.result
+++ b/mysql-test/main/truncate_notembedded.result
@@ -13,7 +13,6 @@ a
 UNLOCK TABLES;
 connection con1;
 TRUNCATE TABLE t1;
-ERROR HY000: The MariaDB server is running with the --max-thread-mem-used=65536 option so it cannot execute this statement
 disconnect con1;
 connection default;
 DROP TABLE t1;
diff --git a/mysql-test/main/truncate_notembedded.test b/mysql-test/main/truncate_notembedded.test
index 7e186c8f7d5..029701b1e26 100644
--- a/mysql-test/main/truncate_notembedded.test
+++ b/mysql-test/main/truncate_notembedded.test
@@ -19,7 +19,10 @@ UNLOCK TABLES;
 
 --connection con1
 --reap
---error ER_OPTION_PREVENTS_STATEMENT
+# This may work or fail as different servers uses different amount of
+# memory and the statement may work or not. What is important is that we
+# don't get a crash here!
+--error 0,ER_OPTION_PREVENTS_STATEMENT
 TRUNCATE TABLE t1;
 
 --disconnect con1
-- 
cgit v1.2.1


From b4074069b2a89a14d9406f4c10c92811cdd78c79 Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Tue, 21 Sep 2021 19:14:07 +0200
Subject: MDEV-26657 : Initialize some fields in create_background_thd()

Avoid reading uninitialized memory by  thd_get_error_context_description().
Note, that THD::real_id can't be initialized at this stage, so it will be zeroed.
---
 sql/sql_class.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/sql/sql_class.cc b/sql/sql_class.cc
index 0d9a8439ccb..883e9c688ff 100644
--- a/sql/sql_class.cc
+++ b/sql/sql_class.cc
@@ -4922,6 +4922,9 @@ MYSQL_THD create_background_thd()
   thd->set_command(COM_DAEMON);
   thd->system_thread= SYSTEM_THREAD_GENERIC;
   thd->security_ctx->host_or_ip= "";
+  thd->real_id= 0;
+  thd->thread_id= 0;
+  thd->query_id= 0;
   return thd;
 }
 
-- 
cgit v1.2.1


From de2b5795f45a4e35a75d640bdc3b9f3f0e919bf6 Mon Sep 17 00:00:00 2001
From: Ian Gilfillan <github@greenman.co.za>
Date: Mon, 20 Sep 2021 14:23:58 +0200
Subject: Remove mention of Freenode

---
 CONTRIBUTING.md | 5 ++---
 README.md       | 3 +--
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 64af450d29f..37899f37958 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -8,8 +8,7 @@ MariaDB Server has a vibrant community contributing in a wide range of areas. Th
 -   [maria-developers mailing list](http://launchpad.net/~maria-developers)
 -   [maria-discuss mailing list](http://launchpad.net/~maria-discuss)
 -   [maria-docs mailing list](http://launchpad.net/~maria-docs)
--   ircs://chat.freenode.net/maria ([see the IRC page on the Knowledge Base](https://mariadb.com/kb/en/meta/irc-chat-servers-and-zulip-instance/)  for help with IRC).
--   The MariaDB Foundation and MariaDB Corporation have a presence on Reddit, Twitter, Facebook and Google Plus. See the  [social media page](https://mariadb.com/kb/en/mariadb/social-media/).
+-   The MariaDB Foundation and MariaDB Corporation have a presence on Reddit, Twitter and Facebook. See the  [social media page](https://mariadb.com/kb/en/mariadb/social-media/).
 
 ### Help document MariaDB
 ----
@@ -36,7 +35,7 @@ You’re very welcome to support MariaDB Server as an individual, or talk your c
 
 ### Live QA for beginner contributors
 ----
-MariaDB has a dedicated time each week when we answer new contributor questions live on Zulip and IRC.
+MariaDB has a dedicated time each week when we answer new contributor questions live on Zulip.
 From 8:00 to 10:00 UTC on Mondays, and 10:00 to 12:00 UTC on Thursdays, anyone can ask any questions they’d like,
 and a live developer will be available to assist.
 New contributors can ask questions any time, but we will provide immediate feedback during that interval.
diff --git a/README.md b/README.md
index f17a882d66e..58dbf105fb9 100644
--- a/README.md
+++ b/README.md
@@ -39,11 +39,10 @@ Help
 More help is available from the Maria Discuss mailing list
 https://launchpad.net/~maria-discuss, MariaDB's Zulip
 instance, https://mariadb.zulipchat.com/ 
-and the #maria IRC channel on Freenode.
 
 Live QA for beginner contributors
 ----
-MariaDB has a dedicated time each week when we answer new contributor questions live on Zulip and IRC.
+MariaDB has a dedicated time each week when we answer new contributor questions live on Zulip.
 From 8:00 to 10:00 UTC on Mondays, and 10:00 to 12:00 UTC on Thursdays,
 anyone can ask any questions they’d like, and a live developer will be available to assist.
 
-- 
cgit v1.2.1


From edabb12509fa5a8c94348d0e803b5925c454cc86 Mon Sep 17 00:00:00 2001
From: Anel Husakovic <anel@mariadb.org>
Date: Mon, 13 Sep 2021 17:43:23 +0200
Subject: MDEV-21286: bison warnings on ubuntu 20.04 on deprecated directive in
 sql_yacc.yy

Results with
```
%define api.pure                                    /* We have threads */
```
in `<build-dir>/sql/[yy_mariadb|yy_oracle].yy` files.

Reviewed by: wlad@mariadb.com
---
 sql/CMakeLists.txt     | 5 +++--
 sql/gen_yy_files.cmake | 3 +++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt
index 5719d098383..642a931c0b3 100644
--- a/sql/CMakeLists.txt
+++ b/sql/CMakeLists.txt
@@ -65,6 +65,8 @@ ADD_CUSTOM_COMMAND(
   DEPENDS gen_lex_token
 )
 
+FIND_PACKAGE(BISON 2.0)
+
 ADD_CUSTOM_COMMAND(
   OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/yy_mariadb.yy
          ${CMAKE_CURRENT_BINARY_DIR}/yy_oracle.yy
@@ -72,6 +74,7 @@ ADD_CUSTOM_COMMAND(
    "-DOUT1=${CMAKE_CURRENT_BINARY_DIR}/yy_oracle.yy"
    "-DOUT2=${CMAKE_CURRENT_BINARY_DIR}/yy_mariadb.yy"
    "-DIN=${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc.yy"
+    "-DBISON_VERSION=${BISON_VERSION}"
    -P ${CMAKE_CURRENT_SOURCE_DIR}/gen_yy_files.cmake
   COMMENT "Building yy_mariadb.yy and yy_oracle.yy from sql_yacc.yy"
   DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/sql_yacc.yy
@@ -321,8 +324,6 @@ IF(WITH_MYSQLD_LDFLAGS)
 ENDIF()
 
 
-FIND_PACKAGE(BISON 2.0)
-
 
 # Handle out-of-source build from source package with possibly broken 
 # bison. Copy bison output to from source to build directory, if not already 
diff --git a/sql/gen_yy_files.cmake b/sql/gen_yy_files.cmake
index da63c72c37c..3ceb60a95de 100644
--- a/sql/gen_yy_files.cmake
+++ b/sql/gen_yy_files.cmake
@@ -5,6 +5,9 @@ file(READ "${IN}" data)
 file(WRITE "${OUT1}" "")
 file(WRITE "${OUT2}" "")
 set(where 0)
+if(NOT(BISON_VERSION VERSION_LESS "3.0.0"))
+  string(REPLACE "\n%pure-parser" "\n%define api.pure" data "${data}")
+endif()
 string(REGEX REPLACE "/\\* sql_yacc\\.yy \\*/" "/* DON'T EDIT THIS FILE. IT'S GENERATED.  EDIT sql_yacc.yy INSTEAD */" data "${data}")
 while(NOT data STREQUAL "")
   string(REGEX MATCH "^(%[ie][^\n]*\n)|((%[^ie\n]|[^%\n])[^\n]*\n)+|\n+" line "${data}")
-- 
cgit v1.2.1


From 7d360060cb6ea2558820b343ef77d59584b0b805 Mon Sep 17 00:00:00 2001
From: Alexey Bychko <alexey.bychko@mariadb.com>
Date: Tue, 21 Sep 2021 22:40:47 +0700
Subject: MDEV-24629 mariadb-connector-c-config conflicts with MariaDB's
 MariaDB-common-10.5.8-1.fc32.x86_64.rpm

this fix is adding alternative name to MariaDB-common on Fedora.
the fix is placed outside of IF/ELSEIF blocks to do not overwrite
existing one for Fedora.
---
 cmake/cpack_rpm.cmake | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake
index 583bb81dafd..630936b746c 100644
--- a/cmake/cpack_rpm.cmake
+++ b/cmake/cpack_rpm.cmake
@@ -252,6 +252,12 @@ ELSEIF(RPM MATCHES "sles")
     "mariadb-server = %{version}-%{release}"
   )
 ENDIF()
+
+# MDEV-24629, we need it outside of ELSIFs
+IF(RPM MATCHES "fedora3[234]")
+  ALTERNATIVE_NAME("common" "mariadb-connector-c-config" ${MARIADB_CONNECTOR_C_VERSION}-1)
+ENDIF()
+
 SET(PYTHON_SHEBANG "/usr/bin/python3" CACHE STRING "python shebang")
 
 # If we want to build build MariaDB-shared-compat,
-- 
cgit v1.2.1


From 3d30458695ed9c8acf8d33b447e4b18844f2988b Mon Sep 17 00:00:00 2001
From: Vladislav Vaintroub <wlad@mariadb.com>
Date: Tue, 21 Sep 2021 20:45:42 +0200
Subject: MDEV-26521 Remove mdev-504.test

---
 mysql-test/main/mdev-504.result | 24 -------------
 mysql-test/main/mdev-504.test   | 80 -----------------------------------------
 2 files changed, 104 deletions(-)
 delete mode 100644 mysql-test/main/mdev-504.result
 delete mode 100644 mysql-test/main/mdev-504.test

diff --git a/mysql-test/main/mdev-504.result b/mysql-test/main/mdev-504.result
deleted file mode 100644
index 9b8b6795e0f..00000000000
--- a/mysql-test/main/mdev-504.result
+++ /dev/null
@@ -1,24 +0,0 @@
-SET GLOBAL net_write_timeout = 900;
-CREATE TABLE A (
-pk INTEGER AUTO_INCREMENT PRIMARY KEY,
-fdate DATE
-) ENGINE=MyISAM;
-CREATE PROCEDURE p_analyze()
-BEGIN
-DECLARE attempts INTEGER DEFAULT 100;
-wl_loop: WHILE attempts > 0 DO
-ANALYZE TABLE A;
-SET attempts = attempts - 1;
-END WHILE wl_loop;
-END |
-CREATE FUNCTION rnd3() RETURNS INT
-BEGIN
-RETURN ROUND(3 * RAND() + 0.5);
-END |
-SET GLOBAL use_stat_tables = PREFERABLY;
-connection default;
-DROP TABLE A;
-DROP PROCEDURE p_analyze;
-DROP FUNCTION rnd3;
-SET GLOBAL use_stat_tables = DEFAULT;
-SET GLOBAL net_write_timeout = DEFAULT;
diff --git a/mysql-test/main/mdev-504.test b/mysql-test/main/mdev-504.test
deleted file mode 100644
index 551c21c37d0..00000000000
--- a/mysql-test/main/mdev-504.test
+++ /dev/null
@@ -1,80 +0,0 @@
---source include/not_valgrind.inc
---source include/no_protocol.inc
-
-SET GLOBAL net_write_timeout = 900; 
-
-CREATE TABLE A (
-  pk INTEGER AUTO_INCREMENT PRIMARY KEY,
-  fdate DATE
-) ENGINE=MyISAM;
-
---delimiter |
-
-CREATE PROCEDURE p_analyze()
-BEGIN
-  DECLARE attempts INTEGER DEFAULT 100;
-  wl_loop: WHILE attempts > 0 DO
-    ANALYZE TABLE A;
-    SET attempts = attempts - 1;
-  END WHILE wl_loop;
-END |
-
-CREATE FUNCTION rnd3() RETURNS INT
-BEGIN
-  RETURN ROUND(3 * RAND() + 0.5);
-END |
-
---delimiter ;
-
-SET GLOBAL use_stat_tables = PREFERABLY;  
-
---let $trial = 100
-
---disable_query_log
---disable_result_log
---disable_warnings
-while ($trial)
-{
-
-  --connect (con1,localhost,root,,)
-  --send CALL p_analyze()
-
-  --connect (con2,localhost,root,,)
-  --send CALL p_analyze()
-
-  --let $run = 100
-
-  while ($run)
-  {
-    --connect (con3,localhost,root,,)
-  
-    let $query = `SELECT CASE rnd3() 
-      WHEN 1 THEN 'INSERT INTO A (pk) VALUES (NULL)'
-      WHEN 2 THEN 'DELETE FROM A LIMIT 1'
-      ELSE 'UPDATE IGNORE A SET fdate = 2 LIMIT 1' END`;
-    --eval $query
-    --disconnect con3
-    --dec $run
-  }
-
-  --connection con2
-  --reap
-  --disconnect con2
-  --connection con1
-  --reap
-  --disconnect con1
-
-  --dec $trial
-}
-
---enable_query_log
---enable_result_log
---enable_warnings
-
-# Cleanup
---connection default
-DROP TABLE A;
-DROP PROCEDURE p_analyze;
-DROP FUNCTION rnd3;
-SET GLOBAL use_stat_tables = DEFAULT;
-SET GLOBAL net_write_timeout = DEFAULT;
-- 
cgit v1.2.1


From ac1c6738f973498e2d136d5e419b961c151aa318 Mon Sep 17 00:00:00 2001
From: Daniel Ye <35718816+GreaTdANie1@users.noreply.github.com>
Date: Wed, 22 Sep 2021 17:54:22 +0800
Subject: MDEV-26545 Spider does not correctly handle UDF and stored function
 in where conds

- Handle stored function conditions correctly, with the same logic as with UDFs.
- When running queries on Spider SE, by default, we do not push down WHERE conditions containing usage of UDFs/stored functions to remote data nodes, unless the user demands (by setting spider_use_pushdown_udf).
---
 .../spider/mysql-test/spider/r/udf_pushdown.result | 222 +++++++++++++++++++++
 .../spider/mysql-test/spider/t/udf_pushdown.inc    |  48 +++++
 .../spider/mysql-test/spider/t/udf_pushdown.test   | 141 +++++++++++++
 storage/spider/spd_db_mysql.cc                     |   6 +
 storage/spider/spd_param.cc                        |   2 +-
 5 files changed, 418 insertions(+), 1 deletion(-)
 create mode 100644 storage/spider/mysql-test/spider/r/udf_pushdown.result
 create mode 100644 storage/spider/mysql-test/spider/t/udf_pushdown.inc
 create mode 100644 storage/spider/mysql-test/spider/t/udf_pushdown.test

diff --git a/storage/spider/mysql-test/spider/r/udf_pushdown.result b/storage/spider/mysql-test/spider/r/udf_pushdown.result
new file mode 100644
index 00000000000..e9182255bcd
--- /dev/null
+++ b/storage/spider/mysql-test/spider/r/udf_pushdown.result
@@ -0,0 +1,222 @@
+for master_1
+for child2
+child2_1
+child2_2
+child2_3
+for child3
+child3_1
+child3_2
+child3_3
+#
+# MDEV-26545 Spider does not correctly handle UDF and stored function in where conds
+#
+
+##### enable general_log #####
+connection child2_1;
+SET @general_log_backup = @@global.general_log;
+SET @log_output_backup = @@global.log_output;
+SET @@global.general_log = 1;
+SET @@global.log_output = "TABLE";
+TRUNCATE TABLE mysql.general_log;
+
+##### create databases #####
+connection master_1;
+CREATE DATABASE auto_test_local;
+USE auto_test_local;
+connection child2_1;
+CREATE DATABASE auto_test_remote;
+USE auto_test_remote;
+
+##### create tables #####
+connection child2_1;
+CHILD_CREATE_TABLE
+connection master_1;
+MASTER_CREATE_TABLE
+CREATE TABLE ta_l (
+id INT NOT NULL,
+a INT,
+PRIMARY KEY(id)
+) MASTER_1_ENGINE MASTER_1_CHARSET MASTER_1_COMMENT_2_1
+INSERT INTO ta_l VALUES
+(1, 11),
+(2, 22),
+(3, 33),
+(4, 44),
+(5, 55);
+
+##### create functions #####
+connection master_1;
+CREATE FUNCTION `plusone`( param INT ) RETURNS INT
+BEGIN
+RETURN param + 1;
+END //
+connection child2_1;
+CREATE FUNCTION `plusone`( param INT ) RETURNS INT
+BEGIN
+RETURN param + 1;
+END //
+
+########## spider_use_pushdown_udf=0 ##########
+connection master_1;
+SET @@spider_use_pushdown_udf = 0;
+
+##### test SELECTs #####
+connection master_1;
+SELECT * FROM ta_l WHERE id = plusone(1);
+id	a
+2	22
+SELECT * FROM ta_l WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32);
+id	a
+3	33
+connection child2_1;
+SELECT argument FROM mysql.general_log WHERE argument LIKE "%select%" AND argument NOT LIKE "%argument%";
+argument
+select `id`,`a` from `auto_test_remote`.`ta_r`
+select `id`,`a` from `auto_test_remote`.`ta_r`
+
+##### test UPDATEs #####
+connection master_1;
+UPDATE ta_l SET a = plusone(221) WHERE id = plusone(1);
+SELECT * FROM ta_l;
+id	a
+1	11
+2	222
+3	33
+4	44
+5	55
+UPDATE ta_l SET a = plusone(332) WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32);
+SELECT * FROM ta_l;
+id	a
+1	11
+2	222
+3	333
+4	44
+5	55
+connection child2_1;
+SELECT argument FROM mysql.general_log WHERE argument LIKE "%update%" AND argument NOT LIKE "%argument%";
+argument
+select `id`,`a` from `auto_test_remote`.`ta_r` for update
+update `auto_test_remote`.`ta_r` set `a` = 222 where `id` = 2 limit 1
+select `id`,`a` from `auto_test_remote`.`ta_r` for update
+update `auto_test_remote`.`ta_r` set `a` = 333 where `id` = 3 and `a` = 33 limit 1
+
+##### test DELETEs #####
+connection master_1;
+DELETE FROM ta_l WHERE id = plusone(1);
+SELECT * FROM ta_l;
+id	a
+1	11
+3	333
+4	44
+5	55
+DELETE FROM ta_l WHERE id IN (plusone(1), plusone(2), plusone(3)) AND a = plusone(43);
+SELECT * FROM ta_l;
+id	a
+1	11
+3	333
+5	55
+connection child2_1;
+SELECT argument FROM mysql.general_log WHERE (argument LIKE "%delete%" OR argument LIKE "%update%") AND argument NOT LIKE "%argument%";
+argument
+select `id` from `auto_test_remote`.`ta_r` for update
+delete from `auto_test_remote`.`ta_r` where `id` = 2 limit 1
+select `id`,`a` from `auto_test_remote`.`ta_r` for update
+delete from `auto_test_remote`.`ta_r` where `id` = 4 and `a` = 44 limit 1
+
+##### reset records #####
+connection master_1;
+TRUNCATE TABLE ta_l;
+INSERT INTO ta_l VALUES
+(1, 11),
+(2, 22),
+(3, 33),
+(4, 44),
+(5, 55);
+
+########## spider_use_pushdown_udf=1 ##########
+connection master_1;
+SET @@spider_use_pushdown_udf = 1;
+
+##### test SELECTs #####
+connection master_1;
+SELECT * FROM ta_l WHERE id = plusone(1);
+id	a
+2	22
+SELECT * FROM ta_l WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32);
+id	a
+3	33
+connection child2_1;
+SELECT argument FROM mysql.general_log WHERE argument LIKE "%select%" AND argument NOT LIKE "%argument%";
+argument
+select `id`,`a` from `auto_test_remote`.`ta_r` where (`id` = (`plusone`(1)))
+select `id`,`a` from `auto_test_remote`.`ta_r` where ((`id` in( (`plusone`(1)) , (`plusone`(2)))) and (`a` = (`plusone`(32))))
+
+##### test UPDATEs #####
+connection master_1;
+UPDATE ta_l SET a = plusone(221) WHERE id = plusone(1);
+SELECT * FROM ta_l;
+id	a
+1	11
+2	222
+3	33
+4	44
+5	55
+UPDATE ta_l SET a = plusone(332) WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32);
+SELECT * FROM ta_l;
+id	a
+1	11
+2	222
+3	333
+4	44
+5	55
+connection child2_1;
+SELECT argument FROM mysql.general_log WHERE argument LIKE "%update%" AND argument NOT LIKE "%argument%";
+argument
+select `id`,`a` from `auto_test_remote`.`ta_r` where (`id` = (`plusone`(1))) for update
+update `auto_test_remote`.`ta_r` set `a` = 222 where `id` = 2 limit 1
+select `id`,`a` from `auto_test_remote`.`ta_r` where ((`id` in( (`plusone`(1)) , (`plusone`(2)))) and (`a` = (`plusone`(32)))) for update
+update `auto_test_remote`.`ta_r` set `a` = 333 where `id` = 3 and `a` = 33 limit 1
+
+##### test DELETEs #####
+connection master_1;
+DELETE FROM ta_l WHERE id = plusone(1);
+SELECT * FROM ta_l;
+id	a
+1	11
+3	333
+4	44
+5	55
+DELETE FROM ta_l WHERE id IN (plusone(1), plusone(2), plusone(3)) AND a = plusone(43);
+SELECT * FROM ta_l;
+id	a
+1	11
+3	333
+5	55
+connection child2_1;
+SELECT argument FROM mysql.general_log WHERE (argument LIKE "%delete%" OR argument LIKE "%update%") AND argument NOT LIKE "%argument%";
+argument
+select `id` from `auto_test_remote`.`ta_r` where (`id` = (`plusone`(1))) for update
+delete from `auto_test_remote`.`ta_r` where `id` = 2 limit 1
+select `id`,`a` from `auto_test_remote`.`ta_r` where ((`id` in( (`plusone`(1)) , (`plusone`(2)) , (`plusone`(3)))) and (`a` = (`plusone`(43)))) for update
+delete from `auto_test_remote`.`ta_r` where `id` = 4 and `a` = 44 limit 1
+
+deinit
+connection master_1;
+DROP FUNCTION `plusone`;
+DROP DATABASE IF EXISTS auto_test_local;
+connection child2_1;
+SET @@global.general_log = @general_log_backup;
+SET @@global.log_output = @log_output_backup;
+DROP FUNCTION `plusone`;
+DROP DATABASE IF EXISTS auto_test_remote;
+for master_1
+for child2
+child2_1
+child2_2
+child2_3
+for child3
+child3_1
+child3_2
+child3_3
+
+end of test
diff --git a/storage/spider/mysql-test/spider/t/udf_pushdown.inc b/storage/spider/mysql-test/spider/t/udf_pushdown.inc
new file mode 100644
index 00000000000..160e8af21b2
--- /dev/null
+++ b/storage/spider/mysql-test/spider/t/udf_pushdown.inc
@@ -0,0 +1,48 @@
+--echo
+--echo ##### test SELECTs #####
+--connection master_1
+SELECT * FROM ta_l WHERE id = plusone(1);
+SELECT * FROM ta_l WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32);
+
+if ($USE_CHILD_GROUP2)
+{
+  --connection child2_1
+  SELECT argument FROM mysql.general_log WHERE argument LIKE "%select%" AND argument NOT LIKE "%argument%";
+  --disable_query_log
+  TRUNCATE TABLE mysql.general_log;
+  --enable_query_log
+}
+
+--echo
+--echo ##### test UPDATEs #####
+--connection master_1
+UPDATE ta_l SET a = plusone(221) WHERE id = plusone(1);
+SELECT * FROM ta_l;
+UPDATE ta_l SET a = plusone(332) WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32);
+SELECT * FROM ta_l;
+
+if ($USE_CHILD_GROUP2)
+{
+  --connection child2_1
+  SELECT argument FROM mysql.general_log WHERE argument LIKE "%update%" AND argument NOT LIKE "%argument%";
+  --disable_query_log
+  TRUNCATE TABLE mysql.general_log;
+  --enable_query_log
+}
+
+--echo
+--echo ##### test DELETEs #####
+--connection master_1
+DELETE FROM ta_l WHERE id = plusone(1);
+SELECT * FROM ta_l;
+DELETE FROM ta_l WHERE id IN (plusone(1), plusone(2), plusone(3)) AND a = plusone(43);
+SELECT * FROM ta_l;
+
+if ($USE_CHILD_GROUP2)
+{
+  --connection child2_1
+  SELECT argument FROM mysql.general_log WHERE (argument LIKE "%delete%" OR argument LIKE "%update%") AND argument NOT LIKE "%argument%";
+  --disable_query_log
+  TRUNCATE TABLE mysql.general_log;
+  --enable_query_log
+}
diff --git a/storage/spider/mysql-test/spider/t/udf_pushdown.test b/storage/spider/mysql-test/spider/t/udf_pushdown.test
new file mode 100644
index 00000000000..2eadbbbb40b
--- /dev/null
+++ b/storage/spider/mysql-test/spider/t/udf_pushdown.test
@@ -0,0 +1,141 @@
+--disable_warnings
+--disable_query_log
+--disable_result_log
+--source test_init.inc
+--enable_result_log
+--enable_query_log
+
+--echo #
+--echo # MDEV-26545 Spider does not correctly handle UDF and stored function in where conds
+--echo #
+
+let $CHILD_CREATE_TABLE=
+  CREATE TABLE ta_r (
+    id INT NOT NULL,
+    a INT,
+    PRIMARY KEY(id)
+  ) $CHILD2_1_ENGINE $CHILD2_1_CHARSET;
+
+let $MASTER_CREATE_TABLE_OUTPUT=
+  CREATE TABLE ta_l (
+    id INT NOT NULL,
+    a INT,
+    PRIMARY KEY(id)
+  ) MASTER_1_ENGINE MASTER_1_CHARSET MASTER_1_COMMENT_2_1;
+
+let $MASTER_CREATE_TABLE=
+  CREATE TABLE ta_l (
+    id INT NOT NULL,
+    a INT,
+    PRIMARY KEY(id)
+  ) $MASTER_1_ENGINE $MASTER_1_CHARSET $MASTER_1_COMMENT_2_1;
+
+--echo
+--echo ##### enable general_log #####
+--connection child2_1
+SET @general_log_backup = @@global.general_log;
+SET @log_output_backup = @@global.log_output;
+SET @@global.general_log = 1;
+SET @@global.log_output = "TABLE";
+TRUNCATE TABLE mysql.general_log;
+
+--echo
+--echo ##### create databases #####
+--connection master_1
+CREATE DATABASE auto_test_local;
+USE auto_test_local;
+if ($USE_CHILD_GROUP2)
+{
+  --connection child2_1
+  CREATE DATABASE auto_test_remote;
+  USE auto_test_remote;
+}
+
+--echo
+--echo ##### create tables #####
+if ($USE_CHILD_GROUP2)
+{
+  --connection child2_1
+  --disable_query_log
+  echo CHILD_CREATE_TABLE;
+  eval $CHILD_CREATE_TABLE;
+  --enable_query_log
+}
+
+--connection master_1
+--disable_query_log
+echo MASTER_CREATE_TABLE;
+echo $MASTER_CREATE_TABLE_OUTPUT;
+eval $MASTER_CREATE_TABLE;
+--enable_query_log
+
+INSERT INTO ta_l VALUES
+    (1, 11),
+    (2, 22),
+    (3, 33),
+    (4, 44),
+    (5, 55);
+
+--echo
+--echo ##### create functions #####
+--connection master_1
+DELIMITER //;
+CREATE FUNCTION `plusone`( param INT ) RETURNS INT
+BEGIN
+    RETURN param + 1;
+END //
+DELIMITER ;//
+
+--connection child2_1
+DELIMITER //;
+CREATE FUNCTION `plusone`( param INT ) RETURNS INT
+BEGIN
+    RETURN param + 1;
+END //
+DELIMITER ;//
+
+--echo
+--echo ########## spider_use_pushdown_udf=0 ##########
+--connection master_1
+SET @@spider_use_pushdown_udf = 0;
+--source udf_pushdown.inc
+
+--echo
+--echo ##### reset records #####
+--connection master_1
+TRUNCATE TABLE ta_l;
+INSERT INTO ta_l VALUES
+    (1, 11),
+    (2, 22),
+    (3, 33),
+    (4, 44),
+    (5, 55);
+
+--echo
+--echo ########## spider_use_pushdown_udf=1 ##########
+--connection master_1
+SET @@spider_use_pushdown_udf = 1;
+--source udf_pushdown.inc
+
+--echo
+--echo deinit
+--disable_warnings
+--connection master_1
+DROP FUNCTION `plusone`;
+DROP DATABASE IF EXISTS auto_test_local;
+if ($USE_CHILD_GROUP2)
+{
+  --connection child2_1
+  SET @@global.general_log = @general_log_backup;
+  SET @@global.log_output = @log_output_backup;
+  DROP FUNCTION `plusone`;
+  DROP DATABASE IF EXISTS auto_test_remote;
+}
+--disable_query_log
+--disable_result_log
+--source test_deinit.inc
+--enable_result_log
+--enable_query_log
+--enable_warnings
+--echo
+--echo end of test
diff --git a/storage/spider/spd_db_mysql.cc b/storage/spider/spd_db_mysql.cc
index 1a24d80a95f..4db1232aebf 100644
--- a/storage/spider/spd_db_mysql.cc
+++ b/storage/spider/spd_db_mysql.cc
@@ -4085,10 +4085,16 @@ int spider_db_mysql_util::open_item_func(
         separete_str_length = SPIDER_SQL_AND_LEN;
       }
       break;
+    case Item_func::FUNC_SP:
     case Item_func::UDF_FUNC:
       use_pushdown_udf = spider_param_use_pushdown_udf(spider->trx->thd,
         spider->share->use_pushdown_udf);
       if (!use_pushdown_udf)
+        /*
+          This is the default behavior because the remote nodes may deal with
+          the function in an unexpected way (e.g. not having the same
+          definition). Users can turn it on if they know what they are doing.
+        */
         DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM);
       if (str)
       {
diff --git a/storage/spider/spd_param.cc b/storage/spider/spd_param.cc
index c4aaef3a404..468e5844189 100644
--- a/storage/spider/spd_param.cc
+++ b/storage/spider/spd_param.cc
@@ -1931,7 +1931,7 @@ static MYSQL_THDVAR_INT(
   "Remote server transmission existence when UDF is used at condition and \"engine_condition_pushdown=1\"", /* comment */
   NULL, /* check */
   NULL, /* update */
-  -1, /* def */
+  0, /* def */
   -1, /* min */
   1, /* max */
   0 /* blk */
-- 
cgit v1.2.1


From 9fc1ef932f0b7499724cfcf76bd0f298f135018f Mon Sep 17 00:00:00 2001
From: Daniel Ye <35718816+GreaTdANie1@users.noreply.github.com>
Date: Wed, 22 Sep 2021 17:55:05 +0800
Subject: MDEV-26545 Spider does not correctly handle UDF and stored function
 in where conds

- Handle stored function conditions correctly, with the same logic as with UDFs.
- When running queries on Spider SE, by default, we do not push down WHERE conditions containing usage of UDFs/stored functions to remote data nodes, unless the user demands (by setting spider_use_pushdown_udf).
- Disable direct update/delete when a udf condition is skipped.
---
 sql/sql_delete.cc                                  |  10 +-
 sql/sql_update.cc                                  |  11 +-
 .../spider/mysql-test/spider/r/udf_pushdown.result | 218 +++++++++++++++++++++
 .../spider/mysql-test/spider/t/udf_pushdown.inc    |  48 +++++
 .../spider/mysql-test/spider/t/udf_pushdown.test   | 141 +++++++++++++
 storage/spider/spd_db_mysql.cc                     |   6 +
 storage/spider/spd_param.cc                        |   2 +-
 7 files changed, 429 insertions(+), 7 deletions(-)
 create mode 100644 storage/spider/mysql-test/spider/r/udf_pushdown.result
 create mode 100644 storage/spider/mysql-test/spider/t/udf_pushdown.inc
 create mode 100644 storage/spider/mysql-test/spider/t/udf_pushdown.test

diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc
index 084b5a76b84..f6c05da9445 100644
--- a/sql/sql_delete.cc
+++ b/sql/sql_delete.cc
@@ -583,14 +583,18 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds,
     if (!table->check_virtual_columns_marked_for_read())
     {
       DBUG_PRINT("info", ("Trying direct delete"));
-      if (select && select->cond &&
-          (select->cond->used_tables() == table->map))
+      bool use_direct_delete= !select || !select->cond;
+      if (!use_direct_delete &&
+          (select->cond->used_tables() & ~RAND_TABLE_BIT) == table->map)
       {
         DBUG_ASSERT(!table->file->pushed_cond);
         if (!table->file->cond_push(select->cond))
+        {
+          use_direct_delete= TRUE;
           table->file->pushed_cond= select->cond;
+        }
       }
-      if (!table->file->direct_delete_rows_init())
+      if (use_direct_delete && !table->file->direct_delete_rows_init())
       {
         /* Direct deleting is supported */
         DBUG_PRINT("info", ("Using direct delete"));
diff --git a/sql/sql_update.cc b/sql/sql_update.cc
index e808d7f5de0..6065d03402f 100644
--- a/sql/sql_update.cc
+++ b/sql/sql_update.cc
@@ -651,15 +651,20 @@ int mysql_update(THD *thd,
       !table->check_virtual_columns_marked_for_write())
   {
     DBUG_PRINT("info", ("Trying direct update"));
-    if (select && select->cond &&
-        (select->cond->used_tables() == table->map))
+    bool use_direct_update= !select || !select->cond;
+    if (!use_direct_update &&
+        (select->cond->used_tables() & ~RAND_TABLE_BIT) == table->map)
     {
       DBUG_ASSERT(!table->file->pushed_cond);
       if (!table->file->cond_push(select->cond))
+      {
+        use_direct_update= TRUE;
         table->file->pushed_cond= select->cond;
+      }
     }
 
-    if (!table->file->info_push(INFO_KIND_UPDATE_FIELDS, &fields) &&
+    if (use_direct_update &&
+        !table->file->info_push(INFO_KIND_UPDATE_FIELDS, &fields) &&
         !table->file->info_push(INFO_KIND_UPDATE_VALUES, &values) &&
         !table->file->direct_update_rows_init(&fields))
     {
diff --git a/storage/spider/mysql-test/spider/r/udf_pushdown.result b/storage/spider/mysql-test/spider/r/udf_pushdown.result
new file mode 100644
index 00000000000..4ca734165e7
--- /dev/null
+++ b/storage/spider/mysql-test/spider/r/udf_pushdown.result
@@ -0,0 +1,218 @@
+for master_1
+for child2
+child2_1
+child2_2
+child2_3
+for child3
+child3_1
+child3_2
+child3_3
+#
+# MDEV-26545 Spider does not correctly handle UDF and stored function in where conds
+#
+
+##### enable general_log #####
+connection child2_1;
+SET @general_log_backup = @@global.general_log;
+SET @log_output_backup = @@global.log_output;
+SET @@global.general_log = 1;
+SET @@global.log_output = "TABLE";
+TRUNCATE TABLE mysql.general_log;
+
+##### create databases #####
+connection master_1;
+CREATE DATABASE auto_test_local;
+USE auto_test_local;
+connection child2_1;
+CREATE DATABASE auto_test_remote;
+USE auto_test_remote;
+
+##### create tables #####
+connection child2_1;
+CHILD_CREATE_TABLE
+connection master_1;
+MASTER_CREATE_TABLE
+CREATE TABLE ta_l (
+id INT NOT NULL,
+a INT,
+PRIMARY KEY(id)
+) MASTER_1_ENGINE MASTER_1_CHARSET MASTER_1_COMMENT_2_1
+INSERT INTO ta_l VALUES
+(1, 11),
+(2, 22),
+(3, 33),
+(4, 44),
+(5, 55);
+
+##### create functions #####
+connection master_1;
+CREATE FUNCTION `plusone`( param INT ) RETURNS INT
+BEGIN
+RETURN param + 1;
+END //
+connection child2_1;
+CREATE FUNCTION `plusone`( param INT ) RETURNS INT
+BEGIN
+RETURN param + 1;
+END //
+
+########## spider_use_pushdown_udf=0 ##########
+connection master_1;
+SET @@spider_use_pushdown_udf = 0;
+
+##### test SELECTs #####
+connection master_1;
+SELECT * FROM ta_l WHERE id = plusone(1);
+id	a
+2	22
+SELECT * FROM ta_l WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32);
+id	a
+3	33
+connection child2_1;
+SELECT argument FROM mysql.general_log WHERE argument LIKE "%select%" AND argument NOT LIKE "%argument%";
+argument
+select `id`,`a` from `auto_test_remote`.`ta_r`
+select `id`,`a` from `auto_test_remote`.`ta_r`
+
+##### test UPDATEs #####
+connection master_1;
+UPDATE ta_l SET a = plusone(221) WHERE id = plusone(1);
+SELECT * FROM ta_l;
+id	a
+1	11
+2	222
+3	33
+4	44
+5	55
+UPDATE ta_l SET a = plusone(332) WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32);
+SELECT * FROM ta_l;
+id	a
+1	11
+2	222
+3	333
+4	44
+5	55
+connection child2_1;
+SELECT argument FROM mysql.general_log WHERE argument LIKE "%update%" AND argument NOT LIKE "%argument%";
+argument
+select `id`,`a` from `auto_test_remote`.`ta_r` for update
+update `auto_test_remote`.`ta_r` set `a` = 222 where `id` = 2 limit 1
+select `id`,`a` from `auto_test_remote`.`ta_r` for update
+update `auto_test_remote`.`ta_r` set `a` = 333 where `id` = 3 and `a` = 33 limit 1
+
+##### test DELETEs #####
+connection master_1;
+DELETE FROM ta_l WHERE id = plusone(1);
+SELECT * FROM ta_l;
+id	a
+1	11
+3	333
+4	44
+5	55
+DELETE FROM ta_l WHERE id IN (plusone(1), plusone(2), plusone(3)) AND a = plusone(43);
+SELECT * FROM ta_l;
+id	a
+1	11
+3	333
+5	55
+connection child2_1;
+SELECT argument FROM mysql.general_log WHERE (argument LIKE "%delete%" OR argument LIKE "%update%") AND argument NOT LIKE "%argument%";
+argument
+select `id` from `auto_test_remote`.`ta_r` for update
+delete from `auto_test_remote`.`ta_r` where `id` = 2 limit 1
+select `id`,`a` from `auto_test_remote`.`ta_r` for update
+delete from `auto_test_remote`.`ta_r` where `id` = 4 and `a` = 44 limit 1
+
+##### reset records #####
+connection master_1;
+TRUNCATE TABLE ta_l;
+INSERT INTO ta_l VALUES
+(1, 11),
+(2, 22),
+(3, 33),
+(4, 44),
+(5, 55);
+
+########## spider_use_pushdown_udf=1 ##########
+connection master_1;
+SET @@spider_use_pushdown_udf = 1;
+
+##### test SELECTs #####
+connection master_1;
+SELECT * FROM ta_l WHERE id = plusone(1);
+id	a
+2	22
+SELECT * FROM ta_l WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32);
+id	a
+3	33
+connection child2_1;
+SELECT argument FROM mysql.general_log WHERE argument LIKE "%select%" AND argument NOT LIKE "%argument%";
+argument
+select t0.`id` `id`,t0.`a` `a` from `auto_test_remote`.`ta_r` t0 where (t0.`id` = (`plusone`(1)))
+select t0.`id` `id`,t0.`a` `a` from `auto_test_remote`.`ta_r` t0 where ((t0.`id` in( (`plusone`(1)) , (`plusone`(2)))) and (t0.`a` = (`plusone`(32))))
+
+##### test UPDATEs #####
+connection master_1;
+UPDATE ta_l SET a = plusone(221) WHERE id = plusone(1);
+SELECT * FROM ta_l;
+id	a
+1	11
+2	222
+3	33
+4	44
+5	55
+UPDATE ta_l SET a = plusone(332) WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32);
+SELECT * FROM ta_l;
+id	a
+1	11
+2	222
+3	333
+4	44
+5	55
+connection child2_1;
+SELECT argument FROM mysql.general_log WHERE argument LIKE "%update%" AND argument NOT LIKE "%argument%";
+argument
+update `auto_test_remote`.`ta_r` set `a` = (`plusone`(221)) where (`id` = (`plusone`(1)))
+update `auto_test_remote`.`ta_r` set `a` = (`plusone`(332)) where ((`id` in( (`plusone`(1)) , (`plusone`(2)))) and (`a` = (`plusone`(32))))
+
+##### test DELETEs #####
+connection master_1;
+DELETE FROM ta_l WHERE id = plusone(1);
+SELECT * FROM ta_l;
+id	a
+1	11
+3	333
+4	44
+5	55
+DELETE FROM ta_l WHERE id IN (plusone(1), plusone(2), plusone(3)) AND a = plusone(43);
+SELECT * FROM ta_l;
+id	a
+1	11
+3	333
+5	55
+connection child2_1;
+SELECT argument FROM mysql.general_log WHERE (argument LIKE "%delete%" OR argument LIKE "%update%") AND argument NOT LIKE "%argument%";
+argument
+delete from `auto_test_remote`.`ta_r` where (`id` = (`plusone`(1)))
+delete from `auto_test_remote`.`ta_r` where ((`id` in( (`plusone`(1)) , (`plusone`(2)) , (`plusone`(3)))) and (`a` = (`plusone`(43))))
+
+deinit
+connection master_1;
+DROP FUNCTION `plusone`;
+DROP DATABASE IF EXISTS auto_test_local;
+connection child2_1;
+SET @@global.general_log = @general_log_backup;
+SET @@global.log_output = @log_output_backup;
+DROP FUNCTION `plusone`;
+DROP DATABASE IF EXISTS auto_test_remote;
+for master_1
+for child2
+child2_1
+child2_2
+child2_3
+for child3
+child3_1
+child3_2
+child3_3
+
+end of test
diff --git a/storage/spider/mysql-test/spider/t/udf_pushdown.inc b/storage/spider/mysql-test/spider/t/udf_pushdown.inc
new file mode 100644
index 00000000000..160e8af21b2
--- /dev/null
+++ b/storage/spider/mysql-test/spider/t/udf_pushdown.inc
@@ -0,0 +1,48 @@
+--echo
+--echo ##### test SELECTs #####
+--connection master_1
+SELECT * FROM ta_l WHERE id = plusone(1);
+SELECT * FROM ta_l WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32);
+
+if ($USE_CHILD_GROUP2)
+{
+  --connection child2_1
+  SELECT argument FROM mysql.general_log WHERE argument LIKE "%select%" AND argument NOT LIKE "%argument%";
+  --disable_query_log
+  TRUNCATE TABLE mysql.general_log;
+  --enable_query_log
+}
+
+--echo
+--echo ##### test UPDATEs #####
+--connection master_1
+UPDATE ta_l SET a = plusone(221) WHERE id = plusone(1);
+SELECT * FROM ta_l;
+UPDATE ta_l SET a = plusone(332) WHERE id IN (plusone(1), plusone(2)) AND a = plusone(32);
+SELECT * FROM ta_l;
+
+if ($USE_CHILD_GROUP2)
+{
+  --connection child2_1
+  SELECT argument FROM mysql.general_log WHERE argument LIKE "%update%" AND argument NOT LIKE "%argument%";
+  --disable_query_log
+  TRUNCATE TABLE mysql.general_log;
+  --enable_query_log
+}
+
+--echo
+--echo ##### test DELETEs #####
+--connection master_1
+DELETE FROM ta_l WHERE id = plusone(1);
+SELECT * FROM ta_l;
+DELETE FROM ta_l WHERE id IN (plusone(1), plusone(2), plusone(3)) AND a = plusone(43);
+SELECT * FROM ta_l;
+
+if ($USE_CHILD_GROUP2)
+{
+  --connection child2_1
+  SELECT argument FROM mysql.general_log WHERE (argument LIKE "%delete%" OR argument LIKE "%update%") AND argument NOT LIKE "%argument%";
+  --disable_query_log
+  TRUNCATE TABLE mysql.general_log;
+  --enable_query_log
+}
diff --git a/storage/spider/mysql-test/spider/t/udf_pushdown.test b/storage/spider/mysql-test/spider/t/udf_pushdown.test
new file mode 100644
index 00000000000..2eadbbbb40b
--- /dev/null
+++ b/storage/spider/mysql-test/spider/t/udf_pushdown.test
@@ -0,0 +1,141 @@
+--disable_warnings
+--disable_query_log
+--disable_result_log
+--source test_init.inc
+--enable_result_log
+--enable_query_log
+
+--echo #
+--echo # MDEV-26545 Spider does not correctly handle UDF and stored function in where conds
+--echo #
+
+let $CHILD_CREATE_TABLE=
+  CREATE TABLE ta_r (
+    id INT NOT NULL,
+    a INT,
+    PRIMARY KEY(id)
+  ) $CHILD2_1_ENGINE $CHILD2_1_CHARSET;
+
+let $MASTER_CREATE_TABLE_OUTPUT=
+  CREATE TABLE ta_l (
+    id INT NOT NULL,
+    a INT,
+    PRIMARY KEY(id)
+  ) MASTER_1_ENGINE MASTER_1_CHARSET MASTER_1_COMMENT_2_1;
+
+let $MASTER_CREATE_TABLE=
+  CREATE TABLE ta_l (
+    id INT NOT NULL,
+    a INT,
+    PRIMARY KEY(id)
+  ) $MASTER_1_ENGINE $MASTER_1_CHARSET $MASTER_1_COMMENT_2_1;
+
+--echo
+--echo ##### enable general_log #####
+--connection child2_1
+SET @general_log_backup = @@global.general_log;
+SET @log_output_backup = @@global.log_output;
+SET @@global.general_log = 1;
+SET @@global.log_output = "TABLE";
+TRUNCATE TABLE mysql.general_log;
+
+--echo
+--echo ##### create databases #####
+--connection master_1
+CREATE DATABASE auto_test_local;
+USE auto_test_local;
+if ($USE_CHILD_GROUP2)
+{
+  --connection child2_1
+  CREATE DATABASE auto_test_remote;
+  USE auto_test_remote;
+}
+
+--echo
+--echo ##### create tables #####
+if ($USE_CHILD_GROUP2)
+{
+  --connection child2_1
+  --disable_query_log
+  echo CHILD_CREATE_TABLE;
+  eval $CHILD_CREATE_TABLE;
+  --enable_query_log
+}
+
+--connection master_1
+--disable_query_log
+echo MASTER_CREATE_TABLE;
+echo $MASTER_CREATE_TABLE_OUTPUT;
+eval $MASTER_CREATE_TABLE;
+--enable_query_log
+
+INSERT INTO ta_l VALUES
+    (1, 11),
+    (2, 22),
+    (3, 33),
+    (4, 44),
+    (5, 55);
+
+--echo
+--echo ##### create functions #####
+--connection master_1
+DELIMITER //;
+CREATE FUNCTION `plusone`( param INT ) RETURNS INT
+BEGIN
+    RETURN param + 1;
+END //
+DELIMITER ;//
+
+--connection child2_1
+DELIMITER //;
+CREATE FUNCTION `plusone`( param INT ) RETURNS INT
+BEGIN
+    RETURN param + 1;
+END //
+DELIMITER ;//
+
+--echo
+--echo ########## spider_use_pushdown_udf=0 ##########
+--connection master_1
+SET @@spider_use_pushdown_udf = 0;
+--source udf_pushdown.inc
+
+--echo
+--echo ##### reset records #####
+--connection master_1
+TRUNCATE TABLE ta_l;
+INSERT INTO ta_l VALUES
+    (1, 11),
+    (2, 22),
+    (3, 33),
+    (4, 44),
+    (5, 55);
+
+--echo
+--echo ########## spider_use_pushdown_udf=1 ##########
+--connection master_1
+SET @@spider_use_pushdown_udf = 1;
+--source udf_pushdown.inc
+
+--echo
+--echo deinit
+--disable_warnings
+--connection master_1
+DROP FUNCTION `plusone`;
+DROP DATABASE IF EXISTS auto_test_local;
+if ($USE_CHILD_GROUP2)
+{
+  --connection child2_1
+  SET @@global.general_log = @general_log_backup;
+  SET @@global.log_output = @log_output_backup;
+  DROP FUNCTION `plusone`;
+  DROP DATABASE IF EXISTS auto_test_remote;
+}
+--disable_query_log
+--disable_result_log
+--source test_deinit.inc
+--enable_result_log
+--enable_query_log
+--enable_warnings
+--echo
+--echo end of test
diff --git a/storage/spider/spd_db_mysql.cc b/storage/spider/spd_db_mysql.cc
index 3cfc1f7eff1..0fd5112380b 100644
--- a/storage/spider/spd_db_mysql.cc
+++ b/storage/spider/spd_db_mysql.cc
@@ -4985,10 +4985,16 @@ int spider_db_mbase_util::open_item_func(
         separator_str_length = SPIDER_SQL_AND_LEN;
       }
       break;
+    case Item_func::FUNC_SP:
     case Item_func::UDF_FUNC:
       use_pushdown_udf = spider_param_use_pushdown_udf(spider->trx->thd,
         spider->share->use_pushdown_udf);
       if (!use_pushdown_udf)
+        /*
+          This is the default behavior because the remote nodes may deal with
+          the function in an unexpected way (e.g. not having the same
+          definition). Users can turn it on if they know what they are doing.
+        */
         DBUG_RETURN(ER_SPIDER_COND_SKIP_NUM);
       if (str)
       {
diff --git a/storage/spider/spd_param.cc b/storage/spider/spd_param.cc
index 61fd4339ca5..8e56cae89a8 100644
--- a/storage/spider/spd_param.cc
+++ b/storage/spider/spd_param.cc
@@ -1978,7 +1978,7 @@ static MYSQL_THDVAR_INT(
   "Remote server transmission existence when UDF is used at condition and \"engine_condition_pushdown=1\"", /* comment */
   NULL, /* check */
   NULL, /* update */
-  -1, /* def */
+  0, /* def */
   -1, /* min */
   1, /* max */
   0 /* blk */
-- 
cgit v1.2.1


From 21d19ed45b1b71c2815559a8ad68888a4bfe902f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Tue, 21 Sep 2021 18:02:04 +0300
Subject: Silence a warning about unused Bison label

---
 sql/sql_yacc.yy | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy
index c9dbc6fa8cc..0f6773c899d 100644
--- a/sql/sql_yacc.yy
+++ b/sql/sql_yacc.yy
@@ -71,6 +71,9 @@
 /* warning C4065: switch statement contains 'default' but no 'case' labels */
 #pragma warning (disable : 4065)
 #endif
+#ifdef __GNUC__
+#pragma GCC diagnostic ignored "-Wunused-label" /* yyexhaustedlab: */
+#endif
 
 int yylex(void *yylval, void *yythd);
 
-- 
cgit v1.2.1


From 1cb218c37cc3fe01a1ff2fe9b1cbfb591e90d5ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Wed, 22 Sep 2021 14:15:00 +0300
Subject: MDEV-26450: Corruption due to innodb_undo_log_truncate

At least since commit 055a3334adc004bd3a897990c2f93178e6bb5f90
(MDEV-13564) the undo log truncation in InnoDB did not work correctly.

The main issue is that during the execution of
trx_purge_truncate_history() some pages of the newly truncated
undo tablespace could be discarded.

fsp_try_extend_data_file(): Apply the peculiar rounding of
fil_space_t::size_in_header only to the system tablespace,
whose size can be expressed in megabytes in a configuration parameter.
Other files may freely grow by a number of pages.

fseg_alloc_free_page_low(): Do allow the extension of undo tablespaces,
and mention the file name in the error message.

mtr_t::commit_shrink(): Implement crash-safe shrinking of a tablespace
file. First, durably write the log, then shrink the file, and finally
release the page latches of the rebuilt tablespace. Refactored from
trx_purge_truncate_history().

log_write_and_flush_prepare(), log_write_and_flush(): New functions
to durably write log during mtr_t::commit_shrink().
---
 mysql-test/suite/innodb/r/undo_truncate.result |  22 +-----
 mysql-test/suite/innodb/t/undo_truncate.test   |  64 +--------------
 storage/innobase/fsp/fsp0fsp.cc                |  20 ++---
 storage/innobase/include/log0log.h             |  14 +++-
 storage/innobase/include/mtr0mtr.h             |   6 +-
 storage/innobase/include/mtr0mtr.ic            |   6 +-
 storage/innobase/log/log0log.cc                | 103 +++++++++++++++++++++++--
 storage/innobase/mtr/mtr0mtr.cc                |  86 ++++++++++++++++++++-
 storage/innobase/trx/trx0purge.cc              |  21 +----
 9 files changed, 219 insertions(+), 123 deletions(-)

diff --git a/mysql-test/suite/innodb/r/undo_truncate.result b/mysql-test/suite/innodb/r/undo_truncate.result
index 89171d36d0f..67a587b3c1c 100644
--- a/mysql-test/suite/innodb/r/undo_truncate.result
+++ b/mysql-test/suite/innodb/r/undo_truncate.result
@@ -10,28 +10,12 @@ SET @trunc_start=
 WHERE variable_name = 'innodb_undo_truncations');
 create table t1(keyc int primary key, c char(100)) engine = innodb;
 create table t2(keyc int primary key, c char(100)) engine = innodb;
-CREATE PROCEDURE populate_t1()
-BEGIN
-DECLARE i INT DEFAULT 1;
-while (i <= 20000) DO
-insert into t1 values (i, 'a');
-SET i = i + 1;
-END WHILE;
-END |
-CREATE PROCEDURE populate_t2()
-BEGIN
-DECLARE i INT DEFAULT 1;
-while (i <= 20000) DO
-insert into t2 values (i, 'a');
-SET i = i + 1;
-END WHILE;
-END |
 connect  con1,localhost,root,,;
 begin;
-call populate_t1();
+insert into t1 select seq,'a' from seq_1_to_20000;
 connect  con2,localhost,root,,;
 begin;
-call populate_t2();
+insert into t2 select seq,'a' from seq_1_to_20000;
 connection con1;
 update t1 set c = 'mysql';
 connection con2;
@@ -53,8 +37,6 @@ commit;
 disconnect con2;
 connection default;
 drop table t1, t2;
-drop PROCEDURE populate_t1;
-drop PROCEDURE populate_t2;
 InnoDB		0 transactions not purged
 SET GLOBAL innodb_undo_logs = @save_undo_logs;
 SET GLOBAL innodb_purge_rseg_truncate_frequency = @save_frequency;
diff --git a/mysql-test/suite/innodb/t/undo_truncate.test b/mysql-test/suite/innodb/t/undo_truncate.test
index 4f350e380ee..9feee84491a 100644
--- a/mysql-test/suite/innodb/t/undo_truncate.test
+++ b/mysql-test/suite/innodb/t/undo_truncate.test
@@ -5,6 +5,7 @@
 # --source include/innodb_page_size.inc
 --source include/innodb_page_size_small.inc
 --source include/have_undo_tablespaces.inc
+--source include/have_sequence.inc
 
 call mtr.add_suppression("InnoDB: The transaction log size is too large");
 
@@ -27,37 +28,14 @@ WHERE variable_name = 'innodb_undo_truncations');
 create table t1(keyc int primary key, c char(100)) engine = innodb;
 create table t2(keyc int primary key, c char(100)) engine = innodb;
 #
-delimiter |;
-CREATE PROCEDURE populate_t1()
-BEGIN
-        DECLARE i INT DEFAULT 1;
-        while (i <= 20000) DO
-                insert into t1 values (i, 'a');
-                SET i = i + 1;
-        END WHILE;
-END |
-delimiter ;|
-#
-delimiter |;
-CREATE PROCEDURE populate_t2()
-BEGIN
-        DECLARE i INT DEFAULT 1;
-        while (i <= 20000) DO
-                insert into t2 values (i, 'a');
-                SET i = i + 1;
-        END WHILE;
-END |
-delimiter ;|
-#
-#
 let DATADIR = `select @@datadir`;
 connect (con1,localhost,root,,);
 begin;
-send call populate_t1();
+send insert into t1 select seq,'a' from seq_1_to_20000;
 
 connect (con2,localhost,root,,);
 begin;
-send call populate_t2();
+send insert into t2 select seq,'a' from seq_1_to_20000;
 
 connection con1; reap; send update t1 set c = 'mysql';
 connection con2; reap; send update t2 set c = 'mysql';
@@ -67,25 +45,12 @@ connection con1; reap; send delete from t1;
 connection con2; reap; delete from t2;
 connection con1; reap;
 
-let CHECKFILE = $MYSQL_TMP_DIR/check.txt;
-perl;
-($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size1)
-	= stat("$ENV{DATADIR}/undo001");
-($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size2)
-	= stat("$ENV{DATADIR}/undo002");
-open(OUT, ">$ENV{CHECKFILE}") || die;
-print OUT "let \$size1='$size1,$size2';\n";
-close(OUT);
-EOF
-
 SET GLOBAL innodb_undo_log_truncate = 1;
 commit; disconnect con1;
 connection con2; commit; disconnect con2;
 
 connection default;
 drop table t1, t2;
-drop PROCEDURE populate_t1;
-drop PROCEDURE populate_t2;
 
 --source include/wait_all_purged.inc
 
@@ -100,29 +65,6 @@ if (`select @@innodb_page_size IN (4096,8192,16384)`)
   source include/wait_condition.inc;
 }
 
---source $CHECKFILE
-perl;
-($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size1)
-	= stat("$ENV{DATADIR}/undo001");
-($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size2)
-	= stat("$ENV{DATADIR}/undo002");
-open(OUT, ">$ENV{CHECKFILE}") || die;
-print OUT "let \$size2='$size1,$size2';\n";
-close(OUT);
-EOF
-
---source $CHECKFILE
---remove_file $CHECKFILE
-
-if ($size1 == $size2)
-{
-  # This fails for innodb_page_size=64k, occasionally also for 32k.
-  if (`select @@innodb_page_size IN (4096,8192,16384)`)
-  {
-    echo Truncation did not happen: $size1;
-  }
-}
-
 SET GLOBAL innodb_undo_logs = @save_undo_logs;
 SET GLOBAL innodb_purge_rseg_truncate_frequency = @save_frequency;
 SET GLOBAL innodb_undo_log_truncate = @save_truncate;
diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc
index 43989d57db8..064eb133ebc 100644
--- a/storage/innobase/fsp/fsp0fsp.cc
+++ b/storage/innobase/fsp/fsp0fsp.cc
@@ -956,11 +956,13 @@ fsp_try_extend_data_file(fil_space_t* space, fsp_header_t* header, mtr_t* mtr)
 		return(0);
 	}
 
-	/* We ignore any fragments of a full megabyte when storing the size
-	to the space header */
+	/* For the system tablespace, we ignore any fragments of a
+	full megabyte when storing the size to the space header */
 
-	space->size_in_header = ut_2pow_round(
-		space->size, (1024 * 1024) / page_size.physical());
+	space->size_in_header = space->id
+		? space->size
+		: ut_2pow_round(space->size,
+				(1024 * 1024) / page_size.physical());
 
 	mlog_write_ulint(
 		header + FSP_SIZE, space->size_in_header, MLOG_4BYTES, mtr);
@@ -1392,7 +1394,7 @@ fsp_alloc_free_page(
 		/* It must be that we are extending a single-table tablespace
 		whose size is still < 64 pages */
 
-		ut_a(!is_system_tablespace(space_id));
+		ut_a(!is_predefined_tablespace(space_id));
 		if (page_no >= FSP_EXTENT_SIZE) {
 			ib::error() << "Trying to extend a single-table"
 				" tablespace " << space->name << " , by single"
@@ -2514,14 +2516,14 @@ take_hinted_page:
 		return(NULL);
 	}
 
-	if (space->size <= ret_page && !is_system_tablespace(space_id)) {
+	if (space->size <= ret_page && !is_predefined_tablespace(space_id)) {
 		/* It must be that we are extending a single-table
 		tablespace whose size is still < 64 pages */
 
 		if (ret_page >= FSP_EXTENT_SIZE) {
-			ib::error() << "Error (2): trying to extend"
-			" a single-table tablespace " << space_id
-			<< " by single page(s) though the"
+			ib::error() << "Trying to extend '"
+			<< space->chain.start->name
+			<< "' by single page(s) though the"
 			<< " space size " << space->size
 			<< ". Page no " << ret_page << ".";
 			ut_ad(!has_done_reservation);
diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h
index 612a27976e7..c0b92fb7497 100644
--- a/storage/innobase/include/log0log.h
+++ b/storage/innobase/include/log0log.h
@@ -2,7 +2,7 @@
 
 Copyright (c) 1995, 2017, Oracle and/or its affiliates. All rights reserved.
 Copyright (c) 2009, Google Inc.
-Copyright (c) 2017, 2020, MariaDB Corporation.
+Copyright (c) 2017, 2021, MariaDB Corporation.
 
 Portions of this file contain modifications contributed and copyrighted by
 Google, Inc. Those modifications are gratefully acknowledged and are described
@@ -189,9 +189,15 @@ log_write_up_to(
 /** write to the log file up to the last log entry.
 @param[in]	sync	whether we want the written log
 also to be flushed to disk. */
-void
-log_buffer_flush_to_disk(
-	bool sync = true);
+void log_buffer_flush_to_disk(bool sync= true);
+
+
+/** Prepare to invoke log_write_and_flush(), before acquiring log_sys.mutex. */
+#define log_write_and_flush_prepare() log_write_mutex_enter()
+
+/** Durably write the log up to log_sys.lsn and release log_sys.mutex. */
+ATTRIBUTE_COLD void log_write_and_flush();
+
 /****************************************************************//**
 This functions writes the log buffer to the log file and if 'flush'
 is set it forces a flush of the log file as well. This is meant to be
diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h
index b57a38f8eab..6d729beab12 100644
--- a/storage/innobase/include/mtr0mtr.h
+++ b/storage/innobase/include/mtr0mtr.h
@@ -2,7 +2,7 @@
 
 Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2012, Facebook Inc.
-Copyright (c) 2013, 2020, MariaDB Corporation.
+Copyright (c) 2013, 2021, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -161,6 +161,10 @@ struct mtr_t {
 	/** Commit the mini-transaction. */
 	void commit();
 
+	/** Commit a mini-transaction that is shrinking a tablespace.
+	@param space   tablespace that is being shrunk */
+	ATTRIBUTE_COLD void commit_shrink(fil_space_t &space);
+
 	/** Commit a mini-transaction that did not modify any pages,
 	but generated some redo log on a higher level, such as
 	MLOG_FILE_NAME records and a MLOG_CHECKPOINT marker.
diff --git a/storage/innobase/include/mtr0mtr.ic b/storage/innobase/include/mtr0mtr.ic
index a45d088d5d7..24477689db8 100644
--- a/storage/innobase/include/mtr0mtr.ic
+++ b/storage/innobase/include/mtr0mtr.ic
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, 2020, MariaDB Corporation.
+Copyright (c) 2017, 2021, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -53,8 +53,8 @@ mtr_t::memo_push(void* object, mtr_memo_type_t type)
 
 	/* If this mtr has x-fixed a clean page then we set
 	the made_dirty flag. This tells us if we need to
-	grab log_flush_order_mutex at mtr_commit so that we
-	can insert the dirtied page to the flush list. */
+	grab log_sys.flush_order_mutex at mtr_t::commit() so that we
+	can insert the dirtied page into the flush list. */
 
 	if ((type == MTR_MEMO_PAGE_X_FIX || type == MTR_MEMO_PAGE_SX_FIX)
 	    && !m_made_dirty) {
diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc
index 4c68f3743e9..945c97daf4a 100644
--- a/storage/innobase/log/log0log.cc
+++ b/storage/innobase/log/log0log.cc
@@ -2,7 +2,7 @@
 
 Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2009, Google Inc.
-Copyright (c) 2014, 2020, MariaDB Corporation.
+Copyright (c) 2014, 2021, MariaDB Corporation.
 
 Portions of this file contain modifications contributed and copyrighted by
 Google, Inc. Those modifications are gratefully acknowledged and are described
@@ -1247,12 +1247,103 @@ loop:
 /** write to the log file up to the last log entry.
 @param[in]	sync	whether we want the written log
 also to be flushed to disk. */
-void
-log_buffer_flush_to_disk(
-	bool sync)
+void log_buffer_flush_to_disk(bool sync)
 {
-	ut_ad(!srv_read_only_mode);
-	log_write_up_to(log_get_lsn(), sync);
+  ut_ad(!srv_read_only_mode);
+  log_write_up_to(log_get_lsn(), sync);
+}
+
+
+/** Durably write the log and release log_sys.mutex */
+ATTRIBUTE_COLD void log_write_and_flush()
+{
+  ut_ad(!srv_read_only_mode);
+  ut_ad(!recv_no_log_write);
+  ut_ad(!recv_recovery_is_on());
+
+  /* The following code is adapted from log_write_up_to(). */
+  DBUG_PRINT("ib_log", ("write " LSN_PF " to " LSN_PF,
+                        log_sys->write_lsn, log_sys->lsn));
+  log_sys->n_pending_flushes++;
+  log_sys->current_flush_lsn= log_sys->lsn;
+  os_event_reset(log_sys->flush_event);
+  ut_ad(log_sys->buf_free != log_sys->buf_next_to_write);
+  ulint start_offset= log_sys->buf_next_to_write;
+  ulint end_offset= log_sys->buf_free;
+  ulint area_start= ut_2pow_round(start_offset, ulint(OS_FILE_LOG_BLOCK_SIZE));
+  ulint area_end= ut_calc_align(end_offset, ulint(OS_FILE_LOG_BLOCK_SIZE));
+  ulong write_ahead_size= srv_log_write_ahead_size;
+
+  log_block_set_flush_bit(log_sys->buf + area_start, TRUE);
+  log_block_set_checkpoint_no(log_sys->buf + area_end - OS_FILE_LOG_BLOCK_SIZE,
+                              log_sys->next_checkpoint_no);
+  lsn_t write_lsn= log_sys->lsn;
+  byte *write_buf= log_sys->buf;
+
+  ut_ad(area_end - area_start > 0);
+
+  log_buffer_switch();
+
+  log_group_set_fields(&log_sys->log, log_sys->write_lsn);
+
+  /* Erase the end of the last log block. */
+  memset(write_buf + end_offset, 0,
+         ~end_offset & (OS_FILE_LOG_BLOCK_SIZE - 1));
+  /* Calculate pad_size if needed. */
+  ulint pad_size= 0;
+  if (write_ahead_size > OS_FILE_LOG_BLOCK_SIZE)
+  {
+    lsn_t end_offset=
+      log_group_calc_lsn_offset(ut_uint64_align_up(write_lsn,
+                                                   OS_FILE_LOG_BLOCK_SIZE),
+                                &log_sys->log);
+    ulint end_offset_in_unit= (ulint) (end_offset % write_ahead_size);
+
+    if (end_offset_in_unit && (area_end - area_start) > end_offset_in_unit)
+    {
+      /* The first block in the unit was initialized after the last
+      writing. Needs to be written padded data once. */
+      pad_size= std::min(ulint(write_ahead_size) - end_offset_in_unit,
+                         log_sys->buf_size - area_end);
+      memset(write_buf + area_end, 0, pad_size);
+    }
+  }
+
+  if (log_sys->is_encrypted())
+    log_crypt(write_buf + area_start, log_sys->write_lsn,
+              area_end - area_start);
+
+  /* Do the write to the log files */
+  log_group_write_buf(&log_sys->log, write_buf + area_start,
+                      area_end - area_start + pad_size,
+#ifdef UNIV_DEBUG
+                      pad_size,
+#endif /* UNIV_DEBUG */
+                      ut_uint64_align_down(log_sys->write_lsn,
+                                           OS_FILE_LOG_BLOCK_SIZE),
+                      start_offset - area_start);
+  srv_stats.log_padded.add(pad_size);
+  log_sys->write_lsn= write_lsn;
+
+  log_write_mutex_exit();
+
+  /* Code adapted from log_write_flush_to_disk_low() */
+
+  ut_a(log_sys->n_pending_flushes == 1); /* No other threads here */
+
+  if (srv_file_flush_method != SRV_O_DSYNC)
+    fil_flush(SRV_LOG_SPACE_FIRST_ID);
+
+  log_sys->flushed_to_disk_lsn= log_sys->current_flush_lsn;
+
+  log_sys->n_pending_flushes--;
+
+  os_event_set(log_sys->flush_event);
+
+  lsn_t wrote_lsn= log_sys->write_lsn, flush_lsn= log_sys->flushed_to_disk_lsn;
+  log_mutex_exit();
+
+  innobase_mysql_log_notify(wrote_lsn, flush_lsn);
 }
 
 /****************************************************************//**
diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc
index fefc0687ddb..4bd90e7842d 100644
--- a/storage/innobase/mtr/mtr0mtr.cc
+++ b/storage/innobase/mtr/mtr0mtr.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, 2020, MariaDB Corporation.
+Copyright (c) 2017, 2021, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -465,6 +465,90 @@ mtr_t::commit()
   release_resources();
 }
 
+#ifdef UNIV_DEBUG
+/** Check that all pages belong to a shrunk tablespace. */
+struct Shrink
+{
+  const fil_space_t &space;
+  Shrink(const fil_space_t &space) : space(space) {}
+
+  bool operator()(const mtr_memo_slot_t *slot) const
+  {
+    if (!slot->object)
+      return true;
+    switch (slot->type) {
+    default:
+      ut_ad("invalid type" == 0);
+      return false;
+    case MTR_MEMO_MODIFY:
+      break;
+    case MTR_MEMO_SPACE_X_LOCK:
+      ut_ad(&space == slot->object);
+      return true;
+    case MTR_MEMO_PAGE_X_FIX:
+    case MTR_MEMO_PAGE_SX_FIX:
+      const buf_page_t &bpage= static_cast<buf_block_t*>(slot->object)->page;
+      const page_id_t &id= bpage.id;
+      if (id.space() == 0 && id.page_no() == TRX_SYS_PAGE_NO)
+      {
+        ut_ad(srv_is_undo_tablespace(space.id));
+        break;
+      }
+      ut_ad(id.space() == space.id);
+      ut_ad(id.page_no() < space.size);
+      ut_ad(bpage.state == BUF_BLOCK_FILE_PAGE);
+      ut_ad(!bpage.oldest_modification);
+      break;
+    }
+    return true;
+  }
+};
+#endif
+
+/** Commit a mini-transaction that is shrinking a tablespace.
+@param space   tablespace that is being shrunk */
+void mtr_t::commit_shrink(fil_space_t &space)
+{
+  ut_ad(is_active());
+  ut_ad(!is_inside_ibuf());
+  ut_ad(!high_level_read_only);
+  ut_ad(m_modifications);
+  ut_ad(m_made_dirty);
+  ut_ad(!recv_recovery_is_on());
+  ut_ad(m_log_mode == MTR_LOG_ALL);
+  ut_ad(UT_LIST_GET_LEN(space.chain) == 1);
+
+  log_write_and_flush_prepare();
+
+  const lsn_t start_lsn= finish_write(prepare_write());
+
+  log_flush_order_mutex_enter();
+  /* Durably write the reduced FSP_SIZE before truncating the data file. */
+  log_write_and_flush();
+
+  os_file_truncate(space.chain.start->name, space.chain.start->handle,
+                   os_offset_t(space.size) << srv_page_size_shift, true);
+
+  ut_d(m_memo.for_each_block_in_reverse(CIterate<Shrink>(space)));
+
+  m_memo.for_each_block_in_reverse(CIterate<const ReleaseBlocks>
+                                   (ReleaseBlocks(start_lsn, m_commit_lsn,
+                                                  m_flush_observer)));
+  log_flush_order_mutex_exit();
+
+  mutex_enter(&fil_system->mutex);
+  ut_ad(space.is_being_truncated);
+  ut_ad(space.stop_new_ops);
+  space.stop_new_ops= false;
+  space.is_being_truncated= false;
+  mutex_exit(&fil_system->mutex);
+
+  m_memo.for_each_block_in_reverse(CIterate<ReleaseLatches>());
+  srv_stats.log_write_requests.inc();
+
+  release_resources();
+}
+
 /** Commit a mini-transaction that did not modify any pages,
 but generated some redo log on a higher level, such as
 MLOG_FILE_NAME records and a MLOG_CHECKPOINT marker.
diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc
index 2025ac70beb..a9593807d53 100644
--- a/storage/innobase/trx/trx0purge.cc
+++ b/storage/innobase/trx/trx0purge.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, 2020, MariaDB Corporation.
+Copyright (c) 2017, 2021, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -1062,26 +1062,11 @@ trx_purge_initiate_truncate(
 		rseg->last_del_marks = FALSE;
 	}
 
-	mtr.commit();
-	/* Write-ahead the redo log record. */
-	log_write_up_to(mtr.commit_lsn(), true);
+	mtr.commit_shrink(*space);
 
-	/* Trim the file size. */
-	os_file_truncate(file->name, file->handle,
-			 os_offset_t(size) << srv_page_size_shift, true);
-
-	/* This is only executed by the srv_purge_coordinator_thread. */
+	/* No mutex; this is only updated by the purge coordinator. */
 	export_vars.innodb_undo_truncations++;
 
-	/* TODO: PUNCH_HOLE the garbage (with write-ahead logging) */
-
-	mutex_enter(&fil_system->mutex);
-	ut_ad(space->stop_new_ops);
-	ut_ad(space->is_being_truncated);
-	space->stop_new_ops = false;
-	space->is_being_truncated = false;
-	mutex_exit(&fil_system->mutex);
-
 	if (purge_sys->rseg != NULL
 	    && purge_sys->rseg->last_page_no == FIL_NULL) {
 		/* If purge_sys->rseg is pointing to rseg that was recently
-- 
cgit v1.2.1


From c8c21a4c8f5daf09ff0c49eb1541bb665e05fb1d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= <jan.lindstrom@mariadb.com>
Date: Wed, 22 Sep 2021 08:30:05 +0300
Subject: MDEV-26571 : galera_sr.GCF-627 MTR failed: Result length mismatch

Test changes only: do not output mysql.wsrep_streaming_log
contents.
---
 mysql-test/suite/galera_sr/r/GCF-627.result | 12 ++++++------
 mysql-test/suite/galera_sr/t/GCF-627.test   |  8 ++------
 2 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/mysql-test/suite/galera_sr/r/GCF-627.result b/mysql-test/suite/galera_sr/r/GCF-627.result
index 65d8c95ad08..7cd2ab63ff3 100644
--- a/mysql-test/suite/galera_sr/r/GCF-627.result
+++ b/mysql-test/suite/galera_sr/r/GCF-627.result
@@ -2,7 +2,6 @@ connection node_2;
 connection node_1;
 connection node_1;
 CREATE TABLE t1 (f1 INTEGER, f2 VARCHAR(10)) ENGINE=InnoDB;
-CREATE TABLE t2 (f1 INTEGER);
 SET SESSION wsrep_trx_fragment_size = 1;
 SET AUTOCOMMIT=OFF;
 START TRANSACTION;
@@ -16,9 +15,10 @@ connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1;
 INSERT INTO t1 VALUES (2);
 ERROR 42S02: Table 'test.t1' doesn't exist
 connection node_1;
-SELECT * FROM mysql.wsrep_streaming_log;
-node_uuid	trx_id	seqno	flags	frag
+SELECT COUNT(*) FROM mysql.wsrep_streaming_log;
+COUNT(*)
+0
 connection node_2;
-SELECT * FROM mysql.wsrep_streaming_log;
-node_uuid	trx_id	seqno	flags	frag
-DROP TABLE t2;
+SELECT COUNT(*) FROM mysql.wsrep_streaming_log;
+COUNT(*)
+0
diff --git a/mysql-test/suite/galera_sr/t/GCF-627.test b/mysql-test/suite/galera_sr/t/GCF-627.test
index ad351eb9da6..6990c12314d 100644
--- a/mysql-test/suite/galera_sr/t/GCF-627.test
+++ b/mysql-test/suite/galera_sr/t/GCF-627.test
@@ -1,9 +1,7 @@
 --source include/galera_cluster.inc
---source include/have_innodb.inc
 
 --connection node_1
 CREATE TABLE t1 (f1 INTEGER, f2 VARCHAR(10)) ENGINE=InnoDB;
-CREATE TABLE t2 (f1 INTEGER);
 
 SET SESSION wsrep_trx_fragment_size = 1;
 SET AUTOCOMMIT=OFF;
@@ -25,11 +23,9 @@ INSERT INTO t1 VALUES (2);
 --let $wait_condition = SELECT COUNT(*) = 0 FROM mysql.wsrep_streaming_log
 --source include/wait_condition.inc
 
-SELECT * FROM mysql.wsrep_streaming_log;
+SELECT COUNT(*) FROM mysql.wsrep_streaming_log;
 
 --connection node_2
 --let $wait_condition = SELECT COUNT(*) = 0 FROM mysql.wsrep_streaming_log
 --source include/wait_condition.inc
-SELECT * FROM mysql.wsrep_streaming_log;
-
-DROP TABLE t2;
+SELECT COUNT(*) FROM mysql.wsrep_streaming_log;
-- 
cgit v1.2.1


From 913efaa328026455cef0e608c25eaed042344b15 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= <jan.lindstrom@mariadb.com>
Date: Thu, 23 Sep 2021 12:59:39 +0300
Subject: MDEV-26566 : galera.galera_var_cluster_address MTR failed: InnoDB:
 Assertion failure in file row0ins.cc line 3206

Actual problem was that we tried to calculate persistent statistics
to wsrep_schema tables in this case wsrep_streaming_log. These tables
should not have persistent statistics. Therefore, in table creation
tables should be created with STATS_PERSISTENT=0 table option. During
rolling-upgrade tables naturally already exists, thus we need to
alter them to contain STATS_PERSISTENT=0 table option.
---
 mysql-test/suite/galera/r/galera_schema.result     | 114 +++++++++++++++++++++
 mysql-test/suite/galera/t/galera_schema.test       |  61 +++++++++++
 .../galera_3nodes/r/galera_wsrep_schema.result     |   4 +-
 .../r/galera_wsrep_schema_init.result              |   4 +-
 sql/wsrep_schema.cc                                |  48 +++++++--
 5 files changed, 220 insertions(+), 11 deletions(-)
 create mode 100644 mysql-test/suite/galera/r/galera_schema.result
 create mode 100644 mysql-test/suite/galera/t/galera_schema.test

diff --git a/mysql-test/suite/galera/r/galera_schema.result b/mysql-test/suite/galera/r/galera_schema.result
new file mode 100644
index 00000000000..24a4099c94d
--- /dev/null
+++ b/mysql-test/suite/galera/r/galera_schema.result
@@ -0,0 +1,114 @@
+connection node_2;
+connection node_1;
+CREATE TABLE IF NOT EXISTS wsrep_cluster
+(
+cluster_uuid CHAR(36) PRIMARY KEY,
+view_id BIGINT NOT NULL,
+view_seqno BIGINT NOT NULL,
+protocol_version INT NOT NULL,
+capabilities INT NOT NULL
+) ENGINE=InnoDB;
+CREATE TABLE IF NOT EXISTS wsrep_cluster_members
+(
+node_uuid CHAR(36) PRIMARY KEY,
+cluster_uuid CHAR(36) NOT NULL,
+node_name CHAR(32) NOT NULL,
+node_incoming_address VARCHAR(256) NOT NULL
+) ENGINE=InnoDB;
+CREATE TABLE IF NOT EXISTS wsrep_cluster_members_history
+(
+node_uuid CHAR(36) PRIMARY KEY,
+cluster_uuid CHAR(36) NOT NULL,
+last_view_id BIGINT NOT NULL,
+last_view_seqno BIGINT NOT NULL,
+node_name CHAR(32) NOT NULL,
+node_incoming_address VARCHAR(256) NOT NULL
+) ENGINE=InnoDB;
+CREATE TABLE IF NOT EXISTS wsrep_streaming_log
+(
+node_uuid CHAR(36),
+trx_id BIGINT,
+seqno BIGINT,
+flags INT NOT NULL,
+frag LONGBLOB NOT NULL,
+PRIMARY KEY (node_uuid, trx_id, seqno)
+) ENGINE=InnoDB;
+DELETE FROM wsrep_cluster;
+DELETE FROM wsrep_cluster_members;
+ALTER TABLE wsrep_cluster STATS_PERSISTENT=0;
+ALTER TABLE wsrep_cluster_members STATS_PERSISTENT=0;
+ALTER TABLE wsrep_cluster_members_history STATS_PERSISTENT=0;
+ALTER TABLE wsrep_streaming_log STATS_PERSISTENT=0;
+SHOW CREATE TABLE wsrep_cluster;
+Table	Create Table
+wsrep_cluster	CREATE TABLE `wsrep_cluster` (
+  `cluster_uuid` char(36) NOT NULL,
+  `view_id` bigint(20) NOT NULL,
+  `view_seqno` bigint(20) NOT NULL,
+  `protocol_version` int(11) NOT NULL,
+  `capabilities` int(11) NOT NULL,
+  PRIMARY KEY (`cluster_uuid`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 STATS_PERSISTENT=0
+SHOW CREATE TABLE wsrep_cluster_members;
+Table	Create Table
+wsrep_cluster_members	CREATE TABLE `wsrep_cluster_members` (
+  `node_uuid` char(36) NOT NULL,
+  `cluster_uuid` char(36) NOT NULL,
+  `node_name` char(32) NOT NULL,
+  `node_incoming_address` varchar(256) NOT NULL,
+  PRIMARY KEY (`node_uuid`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 STATS_PERSISTENT=0
+SHOW CREATE TABLE wsrep_cluster_members_history;
+Table	Create Table
+wsrep_cluster_members_history	CREATE TABLE `wsrep_cluster_members_history` (
+  `node_uuid` char(36) NOT NULL,
+  `cluster_uuid` char(36) NOT NULL,
+  `last_view_id` bigint(20) NOT NULL,
+  `last_view_seqno` bigint(20) NOT NULL,
+  `node_name` char(32) NOT NULL,
+  `node_incoming_address` varchar(256) NOT NULL,
+  PRIMARY KEY (`node_uuid`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 STATS_PERSISTENT=0
+SHOW CREATE TABLE wsrep_streaming_log;
+Table	Create Table
+wsrep_streaming_log	CREATE TABLE `wsrep_streaming_log` (
+  `node_uuid` char(36) NOT NULL,
+  `trx_id` bigint(20) NOT NULL,
+  `seqno` bigint(20) NOT NULL,
+  `flags` int(11) NOT NULL,
+  `frag` longblob NOT NULL,
+  PRIMARY KEY (`node_uuid`,`trx_id`,`seqno`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 STATS_PERSISTENT=0
+SHOW CREATE TABLE mysql.wsrep_cluster;
+Table	Create Table
+wsrep_cluster	CREATE TABLE `wsrep_cluster` (
+  `cluster_uuid` char(36) NOT NULL,
+  `view_id` bigint(20) NOT NULL,
+  `view_seqno` bigint(20) NOT NULL,
+  `protocol_version` int(11) NOT NULL,
+  `capabilities` int(11) NOT NULL,
+  PRIMARY KEY (`cluster_uuid`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 STATS_PERSISTENT=0
+SHOW CREATE TABLE mysql.wsrep_cluster_members;
+Table	Create Table
+wsrep_cluster_members	CREATE TABLE `wsrep_cluster_members` (
+  `node_uuid` char(36) NOT NULL,
+  `cluster_uuid` char(36) NOT NULL,
+  `node_name` char(32) NOT NULL,
+  `node_incoming_address` varchar(256) NOT NULL,
+  PRIMARY KEY (`node_uuid`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 STATS_PERSISTENT=0
+SHOW CREATE TABLE mysql.wsrep_streaming_log;
+Table	Create Table
+wsrep_streaming_log	CREATE TABLE `wsrep_streaming_log` (
+  `node_uuid` char(36) NOT NULL,
+  `trx_id` bigint(20) NOT NULL,
+  `seqno` bigint(20) NOT NULL,
+  `flags` int(11) NOT NULL,
+  `frag` longblob NOT NULL,
+  PRIMARY KEY (`node_uuid`,`trx_id`,`seqno`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 STATS_PERSISTENT=0
+DROP TABLE wsrep_cluster;
+DROP TABLE wsrep_cluster_members;
+DROP TABLE wsrep_cluster_members_history;
+DROP TABLE wsrep_streaming_log;
diff --git a/mysql-test/suite/galera/t/galera_schema.test b/mysql-test/suite/galera/t/galera_schema.test
new file mode 100644
index 00000000000..a3ee814c393
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_schema.test
@@ -0,0 +1,61 @@
+--source include/galera_cluster.inc
+
+CREATE TABLE IF NOT EXISTS wsrep_cluster
+(
+  cluster_uuid CHAR(36) PRIMARY KEY,
+  view_id BIGINT NOT NULL,
+  view_seqno BIGINT NOT NULL,
+  protocol_version INT NOT NULL,
+  capabilities INT NOT NULL
+) ENGINE=InnoDB;
+
+CREATE TABLE IF NOT EXISTS wsrep_cluster_members
+(
+  node_uuid CHAR(36) PRIMARY KEY,
+  cluster_uuid CHAR(36) NOT NULL,
+  node_name CHAR(32) NOT NULL,
+  node_incoming_address VARCHAR(256) NOT NULL
+) ENGINE=InnoDB;
+
+CREATE TABLE IF NOT EXISTS wsrep_cluster_members_history
+(
+  node_uuid CHAR(36) PRIMARY KEY,
+  cluster_uuid CHAR(36) NOT NULL,
+  last_view_id BIGINT NOT NULL,
+  last_view_seqno BIGINT NOT NULL,
+  node_name CHAR(32) NOT NULL,
+  node_incoming_address VARCHAR(256) NOT NULL
+) ENGINE=InnoDB;
+
+CREATE TABLE IF NOT EXISTS wsrep_streaming_log
+(
+  node_uuid CHAR(36),
+  trx_id BIGINT,
+  seqno BIGINT,
+  flags INT NOT NULL,
+  frag LONGBLOB NOT NULL,
+  PRIMARY KEY (node_uuid, trx_id, seqno)
+) ENGINE=InnoDB;
+
+DELETE FROM wsrep_cluster;
+DELETE FROM wsrep_cluster_members;
+
+ALTER TABLE wsrep_cluster STATS_PERSISTENT=0;
+ALTER TABLE wsrep_cluster_members STATS_PERSISTENT=0;
+ALTER TABLE wsrep_cluster_members_history STATS_PERSISTENT=0;
+ALTER TABLE wsrep_streaming_log STATS_PERSISTENT=0;
+
+SHOW CREATE TABLE wsrep_cluster;
+SHOW CREATE TABLE wsrep_cluster_members;
+SHOW CREATE TABLE wsrep_cluster_members_history;
+SHOW CREATE TABLE wsrep_streaming_log;
+
+SHOW CREATE TABLE mysql.wsrep_cluster;
+SHOW CREATE TABLE mysql.wsrep_cluster_members;
+#SHOW CREATE TABLE mysql.wsrep_cluster_members_history;
+SHOW CREATE TABLE mysql.wsrep_streaming_log;
+
+DROP TABLE wsrep_cluster;
+DROP TABLE wsrep_cluster_members;
+DROP TABLE wsrep_cluster_members_history;
+DROP TABLE wsrep_streaming_log;
diff --git a/mysql-test/suite/galera_3nodes/r/galera_wsrep_schema.result b/mysql-test/suite/galera_3nodes/r/galera_wsrep_schema.result
index ba01eab0e26..f51eb815cd5 100644
--- a/mysql-test/suite/galera_3nodes/r/galera_wsrep_schema.result
+++ b/mysql-test/suite/galera_3nodes/r/galera_wsrep_schema.result
@@ -14,7 +14,7 @@ wsrep_cluster	CREATE TABLE `wsrep_cluster` (
   `protocol_version` int(11) NOT NULL,
   `capabilities` int(11) NOT NULL,
   PRIMARY KEY (`cluster_uuid`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 STATS_PERSISTENT=0
 SHOW CREATE TABLE mysql.wsrep_cluster_members;
 Table	Create Table
 wsrep_cluster_members	CREATE TABLE `wsrep_cluster_members` (
@@ -23,7 +23,7 @@ wsrep_cluster_members	CREATE TABLE `wsrep_cluster_members` (
   `node_name` char(32) NOT NULL,
   `node_incoming_address` varchar(256) NOT NULL,
   PRIMARY KEY (`node_uuid`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 STATS_PERSISTENT=0
 SELECT COUNT(*) AS EXPECT_1 FROM mysql.wsrep_cluster;
 EXPECT_1
 1
diff --git a/mysql-test/suite/galera_3nodes/r/galera_wsrep_schema_init.result b/mysql-test/suite/galera_3nodes/r/galera_wsrep_schema_init.result
index 2a29afd62be..d9d3e817bed 100644
--- a/mysql-test/suite/galera_3nodes/r/galera_wsrep_schema_init.result
+++ b/mysql-test/suite/galera_3nodes/r/galera_wsrep_schema_init.result
@@ -14,7 +14,7 @@ wsrep_cluster	CREATE TABLE `wsrep_cluster` (
   `protocol_version` int(11) NOT NULL,
   `capabilities` int(11) NOT NULL,
   PRIMARY KEY (`cluster_uuid`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 STATS_PERSISTENT=0
 SHOW CREATE TABLE mysql.wsrep_cluster_members;
 Table	Create Table
 wsrep_cluster_members	CREATE TABLE `wsrep_cluster_members` (
@@ -23,7 +23,7 @@ wsrep_cluster_members	CREATE TABLE `wsrep_cluster_members` (
   `node_name` char(32) NOT NULL,
   `node_incoming_address` varchar(256) NOT NULL,
   PRIMARY KEY (`node_uuid`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 STATS_PERSISTENT=0
 SELECT @@sql_safe_updates;
 @@sql_safe_updates
 1
diff --git a/sql/wsrep_schema.cc b/sql/wsrep_schema.cc
index 06f0f7840ab..5960a9934c1 100644
--- a/sql/wsrep_schema.cc
+++ b/sql/wsrep_schema.cc
@@ -1,4 +1,4 @@
-/* Copyright (C) 2015-2019 Codership Oy <info@codership.com>
+/* Copyright (C) 2015-2021 Codership Oy <info@codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -54,7 +54,7 @@ static const std::string create_cluster_table_str=
   "view_seqno BIGINT NOT NULL,"
   "protocol_version INT NOT NULL,"
   "capabilities INT NOT NULL"
-  ") ENGINE=InnoDB";
+  ") ENGINE=InnoDB STATS_PERSISTENT=0";
 
 static const std::string create_members_table_str=
   "CREATE TABLE IF NOT EXISTS " + wsrep_schema_str + "." + members_table_str +
@@ -63,7 +63,7 @@ static const std::string create_members_table_str=
   "cluster_uuid CHAR(36) NOT NULL,"
   "node_name CHAR(32) NOT NULL,"
   "node_incoming_address VARCHAR(256) NOT NULL"
-  ") ENGINE=InnoDB";
+  ") ENGINE=InnoDB STATS_PERSISTENT=0";
 
 #ifdef WSREP_SCHEMA_MEMBERS_HISTORY
 static const std::string cluster_member_history_table_str= "wsrep_cluster_member_history";
@@ -76,7 +76,7 @@ static const std::string create_members_history_table_str=
   "last_view_seqno BIGINT NOT NULL,"
   "node_name CHAR(32) NOT NULL,"
   "node_incoming_address VARCHAR(256) NOT NULL"
-  ") ENGINE=InnoDB";
+  ") ENGINE=InnoDB STATS_PERSISTENT=0";
 #endif /* WSREP_SCHEMA_MEMBERS_HISTORY */
 
 static const std::string create_frag_table_str=
@@ -88,7 +88,7 @@ static const std::string create_frag_table_str=
   "flags INT NOT NULL, "
   "frag LONGBLOB NOT NULL, "
   "PRIMARY KEY (node_uuid, trx_id, seqno)"
-  ") ENGINE=InnoDB";
+  ") ENGINE=InnoDB STATS_PERSISTENT=0";
 
 static const std::string delete_from_cluster_table=
   "DELETE FROM " + wsrep_schema_str + "." + cluster_table_str;
@@ -96,6 +96,26 @@ static const std::string delete_from_cluster_table=
 static const std::string delete_from_members_table=
   "DELETE FROM " + wsrep_schema_str + "." + members_table_str;
 
+/* For rolling upgrade we need to use ALTER. We do not want
+persistent statistics to be collected from these tables. */
+static const std::string alter_cluster_table=
+  "ALTER TABLE " + wsrep_schema_str + "." + cluster_table_str +
+  " STATS_PERSISTENT=0";
+
+static const std::string alter_members_table=
+  "ALTER TABLE " + wsrep_schema_str + "." + members_table_str +
+  " STATS_PERSISTENT=0";
+
+#ifdef WSREP_SCHEMA_MEMBERS_HISTORY
+static const std::string alter_members_history_table=
+  "ALTER TABLE " + wsrep_schema_str + "." + members_history_table_str +
+  " STATS_PERSISTENT=0";
+#endif
+
+static const std::string alter_frag_table=
+  "ALTER TABLE " + wsrep_schema_str + "." + sr_table_str +
+  " STATS_PERSISTENT=0";
+
 namespace Wsrep_schema_impl
 {
 
@@ -657,13 +677,27 @@ int Wsrep_schema::init()
       Wsrep_schema_impl::execute_SQL(thd,
                                      create_members_history_table_str.c_str(),
                                      create_members_history_table_str.size()) ||
+      Wsrep_schema_impl::execute_SQL(thd,
+                                     alter_members_history_table.c_str(),
+                                     alter_members_history_table.size()) ||
 #endif /* WSREP_SCHEMA_MEMBERS_HISTORY */
       Wsrep_schema_impl::execute_SQL(thd,
                                      create_frag_table_str.c_str(),
-                                     create_frag_table_str.size())) {
+                                     create_frag_table_str.size()) ||
+      Wsrep_schema_impl::execute_SQL(thd,
+                                     alter_cluster_table.c_str(),
+                                     alter_cluster_table.size()) ||
+      Wsrep_schema_impl::execute_SQL(thd,
+                                     alter_members_table.c_str(),
+                                     alter_members_table.size()) ||
+      Wsrep_schema_impl::execute_SQL(thd,
+                                     alter_frag_table.c_str(),
+	                             alter_frag_table.size()))
+  {
     ret= 1;
   }
-  else {
+  else
+  {
     ret= 0;
   }
 
-- 
cgit v1.2.1


From 9c2d9236e1b3d012e26bd1d76ce06c681927cd03 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= <jan.lindstrom@mariadb.com>
Date: Fri, 24 Sep 2021 08:00:00 +0300
Subject: Update wsrep-lib submodule

---
 wsrep-lib | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/wsrep-lib b/wsrep-lib
index 4f1c201c9d8..efb4aab090c 160000
--- a/wsrep-lib
+++ b/wsrep-lib
@@ -1 +1 @@
-Subproject commit 4f1c201c9d8ba96f9f43e9aafe8bb6a6c8aeceb7
+Subproject commit efb4aab090cb9c1b57b9e7f9988ae1c41f48344f
-- 
cgit v1.2.1


From 15efb7ed48265b8d40897a13c0b8e09c6bdd34c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Wed, 22 Sep 2021 16:40:47 +0300
Subject: MDEV-26626 fixup: Do not advance checkpoint during startup

While the redo log is being resized in srv_start(),
we must not write checkpoint information to the old log.

Thanks to Matthias Leich for noticing this.
---
 storage/innobase/buf/buf0flu.cc | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc
index fb687b4f51b..eada6bde282 100644
--- a/storage/innobase/buf/buf0flu.cc
+++ b/storage/innobase/buf/buf0flu.cc
@@ -2271,7 +2271,9 @@ unemployed:
 
       mysql_mutex_unlock(&buf_pool.flush_list_mutex);
 
-      if (!recv_recovery_is_on() && srv_operation == SRV_OPERATION_NORMAL)
+      if (!recv_recovery_is_on() &&
+          !srv_startup_is_before_trx_rollback_phase &&
+          srv_operation == SRV_OPERATION_NORMAL)
         log_checkpoint();
 
       mysql_mutex_lock(&buf_pool.flush_list_mutex);
-- 
cgit v1.2.1


From f5fddae3cbcff2d2531f0ce61bd144212379aa42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Fri, 24 Sep 2021 08:22:19 +0300
Subject: MDEV-26450: Corruption due to innodb_undo_log_truncate

At least since commit 055a3334adc004bd3a897990c2f93178e6bb5f90
(MDEV-13564) the undo log truncation in InnoDB did not work correctly.

The main issue is that during the execution of
trx_purge_truncate_history() some pages of the newly truncated
undo tablespace could be discarded.

This is improved from commit 1cb218c37cc3fe01a1ff2fe9b1cbfb591e90d5ce
which was applied to earlier-version branches.

fsp_try_extend_data_file(): Apply the peculiar rounding of
fil_space_t::size_in_header only to the system tablespace,
whose size can be expressed in megabytes in a configuration parameter.
Other files may freely grow by a number of pages.

fseg_alloc_free_page_low(): Do allow the extension of undo tablespaces,
and mention the file name in the error message.

mtr_t::commit_shrink(): Implement crash-safe shrinking of a tablespace:
(1) durably write the log
(2) release the page latches of the rebuilt tablespace
(3) release the mutexes
(4) truncate the file
(5) release the tablespace latch
This is refactored from trx_purge_truncate_history().

log_write_and_flush_prepare(), log_write_and_flush(): New functions
to durably write log during mtr_t::commit_shrink().
---
 mysql-test/suite/innodb/r/undo_truncate.result |  22 +----
 mysql-test/suite/innodb/t/undo_truncate.test   |  64 +-------------
 storage/innobase/fsp/fsp0fsp.cc                |  18 ++--
 storage/innobase/include/log0log.h             |  16 ++--
 storage/innobase/include/mtr0mtr.h             |   4 +
 storage/innobase/include/mtr0mtr.ic            |   4 +-
 storage/innobase/log/log0log.cc                |  32 ++++++-
 storage/innobase/mtr/mtr0mtr.cc                | 111 ++++++++++++++++++++++++-
 storage/innobase/trx/trx0purge.cc              |  21 +----
 9 files changed, 173 insertions(+), 119 deletions(-)

diff --git a/mysql-test/suite/innodb/r/undo_truncate.result b/mysql-test/suite/innodb/r/undo_truncate.result
index ad236bdecd4..54eeee9a9df 100644
--- a/mysql-test/suite/innodb/r/undo_truncate.result
+++ b/mysql-test/suite/innodb/r/undo_truncate.result
@@ -7,28 +7,12 @@ SET @trunc_start=
 WHERE variable_name = 'innodb_undo_truncations');
 create table t1(keyc int primary key, c char(100)) engine = innodb;
 create table t2(keyc int primary key, c char(100)) engine = innodb;
-CREATE PROCEDURE populate_t1()
-BEGIN
-DECLARE i INT DEFAULT 1;
-while (i <= 20000) DO
-insert into t1 values (i, 'a');
-SET i = i + 1;
-END WHILE;
-END |
-CREATE PROCEDURE populate_t2()
-BEGIN
-DECLARE i INT DEFAULT 1;
-while (i <= 20000) DO
-insert into t2 values (i, 'a');
-SET i = i + 1;
-END WHILE;
-END |
 connect  con1,localhost,root,,;
 begin;
-call populate_t1();
+insert into t1 select seq,'a' from seq_1_to_20000;
 connect  con2,localhost,root,,;
 begin;
-call populate_t2();
+insert into t2 select seq,'a' from seq_1_to_20000;
 connection con1;
 update t1 set c = 'mysql';
 connection con2;
@@ -50,8 +34,6 @@ commit;
 disconnect con2;
 connection default;
 drop table t1, t2;
-drop PROCEDURE populate_t1;
-drop PROCEDURE populate_t2;
 InnoDB		0 transactions not purged
 SET GLOBAL innodb_purge_rseg_truncate_frequency = @save_frequency;
 SET GLOBAL innodb_undo_log_truncate = @save_truncate;
diff --git a/mysql-test/suite/innodb/t/undo_truncate.test b/mysql-test/suite/innodb/t/undo_truncate.test
index d2a4e287305..9abca6179c4 100644
--- a/mysql-test/suite/innodb/t/undo_truncate.test
+++ b/mysql-test/suite/innodb/t/undo_truncate.test
@@ -1,6 +1,7 @@
 --source include/have_innodb.inc
 --source include/innodb_page_size.inc
 --source include/have_undo_tablespaces.inc
+--source include/have_sequence.inc
 
 SET @save_frequency = @@GLOBAL.innodb_purge_rseg_truncate_frequency;
 SET @save_truncate = @@GLOBAL.innodb_undo_log_truncate;
@@ -19,37 +20,14 @@ WHERE variable_name = 'innodb_undo_truncations');
 create table t1(keyc int primary key, c char(100)) engine = innodb;
 create table t2(keyc int primary key, c char(100)) engine = innodb;
 #
-delimiter |;
-CREATE PROCEDURE populate_t1()
-BEGIN
-        DECLARE i INT DEFAULT 1;
-        while (i <= 20000) DO
-                insert into t1 values (i, 'a');
-                SET i = i + 1;
-        END WHILE;
-END |
-delimiter ;|
-#
-delimiter |;
-CREATE PROCEDURE populate_t2()
-BEGIN
-        DECLARE i INT DEFAULT 1;
-        while (i <= 20000) DO
-                insert into t2 values (i, 'a');
-                SET i = i + 1;
-        END WHILE;
-END |
-delimiter ;|
-#
-#
 let DATADIR = `select @@datadir`;
 connect (con1,localhost,root,,);
 begin;
-send call populate_t1();
+send insert into t1 select seq,'a' from seq_1_to_20000;
 
 connect (con2,localhost,root,,);
 begin;
-send call populate_t2();
+send insert into t2 select seq,'a' from seq_1_to_20000;
 
 connection con1; reap; send update t1 set c = 'mysql';
 connection con2; reap; send update t2 set c = 'mysql';
@@ -59,25 +37,12 @@ connection con1; reap; send delete from t1;
 connection con2; reap; delete from t2;
 connection con1; reap;
 
-let CHECKFILE = $MYSQL_TMP_DIR/check.txt;
-perl;
-($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size1)
-	= stat("$ENV{DATADIR}/undo001");
-($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size2)
-	= stat("$ENV{DATADIR}/undo002");
-open(OUT, ">$ENV{CHECKFILE}") || die;
-print OUT "let \$size1='$size1,$size2';\n";
-close(OUT);
-EOF
-
 SET GLOBAL innodb_undo_log_truncate = 1;
 commit; disconnect con1;
 connection con2; commit; disconnect con2;
 
 connection default;
 drop table t1, t2;
-drop PROCEDURE populate_t1;
-drop PROCEDURE populate_t2;
 
 --source include/wait_all_purged.inc
 
@@ -93,28 +58,5 @@ if (`select @@innodb_page_size IN (8192,16384)`)
   source include/wait_condition.inc;
 }
 
---source $CHECKFILE
-perl;
-($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size1)
-	= stat("$ENV{DATADIR}/undo001");
-($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size2)
-	= stat("$ENV{DATADIR}/undo002");
-open(OUT, ">$ENV{CHECKFILE}") || die;
-print OUT "let \$size2='$size1,$size2';\n";
-close(OUT);
-EOF
-
---source $CHECKFILE
---remove_file $CHECKFILE
-
-if ($size1 == $size2)
-{
-  # This fails for innodb_page_size=64k, occasionally also for 32k.
-  if (`select @@innodb_page_size IN (8192,16384)`)
-  {
-    echo Truncation did not happen: $size1;
-  }
-}
-
 SET GLOBAL innodb_purge_rseg_truncate_frequency = @save_frequency;
 SET GLOBAL innodb_undo_log_truncate = @save_truncate;
diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc
index 3d5a7edd947..ae2ea90c3e5 100644
--- a/storage/innobase/fsp/fsp0fsp.cc
+++ b/storage/innobase/fsp/fsp0fsp.cc
@@ -586,7 +586,7 @@ void fsp_header_init(fil_space_t* space, uint32_t size, mtr_t* mtr)
 	in order to avoid optimizing away any unchanged most
 	significant bytes of FSP_SIZE. */
 	mtr->write<4,mtr_t::FORCED>(*block, FSP_HEADER_OFFSET + FSP_SIZE
-				   + block->frame, size);
+				    + block->frame, size);
 	ut_ad(0 == mach_read_from_4(FSP_HEADER_OFFSET + FSP_FREE_LIMIT
 				    + block->frame));
 	if (auto f = space->flags & ~FSP_FLAGS_MEM_MASK) {
@@ -780,10 +780,12 @@ fsp_try_extend_data_file(fil_space_t *space, buf_block_t *header, mtr_t *mtr)
 		return(0);
 	}
 
-	/* We ignore any fragments of a full megabyte when storing the size
-	to the space header */
+	/* For the system tablespace, we ignore any fragments of a
+	full megabyte when storing the size to the space header */
 
-	space->size_in_header = ut_2pow_round(space->size, (1024 * 1024) / ps);
+	space->size_in_header = space->id
+		? space->size
+		: ut_2pow_round(space->size, (1024 * 1024) / ps);
 
 	/* recv_sys_t::parse() expects to find a WRITE record that
 	covers all 4 bytes. Therefore, we must specify mtr_t::FORCED
@@ -2130,14 +2132,14 @@ take_hinted_page:
 		return(NULL);
 	}
 
-	if (space->size <= ret_page && !is_system_tablespace(space_id)) {
+	if (space->size <= ret_page && !is_predefined_tablespace(space_id)) {
 		/* It must be that we are extending a single-table
 		tablespace whose size is still < 64 pages */
 
 		if (ret_page >= FSP_EXTENT_SIZE) {
-			ib::error() << "Error (2): trying to extend"
-			" a single-table tablespace " << space_id
-			<< " by single page(s) though the"
+			ib::error() << "Trying to extend '"
+			<< space->chain.start->name
+			<< "' by single page(s) though the"
 			<< " space size " << space->size
 			<< ". Page no " << ret_page << ".";
 			ut_ad(!has_done_reservation);
diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h
index 460acaf5b66..1dcff513d7c 100644
--- a/storage/innobase/include/log0log.h
+++ b/storage/innobase/include/log0log.h
@@ -107,12 +107,16 @@ be flushed to the file system
 @param[in]	rotate_key	whether to rotate the encryption key */
 void log_write_up_to(lsn_t lsn, bool flush_to_disk, bool rotate_key = false);
 
-/** write to the log file up to the last log entry.
-@param[in]	sync	whether we want the written log
-also to be flushed to disk. */
-void
-log_buffer_flush_to_disk(
-	bool sync = true);
+/** Write to the log file up to the last log entry.
+@param sync  whether to wait for a durable write to complete */
+void log_buffer_flush_to_disk(bool sync= true);
+
+
+/** Prepare to invoke log_write_and_flush(), before acquiring log_sys.mutex. */
+ATTRIBUTE_COLD void log_write_and_flush_prepare();
+
+/** Durably write the log up to log_sys.lsn() and release log_sys.mutex. */
+ATTRIBUTE_COLD void log_write_and_flush();
 
 /** Make a checkpoint */
 ATTRIBUTE_COLD void log_make_checkpoint();
diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h
index f3db0008c3e..e253038e1ee 100644
--- a/storage/innobase/include/mtr0mtr.h
+++ b/storage/innobase/include/mtr0mtr.h
@@ -98,6 +98,10 @@ struct mtr_t {
   /** Commit the mini-transaction. */
   void commit();
 
+  /** Commit a mini-transaction that is shrinking a tablespace.
+  @param space   tablespace that is being shrunk */
+  ATTRIBUTE_COLD void commit_shrink(fil_space_t &space);
+
   /** Commit a mini-transaction that did not modify any pages,
   but generated some redo log on a higher level, such as
   FILE_MODIFY records and an optional FILE_CHECKPOINT marker.
diff --git a/storage/innobase/include/mtr0mtr.ic b/storage/innobase/include/mtr0mtr.ic
index 4a483379e21..48bdb9bb6d1 100644
--- a/storage/innobase/include/mtr0mtr.ic
+++ b/storage/innobase/include/mtr0mtr.ic
@@ -48,8 +48,8 @@ mtr_t::memo_push(void* object, mtr_memo_type_t type)
 
 	/* If this mtr has x-fixed a clean page then we set
 	the made_dirty flag. This tells us if we need to
-	grab log_flush_order_mutex at mtr_commit so that we
-	can insert the dirtied page to the flush list. */
+	grab log_sys.flush_order_mutex at mtr_t::commit() so that we
+	can insert the dirtied page into the flush list. */
 
 	if ((type == MTR_MEMO_PAGE_X_FIX || type == MTR_MEMO_PAGE_SX_FIX)
 	    && !m_made_dirty) {
diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc
index a6fa50dd753..257645cb6a4 100644
--- a/storage/innobase/log/log0log.cc
+++ b/storage/innobase/log/log0log.cc
@@ -835,15 +835,41 @@ void log_write_up_to(lsn_t lsn, bool flush_to_disk, bool rotate_key)
   log_flush_notify(flush_lsn);
 }
 
-/** write to the log file up to the last log entry.
-@param[in]	sync	whether we want the written log
-also to be flushed to disk. */
+/** Write to the log file up to the last log entry.
+@param sync  whether to wait for a durable write to complete */
 void log_buffer_flush_to_disk(bool sync)
 {
   ut_ad(!srv_read_only_mode);
   log_write_up_to(log_sys.get_lsn(std::memory_order_acquire), sync);
 }
 
+/** Prepare to invoke log_write_and_flush(), before acquiring log_sys.mutex. */
+ATTRIBUTE_COLD void log_write_and_flush_prepare()
+{
+  mysql_mutex_assert_not_owner(&log_sys.mutex);
+
+  while (flush_lock.acquire(log_sys.get_lsn() + 1) !=
+         group_commit_lock::ACQUIRED);
+  while (write_lock.acquire(log_sys.get_lsn() + 1) !=
+         group_commit_lock::ACQUIRED);
+}
+
+/** Durably write the log and release log_sys.mutex */
+ATTRIBUTE_COLD void log_write_and_flush()
+{
+  ut_ad(!srv_read_only_mode);
+  auto lsn= log_sys.get_lsn();
+  write_lock.set_pending(lsn);
+  log_write(false);
+  ut_a(log_sys.write_lsn == lsn);
+  write_lock.release(lsn);
+
+  lsn= write_lock.value();
+  flush_lock.set_pending(lsn);
+  log_write_flush_to_disk_low(lsn);
+  flush_lock.release(lsn);
+}
+
 /********************************************************************
 
 Tries to establish a big enough margin of free space in the log buffer, such
diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc
index 691b393561f..cf1574a56c4 100644
--- a/storage/innobase/mtr/mtr0mtr.cc
+++ b/storage/innobase/mtr/mtr0mtr.cc
@@ -1,7 +1,7 @@
 /*****************************************************************************
 
 Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, 2020, MariaDB Corporation.
+Copyright (c) 2017, 2021, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -459,6 +459,115 @@ void mtr_t::commit()
   release_resources();
 }
 
+#ifdef UNIV_DEBUG
+/** Check that all pages belong to a shrunk tablespace. */
+struct Shrink
+{
+  const page_id_t low, high;
+  Shrink(const fil_space_t &space) :
+    low({space.id, 0}), high({space.id, space.size}) {}
+
+  bool operator()(const mtr_memo_slot_t *slot) const
+  {
+    if (!slot->object)
+      return true;
+    switch (slot->type) {
+    default:
+      ut_ad("invalid type" == 0);
+      return false;
+    case MTR_MEMO_SPACE_X_LOCK:
+      ut_ad(low.space() == static_cast<fil_space_t*>(slot->object)->id);
+      return true;
+    case MTR_MEMO_PAGE_X_MODIFY:
+    case MTR_MEMO_PAGE_SX_MODIFY:
+    case MTR_MEMO_PAGE_X_FIX:
+    case MTR_MEMO_PAGE_SX_FIX:
+      const auto &bpage= static_cast<buf_block_t*>(slot->object)->page;
+      const auto id= bpage.id();
+      if (id == page_id_t{0, TRX_SYS_PAGE_NO})
+      {
+        ut_ad(srv_is_undo_tablespace(low.space()));
+        break;
+      }
+      ut_ad(id >= low);
+      ut_ad(id < high);
+      ut_ad(bpage.state() == BUF_BLOCK_FILE_PAGE);
+      ut_ad(bpage.oldest_modification() <= 1);
+      break;
+    }
+    return true;
+  }
+};
+#endif
+
+/** Commit a mini-transaction that is shrinking a tablespace.
+@param space   tablespace that is being shrunk */
+void mtr_t::commit_shrink(fil_space_t &space)
+{
+  ut_ad(is_active());
+  ut_ad(!is_inside_ibuf());
+  ut_ad(!high_level_read_only);
+  ut_ad(m_modifications);
+  ut_ad(m_made_dirty);
+  ut_ad(!recv_recovery_is_on());
+  ut_ad(m_log_mode == MTR_LOG_ALL);
+  ut_ad(UT_LIST_GET_LEN(space.chain) == 1);
+
+  log_write_and_flush_prepare();
+
+  const lsn_t start_lsn= finish_write(prepare_write()).first;
+
+  mysql_mutex_lock(&log_sys.flush_order_mutex);
+  /* Durably write the reduced FSP_SIZE before truncating the data file. */
+  log_write_and_flush();
+
+  if (m_freed_pages)
+  {
+    ut_ad(!m_freed_pages->empty());
+    ut_ad(m_freed_space == &space);
+    ut_ad(memo_contains(*m_freed_space));
+    ut_ad(is_named_space(m_freed_space));
+    m_freed_space->update_last_freed_lsn(m_commit_lsn);
+
+    if (!is_trim_pages())
+      for (const auto &range : *m_freed_pages)
+        m_freed_space->add_free_range(range);
+    else
+      m_freed_space->clear_freed_ranges();
+    delete m_freed_pages;
+    m_freed_pages= nullptr;
+    m_freed_space= nullptr;
+    /* mtr_t::start() will reset m_trim_pages */
+  }
+  else
+    ut_ad(!m_freed_space);
+
+  ut_d(m_memo.for_each_block_in_reverse(CIterate<Shrink>{space}));
+
+  m_memo.for_each_block_in_reverse(CIterate<const ReleaseBlocks>
+                                   (ReleaseBlocks(start_lsn, m_commit_lsn,
+                                                  m_memo)));
+  mysql_mutex_unlock(&log_sys.flush_order_mutex);
+
+  mutex_enter(&fil_system.mutex);
+  ut_ad(space.is_being_truncated);
+  ut_ad(space.is_stopping());
+  space.set_stopping(false);
+  space.is_being_truncated= false;
+  mutex_exit(&fil_system.mutex);
+
+  /* Truncate the file before releasing the space.latch. File extension
+  (and any allocation of pages beyond the current intended end of the file)
+  is covered by exclusive space.latch, which we are still holding here. */
+  os_file_truncate(space.chain.start->name, space.chain.start->handle,
+                   os_offset_t{space.size} << srv_page_size_shift, true);
+
+  m_memo.for_each_block_in_reverse(CIterate<ReleaseLatches>());
+  srv_stats.log_write_requests.inc();
+
+  release_resources();
+}
+
 /** Commit a mini-transaction that did not modify any pages,
 but generated some redo log on a higher level, such as
 FILE_MODIFY records and an optional FILE_CHECKPOINT marker.
diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc
index 4570d63b3d9..e417d1c5f9f 100644
--- a/storage/innobase/trx/trx0purge.cc
+++ b/storage/innobase/trx/trx0purge.cc
@@ -754,27 +754,11 @@ not_free:
 			rseg->needs_purge = false;
 		}
 
-		mtr.commit();
-		/* Write-ahead the redo log record. */
-		log_write_up_to(mtr.commit_lsn(), true);
-
-		/* Trim the file size. */
-		os_file_truncate(file->name, file->handle,
-				 os_offset_t(size) << srv_page_size_shift,
-				 true);
+		mtr.commit_shrink(space);
 
-		/* This is only executed by srv_purge_coordinator_thread. */
+		/* No mutex; this is only updated by the purge coordinator. */
 		export_vars.innodb_undo_truncations++;
 
-		/* In MDEV-8319 (10.5) we will PUNCH_HOLE the garbage
-		(with write-ahead logging). */
-		mutex_enter(&fil_system.mutex);
-		ut_ad(&space == purge_sys.truncate.current);
-		ut_ad(space.is_being_truncated);
-		purge_sys.truncate.current->set_stopping(false);
-		purge_sys.truncate.current->is_being_truncated = false;
-		mutex_exit(&fil_system.mutex);
-
 		if (purge_sys.rseg != NULL
 		    && purge_sys.rseg->last_page_no == FIL_NULL) {
 			/* If purge_sys.rseg is pointing to rseg that
@@ -806,6 +790,7 @@ not_free:
 
 		ib::info() << "Truncated " << file->name;
 		purge_sys.truncate.last = purge_sys.truncate.current;
+		ut_ad(&space == purge_sys.truncate.current);
 		purge_sys.truncate.current = NULL;
 	}
 }
-- 
cgit v1.2.1


From f5794e1dc6e3d27405daeae850b8e69fd631b62d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Fri, 24 Sep 2021 08:24:03 +0300
Subject: MDEV-26445 innodb_undo_log_truncate is unnecessarily slow

trx_purge_truncate_history(): Do not force a write of the undo tablespace
that is being truncated. Instead, prevent page writes by acquiring
an exclusive latch on all dirty pages of the tablespace.

fseg_create(): Relax an assertion that could fail if a dirty undo page
is being initialized during undo tablespace truncation (and
trx_purge_truncate_history() already acquired an exclusive latch on it).

fsp_page_create(): If we are truncating a tablespace, try to reuse
a page that we may have already latched exclusively (because it was
in buf_pool.flush_list). To some extent, this helps the test
innodb.undo_truncate,16k to avoid running out of buffer pool.

mtr_t::commit_shrink(): Mark as clean all pages that are outside the
new bounds of the tablespace, and only add the newly reinitialized pages
to the buf_pool.flush_list.

buf_page_create(): Do not unnecessarily invoke change buffer merge on
undo tablespaces.

buf_page_t::clear_oldest_modification(bool temporary): Move some
assertions to the caller buf_page_write_complete().

innodb.undo_truncate: Use a bigger innodb_buffer_pool_size=24M.
On my system, it would otherwise hang 1 out of 1547 attempts
(on the 40th repeat of innodb.undo_truncate,16k).
Other page sizes were not affected.
---
 mysql-test/suite/innodb/t/undo_truncate.opt  |   1 +
 mysql-test/suite/innodb/t/undo_truncate.test |  16 +-
 storage/innobase/buf/buf0buf.cc              |   1 +
 storage/innobase/buf/buf0flu.cc              |   2 +
 storage/innobase/fsp/fsp0fsp.cc              |  36 +-
 storage/innobase/include/buf0buf.h           |   2 -
 storage/innobase/mtr/mtr0mtr.cc              |  34 +-
 storage/innobase/trx/trx0purge.cc            | 517 ++++++++++++++-------------
 8 files changed, 330 insertions(+), 279 deletions(-)
 create mode 100644 mysql-test/suite/innodb/t/undo_truncate.opt

diff --git a/mysql-test/suite/innodb/t/undo_truncate.opt b/mysql-test/suite/innodb/t/undo_truncate.opt
new file mode 100644
index 00000000000..f4d78725c6e
--- /dev/null
+++ b/mysql-test/suite/innodb/t/undo_truncate.opt
@@ -0,0 +1 @@
+--innodb-buffer-pool-size=24M
diff --git a/mysql-test/suite/innodb/t/undo_truncate.test b/mysql-test/suite/innodb/t/undo_truncate.test
index 9abca6179c4..8de93814ed8 100644
--- a/mysql-test/suite/innodb/t/undo_truncate.test
+++ b/mysql-test/suite/innodb/t/undo_truncate.test
@@ -3,6 +3,10 @@
 --source include/have_undo_tablespaces.inc
 --source include/have_sequence.inc
 
+--disable_query_log
+call mtr.add_suppression("InnoDB: Difficult to find free blocks in the buffer pool");
+--enable_query_log
+
 SET @save_frequency = @@GLOBAL.innodb_purge_rseg_truncate_frequency;
 SET @save_truncate = @@GLOBAL.innodb_undo_log_truncate;
 SET GLOBAL innodb_undo_log_truncate = 0;
@@ -46,17 +50,5 @@ drop table t1, t2;
 
 --source include/wait_all_purged.inc
 
-# Truncation will normally not occur with innodb_page_size=64k,
-# and occasionally not with innodb_page_size=32k,
-# because the undo log will not grow enough.
-# TODO: For some reason this does not occur on 4k either!
-if (`select @@innodb_page_size IN (8192,16384)`)
-{
-  let $wait_condition = (SELECT variable_value!=@trunc_start
-                         FROM information_schema.global_status
-                         WHERE variable_name = 'innodb_undo_truncations');
-  source include/wait_condition.inc;
-}
-
 SET GLOBAL innodb_purge_rseg_truncate_frequency = @save_frequency;
 SET GLOBAL innodb_undo_log_truncate = @save_truncate;
diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc
index 113da7746fa..1c126191df3 100644
--- a/storage/innobase/buf/buf0buf.cc
+++ b/storage/innobase/buf/buf0buf.cc
@@ -3796,6 +3796,7 @@ loop:
   /* Delete possible entries for the page from the insert buffer:
   such can exist if the page belonged to an index which was dropped */
   if (page_id < page_id_t{SRV_SPACE_ID_UPPER_BOUND, 0} &&
+      !srv_is_undo_tablespace(page_id.space()) &&
       !recv_recovery_is_on())
     ibuf_merge_or_delete_for_page(nullptr, page_id, zip_size);
 
diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc
index eada6bde282..7bf26515e88 100644
--- a/storage/innobase/buf/buf0flu.cc
+++ b/storage/innobase/buf/buf0flu.cc
@@ -363,10 +363,12 @@ void buf_page_write_complete(const IORequest &request)
   const bool temp= fsp_is_system_temporary(bpage->id().space());
 
   mysql_mutex_lock(&buf_pool.mutex);
+  mysql_mutex_assert_not_owner(&buf_pool.flush_list_mutex);
   buf_pool.stat.n_pages_written++;
   /* While we do not need any mutex for clearing oldest_modification
   here, we hope that it will be in the same cache line with io_fix,
   whose changes must be protected by buf_pool.mutex. */
+  ut_ad(temp || bpage->oldest_modification() > 2);
   bpage->clear_oldest_modification(temp);
   ut_ad(bpage->io_fix() == BUF_IO_WRITE);
   bpage->set_io_fix(BUF_IO_NONE);
diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc
index ae2ea90c3e5..a4f622a19ec 100644
--- a/storage/innobase/fsp/fsp0fsp.cc
+++ b/storage/innobase/fsp/fsp0fsp.cc
@@ -1054,11 +1054,36 @@ static
 buf_block_t*
 fsp_page_create(fil_space_t *space, page_no_t offset, mtr_t *mtr)
 {
-  buf_block_t *free_block= buf_LRU_get_free_block(false);
-  buf_block_t *block= buf_page_create(space, static_cast<uint32_t>(offset),
-                                      space->zip_size(), mtr, free_block);
+  buf_block_t *block, *free_block;
+
+  if (UNIV_UNLIKELY(space->is_being_truncated))
+  {
+    const page_id_t page_id{space->id, offset};
+    const ulint fold= page_id.fold();
+    mysql_mutex_lock(&buf_pool.mutex);
+    block= reinterpret_cast<buf_block_t*>
+      (buf_pool.page_hash_get_low(page_id, fold));
+    if (block && block->page.oldest_modification() <= 1)
+      block= nullptr;
+    mysql_mutex_unlock(&buf_pool.mutex);
+
+    if (block)
+    {
+      ut_ad(block->page.buf_fix_count() >= 1);
+      ut_ad(rw_lock_get_x_lock_count(&block->lock) == 1);
+      ut_ad(mtr->have_x_latch(*block));
+      free_block= block;
+      goto got_free_block;
+    }
+  }
+
+  free_block= buf_LRU_get_free_block(false);
+got_free_block:
+  block= buf_page_create(space, static_cast<uint32_t>(offset),
+                         space->zip_size(), mtr, free_block);
   if (UNIV_UNLIKELY(block != free_block))
     buf_pool.free_block(free_block);
+
   fsp_init_file_page(space, block, mtr);
   return block;
 }
@@ -1728,7 +1753,10 @@ fseg_create(fil_space_t *space, ulint byte_offset, mtr_t *mtr,
 			goto funct_exit;
 		}
 
-		ut_ad(rw_lock_get_x_lock_count(&block->lock) == 1);
+		ut_d(const auto x = rw_lock_get_x_lock_count(&block->lock));
+		ut_ad(x > 0);
+		ut_ad(x == 1 || space->is_being_truncated);
+		ut_ad(x <= 2);
 		ut_ad(!fil_page_get_type(block->frame));
 		mtr->write<1>(*block, FIL_PAGE_TYPE + 1 + block->frame,
 			      FIL_PAGE_TYPE_SYS);
diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h
index d1928196989..e9cd1f9a205 100644
--- a/storage/innobase/include/buf0buf.h
+++ b/storage/innobase/include/buf0buf.h
@@ -2232,9 +2232,7 @@ inline void buf_page_t::clear_oldest_modification()
 it from buf_pool.flush_list */
 inline void buf_page_t::clear_oldest_modification(bool temporary)
 {
-  mysql_mutex_assert_not_owner(&buf_pool.flush_list_mutex);
   ut_ad(temporary == fsp_is_system_temporary(id().space()));
-  ut_ad(io_fix_ == BUF_IO_WRITE);
   if (temporary)
   {
     ut_ad(oldest_modification() == 2);
diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc
index cf1574a56c4..37a75ce4c94 100644
--- a/storage/innobase/mtr/mtr0mtr.cc
+++ b/storage/innobase/mtr/mtr0mtr.cc
@@ -459,15 +459,15 @@ void mtr_t::commit()
   release_resources();
 }
 
-#ifdef UNIV_DEBUG
-/** Check that all pages belong to a shrunk tablespace. */
+/** Shrink a tablespace. */
 struct Shrink
 {
-  const page_id_t low, high;
-  Shrink(const fil_space_t &space) :
-    low({space.id, 0}), high({space.id, space.size}) {}
+  /** the first non-existing page in the tablespace */
+  const page_id_t high;
 
-  bool operator()(const mtr_memo_slot_t *slot) const
+  Shrink(const fil_space_t &space) : high({space.id, space.size}) {}
+
+  bool operator()(mtr_memo_slot_t *slot) const
   {
     if (!slot->object)
       return true;
@@ -476,29 +476,31 @@ struct Shrink
       ut_ad("invalid type" == 0);
       return false;
     case MTR_MEMO_SPACE_X_LOCK:
-      ut_ad(low.space() == static_cast<fil_space_t*>(slot->object)->id);
+      ut_ad(high.space() == static_cast<fil_space_t*>(slot->object)->id);
       return true;
     case MTR_MEMO_PAGE_X_MODIFY:
     case MTR_MEMO_PAGE_SX_MODIFY:
     case MTR_MEMO_PAGE_X_FIX:
     case MTR_MEMO_PAGE_SX_FIX:
-      const auto &bpage= static_cast<buf_block_t*>(slot->object)->page;
+      auto &bpage= static_cast<buf_block_t*>(slot->object)->page;
+      ut_ad(bpage.io_fix() == BUF_IO_NONE);
       const auto id= bpage.id();
-      if (id == page_id_t{0, TRX_SYS_PAGE_NO})
+      if (id < high)
       {
-        ut_ad(srv_is_undo_tablespace(low.space()));
+        ut_ad(id.space() == high.space() ||
+              (id == page_id_t{0, TRX_SYS_PAGE_NO} &&
+               srv_is_undo_tablespace(high.space())));
         break;
       }
-      ut_ad(id >= low);
-      ut_ad(id < high);
+      ut_ad(id.space() == high.space());
       ut_ad(bpage.state() == BUF_BLOCK_FILE_PAGE);
-      ut_ad(bpage.oldest_modification() <= 1);
-      break;
+      if (bpage.oldest_modification() > 1)
+        bpage.clear_oldest_modification(false);
+      slot->type= static_cast<mtr_memo_type_t>(slot->type & ~MTR_MEMO_MODIFY);
     }
     return true;
   }
 };
-#endif
 
 /** Commit a mini-transaction that is shrinking a tablespace.
 @param space   tablespace that is being shrunk */
@@ -542,7 +544,7 @@ void mtr_t::commit_shrink(fil_space_t &space)
   else
     ut_ad(!m_freed_space);
 
-  ut_d(m_memo.for_each_block_in_reverse(CIterate<Shrink>{space}));
+  m_memo.for_each_block_in_reverse(CIterate<Shrink>{space});
 
   m_memo.for_each_block_in_reverse(CIterate<const ReleaseBlocks>
                                    (ReleaseBlocks(start_lsn, m_commit_lsn,
diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc
index e417d1c5f9f..3e03f3b5244 100644
--- a/storage/innobase/trx/trx0purge.cc
+++ b/storage/innobase/trx/trx0purge.cc
@@ -546,253 +546,280 @@ function is called, the caller must not have any latches on undo log pages!
 */
 static void trx_purge_truncate_history()
 {
-	ut_ad(purge_sys.head <= purge_sys.tail);
-	purge_sys_t::iterator& head = purge_sys.head.trx_no
-		? purge_sys.head : purge_sys.tail;
-
-	if (head.trx_no >= purge_sys.low_limit_no()) {
-		/* This is sometimes necessary. TODO: find out why. */
-		head.trx_no = purge_sys.low_limit_no();
-		head.undo_no = 0;
-	}
-
-	for (ulint i = 0; i < TRX_SYS_N_RSEGS; ++i) {
-		if (trx_rseg_t* rseg = trx_sys.rseg_array[i]) {
-			ut_ad(rseg->id == i);
-			trx_purge_truncate_rseg_history(*rseg, head);
-		}
-	}
-
-	if (srv_undo_tablespaces_active < 2) {
-		return;
-	}
-
-	while (srv_undo_log_truncate) {
-		if (!purge_sys.truncate.current) {
-			const ulint threshold = ulint(srv_max_undo_log_size
-						      >> srv_page_size_shift);
-			for (ulint i = purge_sys.truncate.last
-				     ? purge_sys.truncate.last->id
-				     - srv_undo_space_id_start
-				     : 0, j = i;; ) {
-				ulint space_id = srv_undo_space_id_start + i;
-				ut_ad(srv_is_undo_tablespace(space_id));
-				fil_space_t* space= fil_space_get(space_id);
-
-				if (space && space->get_size() > threshold) {
-					purge_sys.truncate.current = space;
-					break;
-				}
-
-				++i;
-				i %= srv_undo_tablespaces_active;
-				if (i == j) {
-					break;
-				}
-			}
-		}
-
-		if (!purge_sys.truncate.current) {
-			return;
-		}
-
-		fil_space_t& space = *purge_sys.truncate.current;
-		/* Undo tablespace always are a single file. */
-		ut_a(UT_LIST_GET_LEN(space.chain) == 1);
-		fil_node_t* file = UT_LIST_GET_FIRST(space.chain);
-		/* The undo tablespace files are never closed. */
-		ut_ad(file->is_open());
-
-		DBUG_LOG("undo", "marking for truncate: " << file->name);
-
-		for (ulint i = 0; i < TRX_SYS_N_RSEGS; ++i) {
-			if (trx_rseg_t* rseg = trx_sys.rseg_array[i]) {
-				ut_ad(rseg->is_persistent());
-				if (rseg->space == &space) {
-					/* Once set, this rseg will
-					not be allocated to subsequent
-					transactions, but we will wait
-					for existing active
-					transactions to finish. */
-					rseg->skip_allocation = true;
-				}
-			}
-		}
+  ut_ad(purge_sys.head <= purge_sys.tail);
+  purge_sys_t::iterator &head= purge_sys.head.trx_no
+    ? purge_sys.head : purge_sys.tail;
+
+  if (head.trx_no >= purge_sys.low_limit_no())
+  {
+    /* This is sometimes necessary. TODO: find out why. */
+    head.trx_no= purge_sys.low_limit_no();
+    head.undo_no= 0;
+  }
+
+  for (ulint i= 0; i < TRX_SYS_N_RSEGS; ++i)
+  {
+    if (trx_rseg_t *rseg= trx_sys.rseg_array[i])
+    {
+      ut_ad(rseg->id == i);
+      trx_purge_truncate_rseg_history(*rseg, head);
+    }
+  }
+
+  if (srv_undo_tablespaces_active < 2)
+    return;
 
-		for (ulint i = 0; i < TRX_SYS_N_RSEGS; ++i) {
-			trx_rseg_t*	rseg = trx_sys.rseg_array[i];
-			if (!rseg || rseg->space != &space) {
-				continue;
-			}
-			mutex_enter(&rseg->mutex);
-			ut_ad(rseg->skip_allocation);
-			if (rseg->trx_ref_count) {
+  while (srv_undo_log_truncate)
+  {
+    if (!purge_sys.truncate.current)
+    {
+      const ulint threshold=
+        ulint(srv_max_undo_log_size >> srv_page_size_shift);
+      for (ulint i= purge_sys.truncate.last
+           ? purge_sys.truncate.last->id - srv_undo_space_id_start : 0,
+           j= i;; )
+      {
+        const auto space_id= srv_undo_space_id_start + i;
+        ut_ad(srv_is_undo_tablespace(space_id));
+        fil_space_t *space= fil_space_get(space_id);
+        ut_a(UT_LIST_GET_LEN(space->chain) == 1);
+
+        if (space && space->get_size() > threshold)
+        {
+          purge_sys.truncate.current= space;
+          break;
+        }
+
+        ++i;
+        i %= srv_undo_tablespaces_active;
+        if (i == j)
+          return;
+      }
+    }
+
+    fil_space_t &space= *purge_sys.truncate.current;
+    /* Undo tablespace always are a single file. */
+    fil_node_t *file= UT_LIST_GET_FIRST(space.chain);
+    /* The undo tablespace files are never closed. */
+    ut_ad(file->is_open());
+
+    DBUG_LOG("undo", "marking for truncate: " << file->name);
+
+    for (ulint i= 0; i < TRX_SYS_N_RSEGS; ++i)
+      if (trx_rseg_t *rseg= trx_sys.rseg_array[i])
+        if (rseg->space == &space)
+          /* Once set, this rseg will not be allocated to subsequent
+          transactions, but we will wait for existing active
+          transactions to finish. */
+          rseg->skip_allocation= true;
+
+    for (ulint i= 0; i < TRX_SYS_N_RSEGS; ++i)
+    {
+      trx_rseg_t *rseg= trx_sys.rseg_array[i];
+      if (!rseg || rseg->space != &space)
+        continue;
+      mutex_enter(&rseg->mutex);
+      ut_ad(rseg->skip_allocation);
+      ut_ad(rseg->is_persistent());
+      if (rseg->trx_ref_count)
+      {
 not_free:
-				mutex_exit(&rseg->mutex);
-				return;
-			}
-
-			if (rseg->curr_size != 1) {
-				/* Check if all segments are
-				cached and safe to remove. */
-				ulint cached = 0;
-
-				for (trx_undo_t* undo = UT_LIST_GET_FIRST(
-					     rseg->undo_cached);
-				     undo;
-				     undo = UT_LIST_GET_NEXT(undo_list,
-							     undo)) {
-					if (head.trx_no < undo->trx_id) {
-						goto not_free;
-					} else {
-						cached += undo->size;
-					}
-				}
-
-				ut_ad(rseg->curr_size > cached);
-
-				if (rseg->curr_size > cached + 1) {
-					goto not_free;
-				}
-			}
-
-			mutex_exit(&rseg->mutex);
-		}
-
-		ib::info() << "Truncating " << file->name;
-		trx_purge_cleanse_purge_queue(space);
-
-		/* Flush all to-be-discarded pages of the tablespace.
-
-		During truncation, we do not want any writes to the
-		to-be-discarded area, because we must set the space.size
-		early in order to have deterministic page allocation.
-
-		If a log checkpoint was completed at LSN earlier than our
-		mini-transaction commit and the server was killed, then
-		discarding the to-be-trimmed pages without flushing would
-		break crash recovery. So, we cannot avoid the write. */
-		while (buf_flush_list_space(&space));
-
-		log_free_check();
-
-		/* Adjust the tablespace metadata. */
-		if (!fil_truncate_prepare(space.id)) {
-			ib::error() << "Failed to find UNDO tablespace "
-				<< file->name;
-			return;
-		}
-
-		/* Re-initialize tablespace, in a single mini-transaction. */
-		mtr_t mtr;
-		const ulint size = SRV_UNDO_TABLESPACE_SIZE_IN_PAGES;
-		mtr.start();
-		mtr_x_lock_space(purge_sys.truncate.current, &mtr);
-		/* Associate the undo tablespace with mtr.
-		During mtr::commit(), InnoDB can use the undo
-		tablespace object to clear all freed ranges */
-		mtr.set_named_space(purge_sys.truncate.current);
-		mtr.trim_pages(page_id_t(space.id, size));
-		fsp_header_init(purge_sys.truncate.current, size, &mtr);
-		mutex_enter(&fil_system.mutex);
-		purge_sys.truncate.current->size = file->size = size;
-		mutex_exit(&fil_system.mutex);
-
-		buf_block_t* sys_header = trx_sysf_get(&mtr);
-
-		for (ulint i = 0; i < TRX_SYS_N_RSEGS; ++i) {
-			trx_rseg_t* rseg = trx_sys.rseg_array[i];
-			if (!rseg || rseg->space != &space) {
-				continue;
-			}
-
-			ut_ad(rseg->is_persistent());
-			ut_d(const ulint old_page = rseg->page_no);
-
-			buf_block_t* rblock = trx_rseg_header_create(
-				purge_sys.truncate.current,
-				rseg->id, sys_header, &mtr);
-			ut_ad(rblock);
-			rseg->page_no = rblock
-				? rblock->page.id().page_no() : FIL_NULL;
-			ut_ad(old_page == rseg->page_no);
-
-			/* Before re-initialization ensure that we
-			free the existing structure. There can't be
-			any active transactions. */
-			ut_a(UT_LIST_GET_LEN(rseg->undo_list) == 0);
-
-			trx_undo_t*	next_undo;
-
-			for (trx_undo_t* undo = UT_LIST_GET_FIRST(
-				     rseg->undo_cached);
-			     undo; undo = next_undo) {
-
-				next_undo = UT_LIST_GET_NEXT(undo_list, undo);
-				UT_LIST_REMOVE(rseg->undo_cached, undo);
-				MONITOR_DEC(MONITOR_NUM_UNDO_SLOT_CACHED);
-				ut_free(undo);
-			}
-
-			UT_LIST_INIT(rseg->undo_list,
-				     &trx_undo_t::undo_list);
-			UT_LIST_INIT(rseg->undo_cached,
-				     &trx_undo_t::undo_list);
-
-			/* These were written by trx_rseg_header_create(). */
-			ut_ad(!mach_read_from_4(TRX_RSEG + TRX_RSEG_FORMAT
-						+ rblock->frame));
-			ut_ad(!mach_read_from_4(TRX_RSEG + TRX_RSEG_HISTORY_SIZE
-						+ rblock->frame));
-
-			/* Initialize the undo log lists according to
-			the rseg header */
-			rseg->curr_size = 1;
-			rseg->trx_ref_count = 0;
-			rseg->last_page_no = FIL_NULL;
-			rseg->last_commit_and_offset = 0;
-			rseg->needs_purge = false;
-		}
-
-		mtr.commit_shrink(space);
-
-		/* No mutex; this is only updated by the purge coordinator. */
-		export_vars.innodb_undo_truncations++;
-
-		if (purge_sys.rseg != NULL
-		    && purge_sys.rseg->last_page_no == FIL_NULL) {
-			/* If purge_sys.rseg is pointing to rseg that
-			was recently truncated then move to next rseg
-			element.  Note: Ideally purge_sys.rseg should
-			be NULL because purge should complete
-			processing of all the records but there is
-			purge_batch_size that can force the purge loop
-			to exit before all the records are purged and
-			in this case purge_sys.rseg could point to a
-			valid rseg waiting for next purge cycle. */
-			purge_sys.next_stored = false;
-			purge_sys.rseg = NULL;
-		}
-
-		DBUG_EXECUTE_IF("ib_undo_trunc",
-				ib::info() << "ib_undo_trunc";
-				log_buffer_flush_to_disk();
-				DBUG_SUICIDE(););
-
-		for (ulint i = 0; i < TRX_SYS_N_RSEGS; ++i) {
-			if (trx_rseg_t* rseg = trx_sys.rseg_array[i]) {
-				ut_ad(rseg->is_persistent());
-				if (rseg->space == &space) {
-					rseg->skip_allocation = false;
-				}
-			}
-		}
-
-		ib::info() << "Truncated " << file->name;
-		purge_sys.truncate.last = purge_sys.truncate.current;
-		ut_ad(&space == purge_sys.truncate.current);
-		purge_sys.truncate.current = NULL;
-	}
+        mutex_exit(&rseg->mutex);
+        return;
+      }
+
+      if (rseg->curr_size != 1)
+      {
+        /* Check if all segments are cached and safe to remove. */
+        ulint cached= 0;
+        for (trx_undo_t *undo= UT_LIST_GET_FIRST(rseg->undo_cached); undo;
+             undo= UT_LIST_GET_NEXT(undo_list, undo))
+        {
+          if (head.trx_no < undo->trx_id)
+            goto not_free;
+          else
+            cached+= undo->size;
+        }
+
+        ut_ad(rseg->curr_size > cached);
+
+        if (rseg->curr_size > cached + 1)
+          goto not_free;
+      }
+
+      mutex_exit(&rseg->mutex);
+    }
+
+    ib::info() << "Truncating " << file->name;
+    trx_purge_cleanse_purge_queue(space);
+
+    log_free_check();
+
+    mtr_t mtr;
+    mtr.start();
+    mtr_x_lock_space(&space, &mtr);
+
+    /* Lock all modified pages of the tablespace.
+
+    During truncation, we do not want any writes to the file.
+
+    If a log checkpoint was completed at LSN earlier than our
+    mini-transaction commit and the server was killed, then
+    discarding the to-be-trimmed pages without flushing would
+    break crash recovery. */
+    mysql_mutex_lock(&buf_pool.flush_list_mutex);
+
+    for (buf_page_t *bpage= UT_LIST_GET_LAST(buf_pool.flush_list); bpage; )
+    {
+      ut_ad(bpage->oldest_modification());
+      ut_ad(bpage->in_file());
+
+      buf_page_t *prev= UT_LIST_GET_PREV(list, bpage);
+
+      if (bpage->id().space() == space.id &&
+          bpage->oldest_modification() != 1)
+      {
+        ut_ad(bpage->state() == BUF_BLOCK_FILE_PAGE);
+        auto block= reinterpret_cast<buf_block_t*>(bpage);
+        block->fix();
+        ut_ad(rw_lock_s_lock_nowait(block->debug_latch, __FILE__, __LINE__));
+        buf_pool.flush_hp.set(prev);
+        mysql_mutex_unlock(&buf_pool.flush_list_mutex);
+
+#ifdef BTR_CUR_HASH_ADAPT
+        ut_ad(!block->index); /* There is no AHI on undo tablespaces. */
+#endif
+        rw_lock_x_lock(&block->lock);
+        mysql_mutex_lock(&buf_pool.flush_list_mutex);
+        ut_ad(bpage->io_fix() == BUF_IO_NONE);
+
+        if (bpage->oldest_modification() > 1)
+        {
+          bpage->clear_oldest_modification(false);
+          mtr.memo_push(block, MTR_MEMO_PAGE_X_FIX);
+        }
+        else
+        {
+          rw_lock_x_unlock(&block->lock);
+          block->unfix();
+        }
+
+        if (prev != buf_pool.flush_hp.get())
+        {
+          /* Rescan, because we may have lost the position. */
+          bpage= UT_LIST_GET_LAST(buf_pool.flush_list);
+          continue;
+        }
+      }
+
+      bpage= prev;
+    }
+
+    mysql_mutex_unlock(&buf_pool.flush_list_mutex);
+
+    /* Adjust the tablespace metadata. */
+    if (!fil_truncate_prepare(space.id))
+    {
+      ib::error() << "Failed to find UNDO tablespace " << file->name;
+      mtr.commit();
+      return;
+    }
+
+    /* Re-initialize tablespace, in a single mini-transaction. */
+    const ulint size= SRV_UNDO_TABLESPACE_SIZE_IN_PAGES;
+    /* Associate the undo tablespace with mtr.
+    During mtr::commit_shrink(), InnoDB can use the undo
+    tablespace object to clear all freed ranges */
+    mtr.set_named_space(&space);
+    mtr.trim_pages(page_id_t(space.id, size));
+    fsp_header_init(&space, size, &mtr);
+    mutex_enter(&fil_system.mutex);
+    space.size= file->size= size;
+    mutex_exit(&fil_system.mutex);
+
+    buf_block_t *sys_header= trx_sysf_get(&mtr);
+
+    for (ulint i= 0; i < TRX_SYS_N_RSEGS; ++i)
+    {
+      trx_rseg_t *rseg= trx_sys.rseg_array[i];
+      if (!rseg || rseg->space != &space)
+        continue;
+
+      ut_ad(rseg->id == i);
+      ut_ad(rseg->is_persistent());
+      ut_d(const auto old_page= rseg->page_no);
+
+      buf_block_t *rblock= trx_rseg_header_create(&space, i, sys_header, &mtr);
+      ut_ad(rblock);
+      rseg->page_no= rblock ? rblock->page.id().page_no() : FIL_NULL;
+      ut_ad(old_page == rseg->page_no);
+
+      /* Before re-initialization ensure that we free the existing
+      structure. There can't be any active transactions. */
+      ut_a(UT_LIST_GET_LEN(rseg->undo_list) == 0);
+
+      for (trx_undo_t *undo= UT_LIST_GET_FIRST(rseg->undo_cached), *next_undo;
+           undo; undo= next_undo)
+      {
+        next_undo= UT_LIST_GET_NEXT(undo_list, undo);
+        UT_LIST_REMOVE(rseg->undo_cached, undo);
+        MONITOR_DEC(MONITOR_NUM_UNDO_SLOT_CACHED);
+        ut_free(undo);
+      }
+
+      UT_LIST_INIT(rseg->undo_list, &trx_undo_t::undo_list);
+      UT_LIST_INIT(rseg->undo_cached, &trx_undo_t::undo_list);
+
+      /* These were written by trx_rseg_header_create(). */
+      ut_ad(!mach_read_from_4(TRX_RSEG + TRX_RSEG_FORMAT + rblock->frame));
+      ut_ad(!mach_read_from_4(TRX_RSEG + TRX_RSEG_HISTORY_SIZE +
+                              rblock->frame));
+      /* Initialize the undo log lists according to
+      the rseg header */
+      rseg->curr_size= 1;
+      rseg->trx_ref_count= 0;
+      rseg->last_page_no= FIL_NULL;
+      rseg->last_commit_and_offset= 0;
+      rseg->needs_purge= false;
+    }
+
+    mtr.commit_shrink(space);
+
+    /* No mutex; this is only updated by the purge coordinator. */
+    export_vars.innodb_undo_truncations++;
+
+    if (purge_sys.rseg && purge_sys.rseg->last_page_no == FIL_NULL)
+    {
+      /* If purge_sys.rseg is pointing to rseg that was recently
+      truncated then move to next rseg element.
+
+      Note: Ideally purge_sys.rseg should be NULL because purge should
+      complete processing of all the records but srv_purge_batch_size
+      can force the purge loop to exit before all the records are purged. */
+      purge_sys.rseg= nullptr;
+      purge_sys.next_stored= false;
+    }
+
+    DBUG_EXECUTE_IF("ib_undo_trunc", ib::info() << "ib_undo_trunc";
+                    log_buffer_flush_to_disk();
+                    DBUG_SUICIDE(););
+
+    for (ulint i= 0; i < TRX_SYS_N_RSEGS; ++i)
+    {
+      if (trx_rseg_t *rseg= trx_sys.rseg_array[i])
+      {
+        ut_ad(rseg->id == i);
+        ut_ad(rseg->is_persistent());
+        if (rseg->space == &space)
+          rseg->skip_allocation= false;
+      }
+    }
+
+    ib::info() << "Truncated " << file->name;
+    purge_sys.truncate.last= purge_sys.truncate.current;
+    ut_ad(&space == purge_sys.truncate.current);
+    purge_sys.truncate.current= nullptr;
+  }
 }
 
 /***********************************************************************//**
-- 
cgit v1.2.1


From 37a074f6c35af22f403b3e1d150c17e8ab07f164 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Fri, 24 Sep 2021 09:18:07 +0300
Subject: MDEV-26467 fixup: Fix cmake -DWITH_UNIT_TESTS=ON for SUX_LOCK_GENERIC

---
 storage/innobase/sync/srw_lock.cc | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/storage/innobase/sync/srw_lock.cc b/storage/innobase/sync/srw_lock.cc
index 4cd4b56bdd0..f3c0afd64e3 100644
--- a/storage/innobase/sync/srw_lock.cc
+++ b/storage/innobase/sync/srw_lock.cc
@@ -223,6 +223,14 @@ template void ssux_lock_impl<false>::destroy();
 template void ssux_lock_impl<false>::rd_unlock();
 template void ssux_lock_impl<false>::u_unlock();
 template void ssux_lock_impl<false>::wr_unlock();
+template void ssux_lock_impl<true>::init();
+template void ssux_lock_impl<true>::destroy();
+template void ssux_lock_impl<true>::read_lock(uint32_t);
+template void ssux_lock_impl<true>::rd_unlock();
+template void ssux_lock_impl<true>::u_unlock();
+template void ssux_lock_impl<true>::wr_unlock();
+template void ssux_lock_impl<true>::write_lock(bool);
+template void ssux_lock_impl<true>::update_lock(uint32_t);
 #else /* SUX_LOCK_GENERIC */
 static_assert(4 == sizeof(rw_lock), "ABI");
 # ifdef _WIN32
-- 
cgit v1.2.1


From 4bfdba2e89ce825a9f89b2e6ed7eca3e0ce2c128 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= <marko.makela@mariadb.com>
Date: Fri, 24 Sep 2021 11:23:37 +0300
Subject: MDEV-26672 innodb_undo_log_truncate may reset transaction ID sequence

trx_rseg_header_create(): Add a parameter for the value that is
to be written to TRX_RSEG_MAX_TRX_ID. If we omit this write, then
the updated test innodb.undo_truncate will fail for the 4k, 8k, 16k
page sizes. This was broken ever since
commit 947efe17ed8188ca4feef6deb0c2831a246b5c8f (MDEV-15158)
removed the writes of transaction identifiers to the TRX_SYS page.

srv_do_purge(): Truncate undo tablespaces also during slow shutdown
(innodb_fast_shutdown=0).

Thanks to Krunal Bauskar for noticing this problem.
---
 mysql-test/suite/innodb/r/undo_truncate.result |  9 +-----
 mysql-test/suite/innodb/t/undo_truncate.test   | 39 ++++++++++----------------
 storage/innobase/include/trx0rseg.h            |  2 ++
 storage/innobase/srv/srv0srv.cc                |  2 ++
 storage/innobase/srv/srv0start.cc              |  5 +++-
 storage/innobase/trx/trx0purge.cc              |  3 +-
 storage/innobase/trx/trx0rseg.cc               | 12 ++++++--
 storage/innobase/trx/trx0sys.cc                |  2 +-
 8 files changed, 37 insertions(+), 37 deletions(-)

diff --git a/mysql-test/suite/innodb/r/undo_truncate.result b/mysql-test/suite/innodb/r/undo_truncate.result
index 25fd07cf930..4b185a58d2c 100644
--- a/mysql-test/suite/innodb/r/undo_truncate.result
+++ b/mysql-test/suite/innodb/r/undo_truncate.result
@@ -2,11 +2,7 @@ SET @save_undo_logs = @@GLOBAL.innodb_undo_logs;
 SET @save_frequency = @@GLOBAL.innodb_purge_rseg_truncate_frequency;
 SET @save_truncate = @@GLOBAL.innodb_undo_log_truncate;
 SET GLOBAL innodb_undo_log_truncate = 0;
-SET GLOBAL innodb_undo_logs = 4;
 SET GLOBAL innodb_purge_rseg_truncate_frequency = 1;
-SET @trunc_start=
-(SELECT variable_value FROM information_schema.global_status
-WHERE variable_name = 'innodb_undo_truncations');
 create table t1(keyc int primary key, c char(100)) engine = innodb;
 create table t2(keyc int primary key, c char(100)) engine = innodb;
 connect  con1,localhost,root,,;
@@ -35,8 +31,5 @@ connection con2;
 commit;
 disconnect con2;
 connection default;
+set global innodb_fast_shutdown=0;
 drop table t1, t2;
-InnoDB		0 transactions not purged
-SET GLOBAL innodb_undo_logs = @save_undo_logs;
-SET GLOBAL innodb_purge_rseg_truncate_frequency = @save_frequency;
-SET GLOBAL innodb_undo_log_truncate = @save_truncate;
diff --git a/mysql-test/suite/innodb/t/undo_truncate.test b/mysql-test/suite/innodb/t/undo_truncate.test
index bc2e6a3a119..32697e59c91 100644
--- a/mysql-test/suite/innodb/t/undo_truncate.test
+++ b/mysql-test/suite/innodb/t/undo_truncate.test
@@ -1,23 +1,15 @@
 --source include/have_innodb.inc
-# With 32k, truncation could happen on shutdown after the test,
-# and the mtr.add_suppression() would not filter out the warning.
-# With 64k, no truncation seems to happen.
-# --source include/innodb_page_size.inc
---source include/innodb_page_size_small.inc
+--source include/innodb_page_size.inc
 --source include/have_undo_tablespaces.inc
+--source include/not_embedded.inc
 --source include/have_sequence.inc
 
 SET @save_undo_logs = @@GLOBAL.innodb_undo_logs;
 SET @save_frequency = @@GLOBAL.innodb_purge_rseg_truncate_frequency;
 SET @save_truncate = @@GLOBAL.innodb_undo_log_truncate;
 SET GLOBAL innodb_undo_log_truncate = 0;
-SET GLOBAL innodb_undo_logs = 4;
 SET GLOBAL innodb_purge_rseg_truncate_frequency = 1;
 
-SET @trunc_start=
-(SELECT variable_value FROM information_schema.global_status
-WHERE variable_name = 'innodb_undo_truncations');
-
 #-----------------------------------------------------------------------------
 #
 # Perform DML action using multiple clients and multiple undo tablespace.
@@ -48,21 +40,20 @@ commit; disconnect con1;
 connection con2; commit; disconnect con2;
 
 connection default;
-drop table t1, t2;
 
---source include/wait_all_purged.inc
+--replace_regex /.*Trx id counter ([0-9]+).*/\1/
+let $trx_before= `SHOW ENGINE INNODB STATUS`;
+let $trx_before= `select substr('$trx_before',9)+2`;
+
+set global innodb_fast_shutdown=0;
+--source include/restart_mysqld.inc
+--replace_regex /.*Trx id counter ([0-9]+).*/\1/
+let $trx_after= `SHOW ENGINE INNODB STATUS`;
+let $trx_after= `select substr('$trx_after',9)`;
 
-# Truncation will normally not occur with innodb_page_size=64k,
-# and occasionally not with innodb_page_size=32k,
-# because the undo log will not grow enough.
-if (`select @@innodb_page_size IN (4096,8192,16384)`)
+drop table t1, t2;
+
+if ($trx_before != $trx_after)
 {
-  let $wait_condition = (SELECT variable_value!=@trunc_start
-                         FROM information_schema.global_status
-                         WHERE variable_name = 'innodb_undo_truncations');
-  source include/wait_condition.inc;
+  echo Transaction sequence mismatch: $trx_before != $trx_after;
 }
-
-SET GLOBAL innodb_undo_logs = @save_undo_logs;
-SET GLOBAL innodb_purge_rseg_truncate_frequency = @save_frequency;
-SET GLOBAL innodb_undo_log_truncate = @save_truncate;
diff --git a/storage/innobase/include/trx0rseg.h b/storage/innobase/include/trx0rseg.h
index 687c6fa0e97..27548e96b0a 100644
--- a/storage/innobase/include/trx0rseg.h
+++ b/storage/innobase/include/trx0rseg.h
@@ -71,6 +71,7 @@ trx_rsegf_undo_find_free(const trx_rsegf_t* rsegf);
 /** Create a rollback segment header.
 @param[in,out]	space		system, undo, or temporary tablespace
 @param[in]	rseg_id		rollback segment identifier
+@param[in]	max_trx_id	new value of TRX_RSEG_MAX_TRX_ID
 @param[in,out]	sys_header	the TRX_SYS page (NULL for temporary rseg)
 @param[in,out]	mtr		mini-transaction
 @return the created rollback segment
@@ -79,6 +80,7 @@ buf_block_t*
 trx_rseg_header_create(
 	fil_space_t*	space,
 	ulint		rseg_id,
+	trx_id_t	max_trx_id,
 	buf_block_t*	sys_header,
 	mtr_t*		mtr);
 
diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc
index 9d19a14cb66..e283c594456 100644
--- a/storage/innobase/srv/srv0srv.cc
+++ b/storage/innobase/srv/srv0srv.cc
@@ -2640,6 +2640,8 @@ static ulint srv_do_purge(ulint* n_total_purged
 		n_pages_purged = trx_purge(
 			n_use_threads,
 			(++count % rseg_truncate_frequency) == 0
+			|| (srv_shutdown_state != SRV_SHUTDOWN_NONE
+			    && srv_fast_shutdown == 0)
 #ifdef UNIV_DEBUG
 			, slot
 #endif
diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc
index 7d5679e889e..8b72a6c0aaf 100644
--- a/storage/innobase/srv/srv0start.cc
+++ b/storage/innobase/srv/srv0start.cc
@@ -1029,6 +1029,8 @@ srv_undo_tablespaces_init(bool create_new_db)
 			return DB_CORRUPTION;
 		}
 
+		const trx_id_t max_trx_id = trx_sys.get_max_trx_id();
+
 		for (undo::undo_spaces_t::const_iterator it
 			     = undo::Truncate::s_fix_up_spaces.begin();
 		     it != undo::Truncate::s_fix_up_spaces.end();
@@ -1046,7 +1048,8 @@ srv_undo_tablespaces_init(bool create_new_db)
 				if (trx_sysf_rseg_get_space(sys_header, i)
 				    == *it) {
 					trx_rseg_header_create(
-						space, i, sys_header, &mtr);
+						space, i, max_trx_id,
+						sys_header, &mtr);
 				}
 			}
 
diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc
index d300e3b54e3..208f400c8a3 100644
--- a/storage/innobase/trx/trx0purge.cc
+++ b/storage/innobase/trx/trx0purge.cc
@@ -977,7 +977,8 @@ not_found:
 	for (ulint i = 0; i < undo_trunc->rsegs_size(); ++i) {
 		trx_rseg_t*	rseg = undo_trunc->get_ith_rseg(i);
 		buf_block_t* rblock = trx_rseg_header_create(
-			space, rseg->id, sys_header, &mtr);
+			space, rseg->id, trx_sys.get_max_trx_id(),
+			sys_header, &mtr);
 		ut_ad(rblock);
 		rseg->page_no = rblock ? rblock->page.id.page_no() : FIL_NULL;
 
diff --git a/storage/innobase/trx/trx0rseg.cc b/storage/innobase/trx/trx0rseg.cc
index c5ab4d37aae..0ae11390f0b 100644
--- a/storage/innobase/trx/trx0rseg.cc
+++ b/storage/innobase/trx/trx0rseg.cc
@@ -286,6 +286,7 @@ void trx_rseg_format_upgrade(trx_rsegf_t* rseg_header, mtr_t* mtr)
 /** Create a rollback segment header.
 @param[in,out]	space		system, undo, or temporary tablespace
 @param[in]	rseg_id		rollback segment identifier
+@param[in]	max_trx_id	new value of TRX_RSEG_MAX_TRX_ID
 @param[in,out]	sys_header	the TRX_SYS page (NULL for temporary rseg)
 @param[in,out]	mtr		mini-transaction
 @return the created rollback segment
@@ -294,6 +295,7 @@ buf_block_t*
 trx_rseg_header_create(
 	fil_space_t*	space,
 	ulint		rseg_id,
+	trx_id_t	max_trx_id,
 	buf_block_t*	sys_header,
 	mtr_t*		mtr)
 {
@@ -319,6 +321,12 @@ trx_rseg_header_create(
 
 	mlog_write_ulint(TRX_RSEG + TRX_RSEG_HISTORY_SIZE + block->frame, 0,
 			 MLOG_4BYTES, mtr);
+
+	if (max_trx_id) {
+		mlog_write_ull(TRX_RSEG + TRX_RSEG_MAX_TRX_ID + block->frame,
+			       max_trx_id, mtr);
+	}
+
 	flst_init(TRX_RSEG + TRX_RSEG_HISTORY + block->frame, mtr);
 	trx_rsegf_t* rsegf = TRX_RSEG + block->frame;
 
@@ -684,7 +692,7 @@ trx_rseg_create(ulint space_id)
 		ulint	rseg_id = trx_sys_rseg_find_free(sys_header);
 		if (buf_block_t* rblock = rseg_id == ULINT_UNDEFINED
 		    ? NULL
-		    : trx_rseg_header_create(space, rseg_id, sys_header,
+		    : trx_rseg_header_create(space, rseg_id, 0, sys_header,
 					     &mtr)) {
 			ut_ad(trx_sysf_rseg_get_space(sys_header, rseg_id)
 			      == space_id);
@@ -714,7 +722,7 @@ trx_temp_rseg_create()
 		mtr_x_lock_space(fil_system.temp_space, &mtr);
 
 		buf_block_t* rblock = trx_rseg_header_create(
-			fil_system.temp_space, i, NULL, &mtr);
+			fil_system.temp_space, i, 0, NULL, &mtr);
 		trx_rseg_t* rseg = trx_rseg_mem_create(
 			i, fil_system.temp_space, rblock->page.id.page_no());
 		ut_ad(!rseg->is_persistent());
diff --git a/storage/innobase/trx/trx0sys.cc b/storage/innobase/trx/trx0sys.cc
index 87814fa6c69..0b1f72412f6 100644
--- a/storage/innobase/trx/trx0sys.cc
+++ b/storage/innobase/trx/trx0sys.cc
@@ -197,7 +197,7 @@ trx_sysf_create(
 	/* Create the first rollback segment in the SYSTEM tablespace */
 	slot_no = trx_sys_rseg_find_free(block);
 	buf_block_t* rblock = trx_rseg_header_create(fil_system.sys_space,
-						     slot_no, block, mtr);
+						     slot_no, 0, block, mtr);
 
 	ut_a(slot_no == TRX_SYS_SYSTEM_RSEG_ID);
 	ut_a(rblock->page.id.page_no() == FSP_FIRST_RSEG_PAGE_NO);
-- 
cgit v1.2.1