diff options
27 files changed, 311 insertions, 575 deletions
diff --git a/extra/mariabackup/datasink.cc b/extra/mariabackup/datasink.cc index 29bdc061014..a576526d7ff 100644 --- a/extra/mariabackup/datasink.cc +++ b/extra/mariabackup/datasink.cc @@ -23,7 +23,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA #include "common.h" #include "datasink.h" #include "ds_compress.h" -#include "ds_archive.h" #include "ds_xbstream.h" #include "ds_local.h" #include "ds_stdout.h" @@ -45,13 +44,6 @@ ds_create(const char *root, ds_type_t type) case DS_TYPE_LOCAL: ds = &datasink_local; break; - case DS_TYPE_ARCHIVE: -#ifdef HAVE_LIBARCHIVE - ds = &datasink_archive; -#else - die("mariabackup was built without libarchive support"); -#endif - break; case DS_TYPE_XBSTREAM: ds = &datasink_xbstream; break; diff --git a/extra/mariabackup/datasink.h b/extra/mariabackup/datasink.h index 5c82556b9ba..4bede4ec9e7 100644 --- a/extra/mariabackup/datasink.h +++ b/extra/mariabackup/datasink.h @@ -63,7 +63,6 @@ static inline int dummy_remove(const char *) { typedef enum { DS_TYPE_STDOUT, DS_TYPE_LOCAL, - DS_TYPE_ARCHIVE, DS_TYPE_XBSTREAM, DS_TYPE_COMPRESS, DS_TYPE_ENCRYPT, diff --git a/extra/mariabackup/ds_archive.cc b/extra/mariabackup/ds_archive.cc deleted file mode 100644 index 3a5081119b3..00000000000 --- a/extra/mariabackup/ds_archive.cc +++ /dev/null @@ -1,282 +0,0 @@ -/****************************************************** -Copyright (c) 2013 Percona LLC and/or its affiliates. - -Streaming implementation for XtraBackup. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*******************************************************/ - -#include <my_global.h> -#include <my_base.h> -#include <archive.h> -#include <archive_entry.h> -#include "common.h" -#include "datasink.h" - -#if ARCHIVE_VERSION_NUMBER < 3000000 -#define archive_write_add_filter_none(X) archive_write_set_compression_none(X) -#define archive_write_free(X) archive_write_finish(X) -#endif - -typedef struct { - struct archive *archive; - ds_file_t *dest_file; - pthread_mutex_t mutex; -} ds_archive_ctxt_t; - -typedef struct { - struct archive_entry *entry; - ds_archive_ctxt_t *archive_ctxt; -} ds_archive_file_t; - - -/*********************************************************************** -General archive interface */ - -static ds_ctxt_t *archive_init(const char *root); -static ds_file_t *archive_open(ds_ctxt_t *ctxt, const char *path, - MY_STAT *mystat); -static int archive_write(ds_file_t *file, const void *buf, size_t len); -static int archive_close(ds_file_t *file); -static void archive_deinit(ds_ctxt_t *ctxt); - -datasink_t datasink_archive = { - &archive_init, - &archive_open, - &archive_write, - &archive_close, - &dummy_remove, - &archive_deinit -}; - -static -int -my_archive_open_callback(struct archive *a __attribute__((unused)), - void *data __attribute__((unused))) -{ - return ARCHIVE_OK; -} - -static -ssize_t -my_archive_write_callback(struct archive *a __attribute__((unused)), - void *data, const void *buffer, size_t length) -{ - ds_archive_ctxt_t *archive_ctxt; - - archive_ctxt = (ds_archive_ctxt_t *) data; - - xb_ad(archive_ctxt != NULL); - xb_ad(archive_ctxt->dest_file != NULL); - - if (!ds_write(archive_ctxt->dest_file, buffer, length)) { - return length; - } - return -1; -} - -static -int -my_archive_close_callback(struct archive *a __attribute__((unused)), - void *data __attribute__((unused))) -{ - return ARCHIVE_OK; -} - -static -ds_ctxt_t * -archive_init(const char *root __attribute__((unused))) -{ - ds_ctxt_t *ctxt; - ds_archive_ctxt_t *archive_ctxt; - struct archive *a; - - ctxt = my_malloc(sizeof(ds_ctxt_t) + sizeof(ds_archive_ctxt_t), - MYF(MY_FAE)); - archive_ctxt = (ds_archive_ctxt_t *)(ctxt + 1); - - if (pthread_mutex_init(&archive_ctxt->mutex, NULL)) { - msg("archive_init: pthread_mutex_init() failed.\n"); - goto err; - } - - a = archive_write_new(); - if (a == NULL) { - msg("archive_write_new() failed.\n"); - goto err; - } - - archive_ctxt->archive = a; - archive_ctxt->dest_file = NULL; - - if(archive_write_add_filter_none(a) != ARCHIVE_OK || - archive_write_set_format_pax_restricted(a) != ARCHIVE_OK || - /* disable internal buffering so we don't have to flush the - output in xtrabackup */ - archive_write_set_bytes_per_block(a, 0) != ARCHIVE_OK) { - msg("failed to set libarchive archive options: %s\n", - archive_error_string(a)); - archive_write_free(a); - goto err; - } - - if (archive_write_open(a, archive_ctxt, my_archive_open_callback, - my_archive_write_callback, - my_archive_close_callback) != ARCHIVE_OK) { - msg("cannot open output archive.\n"); - return NULL; - } - - ctxt->ptr = archive_ctxt; - - return ctxt; - -err: - my_free(ctxt); - return NULL; -} - -static -ds_file_t * -archive_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) -{ - ds_archive_ctxt_t *archive_ctxt; - ds_ctxt_t *dest_ctxt; - ds_file_t *file; - ds_archive_file_t *archive_file; - - struct archive *a; - struct archive_entry *entry; - - xb_ad(ctxt->pipe_ctxt != NULL); - dest_ctxt = ctxt->pipe_ctxt; - - archive_ctxt = (ds_archive_ctxt_t *) ctxt->ptr; - - pthread_mutex_lock(&archive_ctxt->mutex); - if (archive_ctxt->dest_file == NULL) { - archive_ctxt->dest_file = ds_open(dest_ctxt, path, mystat); - if (archive_ctxt->dest_file == NULL) { - return NULL; - } - } - pthread_mutex_unlock(&archive_ctxt->mutex); - - file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + - sizeof(ds_archive_file_t), - MYF(MY_FAE)); - - archive_file = (ds_archive_file_t *) (file + 1); - - a = archive_ctxt->archive; - - entry = archive_entry_new(); - if (entry == NULL) { - msg("archive_entry_new() failed.\n"); - goto err; - } - - archive_entry_set_size(entry, mystat->st_size); - archive_entry_set_mode(entry, 0660); - archive_entry_set_filetype(entry, AE_IFREG); - archive_entry_set_pathname(entry, path); - archive_entry_set_mtime(entry, mystat->st_mtime, 0); - - archive_file->entry = entry; - archive_file->archive_ctxt = archive_ctxt; - - if (archive_write_header(a, entry) != ARCHIVE_OK) { - msg("archive_write_header() failed.\n"); - archive_entry_free(entry); - goto err; - } - - file->ptr = archive_file; - file->path = archive_ctxt->dest_file->path; - - return file; - -err: - if (archive_ctxt->dest_file) { - ds_close(archive_ctxt->dest_file); - archive_ctxt->dest_file = NULL; - } - my_free(file); - - return NULL; -} - -static -int -archive_write(ds_file_t *file, const void *buf, size_t len) -{ - ds_archive_file_t *archive_file; - struct archive *a; - - archive_file = (ds_archive_file_t *) file->ptr; - - a = archive_file->archive_ctxt->archive; - - xb_ad(archive_file->archive_ctxt->dest_file != NULL); - if (archive_write_data(a, buf, len) < 0) { - msg("archive_write_data() failed: %s (errno = %d)\n", - archive_error_string(a), archive_errno(a)); - return 1; - } - - return 0; -} - -static -int -archive_close(ds_file_t *file) -{ - ds_archive_file_t *archive_file; - int rc = 0; - - archive_file = (ds_archive_file_t *)file->ptr; - - archive_entry_free(archive_file->entry); - - my_free(file); - - return rc; -} - -static -void -archive_deinit(ds_ctxt_t *ctxt) -{ - struct archive *a; - ds_archive_ctxt_t *archive_ctxt; - - archive_ctxt = (ds_archive_ctxt_t *) ctxt->ptr; - - a = archive_ctxt->archive; - - if (archive_write_close(a) != ARCHIVE_OK) { - msg("archive_write_close() failed.\n"); - } - archive_write_free(a); - - if (archive_ctxt->dest_file) { - ds_close(archive_ctxt->dest_file); - archive_ctxt->dest_file = NULL; - } - - pthread_mutex_destroy(&archive_ctxt->mutex); - - my_free(ctxt); -} diff --git a/extra/mariabackup/ds_archive.h b/extra/mariabackup/ds_archive.h deleted file mode 100644 index f419fca0c9f..00000000000 --- a/extra/mariabackup/ds_archive.h +++ /dev/null @@ -1,28 +0,0 @@ -/****************************************************** -Copyright (c) 2013 Percona LLC and/or its affiliates. - -Streaming interface for XtraBackup. - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*******************************************************/ - -#ifndef DS_ARCHIVE_H -#define DS_ARCHIVE_H - -#include "datasink.h" - -extern datasink_t datasink_archive; - -#endif diff --git a/extra/mariabackup/ds_xbstream.cc b/extra/mariabackup/ds_xbstream.cc index daf1cc73038..d527b6751ba 100644 --- a/extra/mariabackup/ds_xbstream.cc +++ b/extra/mariabackup/ds_xbstream.cc @@ -126,15 +126,19 @@ xbstream_open(ds_ctxt_t *ctxt, const char *path, MY_STAT *mystat) pthread_mutex_lock(&stream_ctxt->mutex); if (stream_ctxt->dest_file == NULL) { stream_ctxt->dest_file = ds_open(dest_ctxt, path, mystat); - if (stream_ctxt->dest_file == NULL) { - return NULL; - } } pthread_mutex_unlock(&stream_ctxt->mutex); + if (stream_ctxt->dest_file == NULL) { + return NULL; + } file = (ds_file_t *) my_malloc(sizeof(ds_file_t) + sizeof(ds_stream_file_t), MYF(MY_FAE)); + if (!file) { + msg("my_malloc() failed."); + goto err; + } stream_file = (ds_stream_file_t *) (file + 1); xbstream = stream_ctxt->xbstream; diff --git a/man/mysqldump.1 b/man/mysqldump.1 index 8af2acf9b7f..ae6d76f24c3 100644 --- a/man/mysqldump.1 +++ b/man/mysqldump.1 @@ -2261,7 +2261,7 @@ servers \- remote (federated) servers as \fBCREATE SERVER\fR\&. .sp -1 .IP \(bu 2.3 .\} -stats \- statistics tables, InnoDB and Engine Independent Table Statistics (EITS), are dumped as \fBREPLACE INTO\fR (or \fBINSERT IGNORE\fR if \fB\-\-insert\-into\fR is specified) statements without (re)creating tables\&. +stats \- statistics tables, InnoDB and Engine Independent Table Statistics (EITS), are dumped as \fBREPLACE INTO\fR (or \fBINSERT IGNORE\fR if \fB\-\-insert\-ignore\fR is specified) statements without (re)creating tables\&. .RE .RS 4 .ie n \{\ @@ -2271,17 +2271,17 @@ stats \- statistics tables, InnoDB and Engine Independent Table Statistics (EITS .sp -1 .IP \(bu 2.3 .\} -timezones \- timezone related system tables dumped as \fBREPLACE INTO\fR (or \fBINSERT IGNORE\fR if \fB\-\-insert\-into\fR is specified) statements without (re)creating tables\&. +timezones \- timezone related system tables dumped as \fBREPLACE INTO\fR (or \fBINSERT IGNORE\fR if \fB\-\-insert\-ignore\fR is specified) statements without (re)creating tables\&. .RE .sp -The format of the output is affected by \fB\-\-replace\fR and \fB\-\-insert\-into\fR\&. The \fB\-\-replace\fR option will output \fBCREATE OR REPLACE\fR +The format of the output is affected by \fB\-\-replace\fR and \fB\-\-insert\-ignore\fR\&. The \fB\-\-replace\fR option will output \fBCREATE OR REPLACE\fR forms of SQL, and also \fBDROP IF EXISTS\fR prior to \fBCREATE\fR, if a \fBCREATE OR REPLACE\fR option isn't available. .sp With \fB\-\-system=user\fR (or \fBall\fR), and \fB\-\-replace\fR, SQL is generated to generate an error if attempting to import the dump with a connection user that is being replaced within the dump\&. .sp -The \fB\-\-insert\-into\fR option will cause \fBCREATE IF NOT EXIST\fR forms of SQL to generated if available. +The \fB\-\-insert\-ignore\fR option will cause \fBCREATE IF NOT EXIST\fR forms of SQL to generated if available. .sp -For stats, and timezones, \fB\-\-replace\fR and \fB\-\-insert\-into\fR have the usual effects. +For stats, and timezones, \fB\-\-replace\fR and \fB\-\-insert\-ignore\fR have the usual effects. .sp Enabling specific options here will cause the relevant tables in the mysql database to be ignored when dumping the mysql database or \fB\-\-all\-databases\fR\&. .sp diff --git a/mysql-test/main/truncate_notembedded.result b/mysql-test/main/truncate_notembedded.result index 18a01c684e7..dabd5474141 100644 --- a/mysql-test/main/truncate_notembedded.result +++ b/mysql-test/main/truncate_notembedded.result @@ -5,7 +5,7 @@ CREATE TABLE t1 (a INT) ENGINE=MyISAM; LOCK TABLE t1 READ; connect con1,localhost,root,,test; -SET SESSION max_session_mem_used= 65536; +SET SESSION max_session_mem_used= 45500; LOCK TABLE t1 WRITE; connection default; SELECT * FROM t1; @@ -13,7 +13,7 @@ a UNLOCK TABLES; connection con1; TRUNCATE TABLE t1; -ERROR HY000: The MariaDB server is running with the --max-thread-mem-used=65536 option so it cannot execute this statement +ERROR HY000: The MariaDB server is running with the --max-thread-mem-used=45500 option so it cannot execute this statement disconnect con1; connection default; DROP TABLE t1; diff --git a/mysql-test/main/truncate_notembedded.test b/mysql-test/main/truncate_notembedded.test index 7e186c8f7d5..3addad2d975 100644 --- a/mysql-test/main/truncate_notembedded.test +++ b/mysql-test/main/truncate_notembedded.test @@ -9,7 +9,7 @@ CREATE TABLE t1 (a INT) ENGINE=MyISAM; LOCK TABLE t1 READ; --connect (con1,localhost,root,,test) -SET SESSION max_session_mem_used= 65536; +SET SESSION max_session_mem_used= 45500; --send LOCK TABLE t1 WRITE; diff --git a/mysql-test/suite/encryption/r/innodb_encryption.result b/mysql-test/suite/encryption/r/innodb_encryption.result index 9fa422bfd91..aa511b0b9d7 100644 --- a/mysql-test/suite/encryption/r/innodb_encryption.result +++ b/mysql-test/suite/encryption/r/innodb_encryption.result @@ -19,7 +19,7 @@ innodb_system # Success! # Now turn off encryption and wait for threads to decrypt everything SET GLOBAL innodb_encrypt_tables = off; -# Wait max 10 min for key encryption threads to encrypt all spaces +# Wait max 10 min for key encryption threads to decrypt all spaces SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME NOT LIKE 'innodb_undo%' AND NAME NOT LIKE 'mysql/innodb_%_stats' AND NAME NOT LIKE 'mysql/transaction_registry'; NAME diff --git a/mysql-test/suite/encryption/t/innodb_encryption.test b/mysql-test/suite/encryption/t/innodb_encryption.test index a1abfb51462..f10b5735527 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption.test +++ b/mysql-test/suite/encryption/t/innodb_encryption.test @@ -14,7 +14,9 @@ SHOW VARIABLES LIKE 'innodb_encrypt%'; SET GLOBAL innodb_encrypt_tables = ON; ---let $tables_count= `select count(*) + 1 from information_schema.tables where engine = 'InnoDB'` +let $undo_count= `select @@global.innodb_undo_tablespaces`; + +--let $tables_count= `select count(*) + 1 + $undo_count from information_schema.tables where engine = 'InnoDB'` --echo # Wait max 10 min for key encryption threads to encrypt all spaces --let $wait_timeout= 600 @@ -33,7 +35,7 @@ AND NAME NOT LIKE 'innodb_undo%' AND NAME NOT LIKE 'mysql/innodb_%_stats' AND NA --echo # Now turn off encryption and wait for threads to decrypt everything SET GLOBAL innodb_encrypt_tables = off; ---echo # Wait max 10 min for key encryption threads to encrypt all spaces +--echo # Wait max 10 min for key encryption threads to decrypt all spaces --let $wait_timeout= 600 --let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0; --source include/wait_condition.inc diff --git a/mysql-test/suite/innodb/r/innodb_defrag_stats.result b/mysql-test/suite/innodb/r/innodb_defrag_stats.result index 7092688f07b..6c5fe1817e2 100644 --- a/mysql-test/suite/innodb/r/innodb_defrag_stats.result +++ b/mysql-test/suite/innodb/r/innodb_defrag_stats.result @@ -1,22 +1,8 @@ -DROP TABLE if exists t1; -select @@global.innodb_stats_persistent; -@@global.innodb_stats_persistent -0 -set global innodb_defragment_stats_accuracy = 20; +SET GLOBAL innodb_defragment_stats_accuracy = 20; +DELETE FROM mysql.innodb_index_stats; # Create table. CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARCHAR(256), KEY SECOND(a, b)) ENGINE=INNODB; -# Populate data -INSERT INTO t1 VALUES(1, REPEAT('A', 256)); -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; +INSERT INTO t1 SELECT seq, REPEAT('A', 256) FROM seq_1_to_1024; # Not enough page splits to trigger persistent stats write yet. select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); count(stat_value) = 0 @@ -27,7 +13,7 @@ count(stat_value) = 0 select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_leaf_pages_defrag'); count(stat_value) = 0 1 -INSERT INTO t1 (b) SELECT b from t1; +INSERT INTO t1 SELECT seq, REPEAT('A', 256) FROM seq_1025_to_2048; # Persistent stats recorded. select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); count(stat_value) > 0 @@ -39,6 +25,7 @@ select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like count(stat_value) > 0 1 # Delete some rows. +BEGIN; delete from t1 where a between 100 * 20 and 100 * 20 + 30; delete from t1 where a between 100 * 19 and 100 * 19 + 30; delete from t1 where a between 100 * 18 and 100 * 18 + 30; @@ -59,8 +46,7 @@ delete from t1 where a between 100 * 4 and 100 * 4 + 30; delete from t1 where a between 100 * 3 and 100 * 3 + 30; delete from t1 where a between 100 * 2 and 100 * 2 + 30; delete from t1 where a between 100 * 1 and 100 * 1 + 30; -# Server Restarted -# Confirm persistent stats still there after restart. +COMMIT; select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); count(stat_value) > 0 1 @@ -73,9 +59,6 @@ count(stat_value) > 0 optimize table t1; Table Op Msg_type Msg_text test.t1 optimize status OK -select sleep(2); -sleep(2) -0 select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); count(stat_value) > 0 1 @@ -108,9 +91,6 @@ count(stat_value) > 0 1 # Table rename should cause stats rename. rename table t1 to t2; -select sleep(1); -sleep(1) -0 select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); count(stat_value) = 0 1 @@ -129,47 +109,36 @@ count(stat_value) > 0 select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_leaf_pages_defrag'); count(stat_value) > 0 1 -# Drop index should cause stats drop. +# Drop index should cause stats drop, but will not. drop index SECOND on t2; -select sleep(3); -sleep(3) -0 -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and index_name = 'SECOND' and stat_name in ('n_page_split'); -count(stat_value) > 0 -1 -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and index_name = 'SECOND' and stat_name in ('n_pages_freed'); -count(stat_value) > 0 -1 -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and index_name = 'SECOND' and stat_name in ('n_leaf_pages_defrag'); -count(stat_value) > 0 -1 -Server Restarted -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); -count(stat_value) = 0 -1 -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_pages_freed'); -count(stat_value) = 0 -1 -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_leaf_pages_defrag'); -count(stat_value) = 0 -1 -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_page_split'); -count(stat_value) > 0 -1 -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_pages_freed'); -count(stat_value) > 0 -1 -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_leaf_pages_defrag'); -count(stat_value) > 0 -1 +SELECT stat_name, stat_value>0 FROM mysql.innodb_index_stats +WHERE table_name like '%t2%' AND index_name='SECOND'; +stat_name stat_value>0 +n_leaf_pages_defrag 1 +n_leaf_pages_reserved 1 +n_page_split 1 +n_pages_freed 1 +# +# MDEV-26636: Statistics must not be written for temporary tables +# +SET GLOBAL innodb_defragment_stats_accuracy = 1; +CREATE TEMPORARY TABLE t (a INT PRIMARY KEY, c CHAR(255) NOT NULL) +ENGINE=InnoDB; +INSERT INTO t SELECT seq, '' FROM seq_1_to_100; +SELECT * FROM mysql.innodb_index_stats where table_name like '%t1%'; +database_name table_name index_name last_update stat_name stat_value sample_size stat_description +SELECT table_name, index_name, stat_name, stat_value>0 +FROM mysql.innodb_index_stats; +table_name index_name stat_name stat_value>0 +t2 PRIMARY n_leaf_pages_defrag 1 +t2 PRIMARY n_leaf_pages_reserved 1 +t2 PRIMARY n_page_split 1 +t2 PRIMARY n_pages_freed 1 +t2 SECOND n_leaf_pages_defrag 1 +t2 SECOND n_leaf_pages_reserved 1 +t2 SECOND n_page_split 1 +t2 SECOND n_pages_freed 1 # Clean up DROP TABLE t2; -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_page_split'); -count(stat_value) = 0 -1 -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_pages_freed'); -count(stat_value) = 0 -1 -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_leaf_pages_defrag'); -count(stat_value) = 0 -1 +SELECT * FROM mysql.innodb_index_stats; +database_name table_name index_name last_update stat_name stat_value sample_size stat_description diff --git a/mysql-test/suite/innodb/r/undo_truncate.result b/mysql-test/suite/innodb/r/undo_truncate.result index dce91a7461e..25fd07cf930 100644 --- a/mysql-test/suite/innodb/r/undo_truncate.result +++ b/mysql-test/suite/innodb/r/undo_truncate.result @@ -9,28 +9,12 @@ SET @trunc_start= WHERE variable_name = 'innodb_undo_truncations'); create table t1(keyc int primary key, c char(100)) engine = innodb; create table t2(keyc int primary key, c char(100)) engine = innodb; -CREATE PROCEDURE populate_t1() -BEGIN -DECLARE i INT DEFAULT 1; -while (i <= 20000) DO -insert into t1 values (i, 'a'); -SET i = i + 1; -END WHILE; -END | -CREATE PROCEDURE populate_t2() -BEGIN -DECLARE i INT DEFAULT 1; -while (i <= 20000) DO -insert into t2 values (i, 'a'); -SET i = i + 1; -END WHILE; -END | connect con1,localhost,root,,; begin; -call populate_t1(); +insert into t1 select seq,'a' from seq_1_to_20000; connect con2,localhost,root,,; begin; -call populate_t2(); +insert into t2 select seq,'a' from seq_1_to_20000; connection con1; update t1 set c = 'mysql'; connection con2; @@ -52,8 +36,6 @@ commit; disconnect con2; connection default; drop table t1, t2; -drop PROCEDURE populate_t1; -drop PROCEDURE populate_t2; InnoDB 0 transactions not purged SET GLOBAL innodb_undo_logs = @save_undo_logs; SET GLOBAL innodb_purge_rseg_truncate_frequency = @save_frequency; diff --git a/mysql-test/suite/innodb/t/innodb_defrag_stats.test b/mysql-test/suite/innodb/t/innodb_defrag_stats.test index 2a5026a68e5..e1e88a07477 100644 --- a/mysql-test/suite/innodb/t/innodb_defrag_stats.test +++ b/mysql-test/suite/innodb/t/innodb_defrag_stats.test @@ -1,41 +1,23 @@ --source include/have_innodb.inc ---source include/big_test.inc --source include/not_valgrind.inc --source include/not_embedded.inc +--source include/have_sequence.inc ---disable_warnings -DROP TABLE if exists t1; ---enable_warnings +SET GLOBAL innodb_defragment_stats_accuracy = 20; ---disable_query_log -let $innodb_defragment_stats_accuracy_orig=`select @@innodb_defragment_stats_accuracy`; ---enable_query_log - -select @@global.innodb_stats_persistent; -set global innodb_defragment_stats_accuracy = 20; +DELETE FROM mysql.innodb_index_stats; --echo # Create table. CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARCHAR(256), KEY SECOND(a, b)) ENGINE=INNODB; ---echo # Populate data -INSERT INTO t1 VALUES(1, REPEAT('A', 256)); -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; -INSERT INTO t1 (b) SELECT b from t1; +INSERT INTO t1 SELECT seq, REPEAT('A', 256) FROM seq_1_to_1024; --echo # Not enough page splits to trigger persistent stats write yet. select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_pages_freed'); select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_leaf_pages_defrag'); -INSERT INTO t1 (b) SELECT b from t1; +INSERT INTO t1 SELECT seq, REPEAT('A', 256) FROM seq_1025_to_2048; --echo # Persistent stats recorded. select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); @@ -43,6 +25,7 @@ select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_leaf_pages_defrag'); --echo # Delete some rows. +BEGIN; let $num_delete = 20; while ($num_delete) { @@ -50,17 +33,13 @@ while ($num_delete) eval delete from t1 where a between $j and $j + 30; dec $num_delete; } +COMMIT; ---source include/restart_mysqld.inc ---echo # Server Restarted - ---echo # Confirm persistent stats still there after restart. select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_pages_freed'); select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_leaf_pages_defrag'); optimize table t1; -select sleep(2); select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_pages_freed'); @@ -84,7 +63,6 @@ select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like --echo # Table rename should cause stats rename. rename table t1 to t2; -select sleep(1); select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_pages_freed'); @@ -94,32 +72,30 @@ select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_pages_freed'); select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_leaf_pages_defrag'); ---echo # Drop index should cause stats drop. +--echo # Drop index should cause stats drop, but will not. drop index SECOND on t2; -select sleep(3); -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and index_name = 'SECOND' and stat_name in ('n_page_split'); -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and index_name = 'SECOND' and stat_name in ('n_pages_freed'); -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and index_name = 'SECOND' and stat_name in ('n_leaf_pages_defrag'); +--sorted_result +SELECT stat_name, stat_value>0 FROM mysql.innodb_index_stats +WHERE table_name like '%t2%' AND index_name='SECOND'; + +--echo # +--echo # MDEV-26636: Statistics must not be written for temporary tables +--echo # +SET GLOBAL innodb_defragment_stats_accuracy = 1; +CREATE TEMPORARY TABLE t (a INT PRIMARY KEY, c CHAR(255) NOT NULL) +ENGINE=InnoDB; +INSERT INTO t SELECT seq, '' FROM seq_1_to_100; --source include/restart_mysqld.inc ---echo Server Restarted -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_page_split'); -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_pages_freed'); -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t1%' and stat_name in ('n_leaf_pages_defrag'); +SELECT * FROM mysql.innodb_index_stats where table_name like '%t1%'; -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_page_split'); -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_pages_freed'); -select count(stat_value) > 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_leaf_pages_defrag'); +--sorted_result +SELECT table_name, index_name, stat_name, stat_value>0 +FROM mysql.innodb_index_stats; --echo # Clean up DROP TABLE t2; -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_page_split'); -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_pages_freed'); -select count(stat_value) = 0 from mysql.innodb_index_stats where table_name like '%t2%' and stat_name in ('n_leaf_pages_defrag'); - ---disable_query_log -EVAL SET GLOBAL innodb_defragment_stats_accuracy = $innodb_defragment_stats_accuracy_orig; ---enable_query_log +SELECT * FROM mysql.innodb_index_stats; diff --git a/mysql-test/suite/innodb/t/undo_truncate.test b/mysql-test/suite/innodb/t/undo_truncate.test index 9b91c78e35a..bc2e6a3a119 100644 --- a/mysql-test/suite/innodb/t/undo_truncate.test +++ b/mysql-test/suite/innodb/t/undo_truncate.test @@ -5,6 +5,7 @@ # --source include/innodb_page_size.inc --source include/innodb_page_size_small.inc --source include/have_undo_tablespaces.inc +--source include/have_sequence.inc SET @save_undo_logs = @@GLOBAL.innodb_undo_logs; SET @save_frequency = @@GLOBAL.innodb_purge_rseg_truncate_frequency; @@ -25,37 +26,14 @@ WHERE variable_name = 'innodb_undo_truncations'); create table t1(keyc int primary key, c char(100)) engine = innodb; create table t2(keyc int primary key, c char(100)) engine = innodb; # -delimiter |; -CREATE PROCEDURE populate_t1() -BEGIN - DECLARE i INT DEFAULT 1; - while (i <= 20000) DO - insert into t1 values (i, 'a'); - SET i = i + 1; - END WHILE; -END | -delimiter ;| -# -delimiter |; -CREATE PROCEDURE populate_t2() -BEGIN - DECLARE i INT DEFAULT 1; - while (i <= 20000) DO - insert into t2 values (i, 'a'); - SET i = i + 1; - END WHILE; -END | -delimiter ;| -# -# let DATADIR = `select @@datadir`; connect (con1,localhost,root,,); begin; -send call populate_t1(); +send insert into t1 select seq,'a' from seq_1_to_20000; connect (con2,localhost,root,,); begin; -send call populate_t2(); +send insert into t2 select seq,'a' from seq_1_to_20000; connection con1; reap; send update t1 set c = 'mysql'; connection con2; reap; send update t2 set c = 'mysql'; @@ -65,25 +43,12 @@ connection con1; reap; send delete from t1; connection con2; reap; delete from t2; connection con1; reap; -let CHECKFILE = $MYSQL_TMP_DIR/check.txt; -perl; -($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size1) - = stat("$ENV{DATADIR}/undo001"); -($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size2) - = stat("$ENV{DATADIR}/undo002"); -open(OUT, ">$ENV{CHECKFILE}") || die; -print OUT "let \$size1='$size1,$size2';\n"; -close(OUT); -EOF - SET GLOBAL innodb_undo_log_truncate = 1; commit; disconnect con1; connection con2; commit; disconnect con2; connection default; drop table t1, t2; -drop PROCEDURE populate_t1; -drop PROCEDURE populate_t2; --source include/wait_all_purged.inc @@ -98,29 +63,6 @@ if (`select @@innodb_page_size IN (4096,8192,16384)`) source include/wait_condition.inc; } ---source $CHECKFILE -perl; -($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size1) - = stat("$ENV{DATADIR}/undo001"); -($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size2) - = stat("$ENV{DATADIR}/undo002"); -open(OUT, ">$ENV{CHECKFILE}") || die; -print OUT "let \$size2='$size1,$size2';\n"; -close(OUT); -EOF - ---source $CHECKFILE ---remove_file $CHECKFILE - -if ($size1 == $size2) -{ - # This fails for innodb_page_size=64k, occasionally also for 32k. - if (`select @@innodb_page_size IN (4096,8192,16384)`) - { - echo Truncation did not happen: $size1; - } -} - SET GLOBAL innodb_undo_logs = @save_undo_logs; SET GLOBAL innodb_purge_rseg_truncate_frequency = @save_frequency; SET GLOBAL innodb_undo_log_truncate = @save_truncate; diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 562b9b929f2..54632e5f79b 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -949,8 +949,10 @@ then tmpdir=$(parse_cnf "$encgroups" 'tmpdir') if [ -z "$tmpdir" ]; then xtmpdir="$(mktemp -d)" - else + elif [ "$OS" = 'Linux' ]; then xtmpdir=$(mktemp '-d' "--tmpdir=$tmpdir") + else + xtmpdir=$(TMPDIR="$tmpdir"; mktemp '-d') fi wsrep_log_info "Using '$xtmpdir' as mariabackup temporary directory" diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index d90e87b68f2..e16ed75cb16 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -725,8 +725,10 @@ EOF tmpdir=$(parse_cnf '--mysqld|sst' 'tmpdir') if [ -z "$tmpdir" ]; then tmpfile="$(mktemp)" - else + elif [ "$OS" = 'Linux' ]; then tmpfile=$(mktemp "--tmpdir=$tmpdir") + else + tmpfile=$(TMPDIR="$tmpdir"; mktemp '-d') fi wsrep_log_info "Extracting binlog files:" diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 711cd5dcca4..b0470776660 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -74,6 +74,9 @@ /* warning C4065: switch statement contains 'default' but no 'case' labels */ #pragma warning (disable : 4065) #endif +#ifdef __GNUC__ +#pragma GCC diagnostic ignored "-Wunused-label" /* yyexhaustedlab: */ +#endif int yylex(void *yylval, void *yythd); diff --git a/sql/sql_yacc_ora.yy b/sql/sql_yacc_ora.yy index e7d28eeb365..25264fbbde8 100644 --- a/sql/sql_yacc_ora.yy +++ b/sql/sql_yacc_ora.yy @@ -74,6 +74,9 @@ /* warning C4065: switch statement contains 'default' but no 'case' labels */ #pragma warning (disable : 4065) #endif +#ifdef __GNUC__ +#pragma GCC diagnostic ignored "-Wunused-label" /* yyexhaustedlab: */ +#endif int yylex(void *yylval, void *yythd); diff --git a/storage/innobase/btr/btr0defragment.cc b/storage/innobase/btr/btr0defragment.cc index 08a110c55af..3fefd77041e 100644 --- a/storage/innobase/btr/btr0defragment.cc +++ b/storage/innobase/btr/btr0defragment.cc @@ -309,6 +309,7 @@ btr_defragment_save_defrag_stats_if_needed( { if (srv_defragment_stats_accuracy != 0 // stats tracking disabled && index->table->space_id != 0 // do not track system tables + && !index->table->is_temporary() && index->stat_defrag_modified_counter >= srv_defragment_stats_accuracy) { dict_stats_defrag_pool_add(index); diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index b11e62cff5f..ab143a16417 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -858,11 +858,13 @@ fsp_try_extend_data_file(fil_space_t* space, fsp_header_t* header, mtr_t* mtr) return(0); } - /* We ignore any fragments of a full megabyte when storing the size - to the space header */ + /* For the system tablespace, we ignore any fragments of a + full megabyte when storing the size to the space header */ - space->size_in_header = ut_2pow_round( - space->size, (1024 * 1024) / page_size.physical()); + space->size_in_header = space->id + ? space->size + : ut_2pow_round(space->size, + (1024 * 1024) / page_size.physical()); mlog_write_ulint( header + FSP_SIZE, space->size_in_header, MLOG_4BYTES, mtr); @@ -1294,7 +1296,7 @@ fsp_alloc_free_page( /* It must be that we are extending a single-table tablespace whose size is still < 64 pages */ - ut_a(!is_system_tablespace(space_id)); + ut_a(!is_predefined_tablespace(space_id)); if (page_no >= FSP_EXTENT_SIZE) { ib::error() << "Trying to extend a single-table" " tablespace " << space->name << " , by single" @@ -2381,14 +2383,14 @@ take_hinted_page: return(NULL); } - if (space->size <= ret_page && !is_system_tablespace(space_id)) { + if (space->size <= ret_page && !is_predefined_tablespace(space_id)) { /* It must be that we are extending a single-table tablespace whose size is still < 64 pages */ if (ret_page >= FSP_EXTENT_SIZE) { - ib::error() << "Error (2): trying to extend" - " a single-table tablespace " << space_id - << " by single page(s) though the" + ib::error() << "Trying to extend '" + << space->chain.start->name + << "' by single page(s) though the" << " space size " << space->size << ". Page no " << ret_page << "."; ut_ad(!has_done_reservation); diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index b201de06d17..75356a43f4d 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2009, Google Inc. -Copyright (c) 2017, 2020, MariaDB Corporation. +Copyright (c) 2017, 2021, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -174,9 +174,15 @@ log_write_up_to( /** write to the log file up to the last log entry. @param[in] sync whether we want the written log also to be flushed to disk. */ -void -log_buffer_flush_to_disk( - bool sync = true); +void log_buffer_flush_to_disk(bool sync= true); + + +/** Prepare to invoke log_write_and_flush(), before acquiring log_sys.mutex. */ +#define log_write_and_flush_prepare() log_write_mutex_enter() + +/** Durably write the log up to log_sys.lsn and release log_sys.mutex. */ +ATTRIBUTE_COLD void log_write_and_flush(); + /****************************************************************//** This functions writes the log buffer to the log file and if 'flush' is set it forces a flush of the log file as well. This is meant to be diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index 1ddfda05186..ec9525f47fe 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2020, MariaDB Corporation. +Copyright (c) 2013, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -137,6 +137,10 @@ struct mtr_t { /** Commit the mini-transaction. */ void commit(); + /** Commit a mini-transaction that is shrinking a tablespace. + @param space tablespace that is being shrunk */ + ATTRIBUTE_COLD void commit_shrink(fil_space_t &space); + /** Commit a mini-transaction that did not modify any pages, but generated some redo log on a higher level, such as MLOG_FILE_NAME records and a MLOG_CHECKPOINT marker. diff --git a/storage/innobase/include/mtr0mtr.ic b/storage/innobase/include/mtr0mtr.ic index 17b7f04a29d..3714d4bce06 100644 --- a/storage/innobase/include/mtr0mtr.ic +++ b/storage/innobase/include/mtr0mtr.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2020, MariaDB Corporation. +Copyright (c) 2017, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -53,8 +53,8 @@ mtr_t::memo_push(void* object, mtr_memo_type_t type) /* If this mtr has x-fixed a clean page then we set the made_dirty flag. This tells us if we need to - grab log_flush_order_mutex at mtr_commit so that we - can insert the dirtied page to the flush list. */ + grab log_sys.flush_order_mutex at mtr_t::commit() so that we + can insert the dirtied page into the flush list. */ if ((type == MTR_MEMO_PAGE_X_FIX || type == MTR_MEMO_PAGE_SX_FIX) && !m_made_dirty) { diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 997430497bc..72fbf139300 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Google Inc. -Copyright (c) 2014, 2020, MariaDB Corporation. +Copyright (c) 2014, 2021, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1048,12 +1048,101 @@ loop: /** write to the log file up to the last log entry. @param[in] sync whether we want the written log also to be flushed to disk. */ -void -log_buffer_flush_to_disk( - bool sync) +void log_buffer_flush_to_disk(bool sync) { - ut_ad(!srv_read_only_mode); - log_write_up_to(log_get_lsn(), sync); + ut_ad(!srv_read_only_mode); + log_write_up_to(log_get_lsn(), sync); +} + + +/** Durably write the log and release log_sys.mutex */ +ATTRIBUTE_COLD void log_write_and_flush() +{ + ut_ad(!srv_read_only_mode); + ut_ad(!recv_no_log_write); + ut_ad(!recv_recovery_is_on()); + + /* The following code is adapted from log_write_up_to(). */ + DBUG_PRINT("ib_log", ("write " LSN_PF " to " LSN_PF, + log_sys.write_lsn, log_sys.lsn)); + log_sys.n_pending_flushes++; + log_sys.current_flush_lsn= log_sys.lsn; + os_event_reset(log_sys.flush_event); + ut_ad(log_sys.buf_free != log_sys.buf_next_to_write); + ulint start_offset= log_sys.buf_next_to_write; + ulint end_offset= log_sys.buf_free; + ulint area_start= ut_2pow_round(start_offset, ulint(OS_FILE_LOG_BLOCK_SIZE)); + ulint area_end= ut_calc_align(end_offset, ulint(OS_FILE_LOG_BLOCK_SIZE)); + ulong write_ahead_size= srv_log_write_ahead_size; + + log_block_set_flush_bit(log_sys.buf + area_start, TRUE); + log_block_set_checkpoint_no(log_sys.buf + area_end - OS_FILE_LOG_BLOCK_SIZE, + log_sys.next_checkpoint_no); + lsn_t write_lsn= log_sys.lsn; + byte *write_buf= log_sys.buf; + + ut_ad(area_end - area_start > 0); + + log_buffer_switch(); + + log_sys.log.set_fields(log_sys.write_lsn); + + /* Erase the end of the last log block. */ + memset(write_buf + end_offset, 0, + ~end_offset & (OS_FILE_LOG_BLOCK_SIZE - 1)); + /* Calculate pad_size if needed. */ + ulint pad_size= 0; + if (write_ahead_size > OS_FILE_LOG_BLOCK_SIZE) + { + lsn_t end_offset= + log_sys.log.calc_lsn_offset(ut_uint64_align_up(write_lsn, + OS_FILE_LOG_BLOCK_SIZE)); + ulint end_offset_in_unit= (ulint) (end_offset % write_ahead_size); + + if (end_offset_in_unit && (area_end - area_start) > end_offset_in_unit) + { + /* The first block in the unit was initialized after the last + writing. Needs to be written padded data once. */ + pad_size= std::min<ulint>(ulint(write_ahead_size) - end_offset_in_unit, + srv_log_buffer_size - area_end); + memset(write_buf + area_end, 0, pad_size); + } + } + + if (log_sys.is_encrypted()) + log_crypt(write_buf + area_start, log_sys.write_lsn, + area_end - area_start); + + /* Do the write to the log files */ + log_write_buf(write_buf + area_start, area_end - area_start + pad_size, +#ifdef UNIV_DEBUG + pad_size, +#endif /* UNIV_DEBUG */ + ut_uint64_align_down(log_sys.write_lsn, + OS_FILE_LOG_BLOCK_SIZE), + start_offset - area_start); + srv_stats.log_padded.add(pad_size); + log_sys.write_lsn= write_lsn; + + log_write_mutex_exit(); + + /* Code adapted from log_write_flush_to_disk_low() */ + + ut_a(log_sys.n_pending_flushes == 1); /* No other threads here */ + + if (srv_file_flush_method != SRV_O_DSYNC) + fil_flush(SRV_LOG_SPACE_FIRST_ID); + + log_sys.flushed_to_disk_lsn= log_sys.current_flush_lsn; + + log_sys.n_pending_flushes--; + + os_event_set(log_sys.flush_event); + + const lsn_t flush_lsn= log_sys.flushed_to_disk_lsn; + log_mutex_exit(); + + innobase_mysql_log_notify(flush_lsn); } /****************************************************************//** diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 2e364ba4945..d1885da83f4 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2020, MariaDB Corporation. +Copyright (c) 2017, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -463,6 +463,89 @@ mtr_t::commit() release_resources(); } +#ifdef UNIV_DEBUG +/** Check that all pages belong to a shrunk tablespace. */ +struct Shrink +{ + const fil_space_t &space; + Shrink(const fil_space_t &space) : space(space) {} + + bool operator()(const mtr_memo_slot_t *slot) const + { + if (!slot->object) + return true; + switch (slot->type) { + default: + ut_ad("invalid type" == 0); + return false; + case MTR_MEMO_MODIFY: + break; + case MTR_MEMO_SPACE_X_LOCK: + ut_ad(&space == slot->object); + return true; + case MTR_MEMO_PAGE_X_FIX: + case MTR_MEMO_PAGE_SX_FIX: + const buf_page_t &bpage= static_cast<buf_block_t*>(slot->object)->page; + const page_id_t &id= bpage.id; + if (id.space() == 0 && id.page_no() == TRX_SYS_PAGE_NO) + { + ut_ad(srv_is_undo_tablespace(space.id)); + break; + } + ut_ad(id.space() == space.id); + ut_ad(id.page_no() < space.size); + ut_ad(bpage.state == BUF_BLOCK_FILE_PAGE); + ut_ad(!bpage.oldest_modification); + break; + } + return true; + } +}; +#endif + +/** Commit a mini-transaction that is shrinking a tablespace. +@param space tablespace that is being shrunk */ +void mtr_t::commit_shrink(fil_space_t &space) +{ + ut_ad(is_active()); + ut_ad(!is_inside_ibuf()); + ut_ad(!high_level_read_only); + ut_ad(m_modifications); + ut_ad(m_made_dirty); + ut_ad(!recv_recovery_is_on()); + ut_ad(m_log_mode == MTR_LOG_ALL); + ut_ad(UT_LIST_GET_LEN(space.chain) == 1); + + log_write_and_flush_prepare(); + + const lsn_t start_lsn= finish_write(prepare_write()); + + log_flush_order_mutex_enter(); + /* Durably write the reduced FSP_SIZE before truncating the data file. */ + log_write_and_flush(); + + os_file_truncate(space.chain.start->name, space.chain.start->handle, + os_offset_t(space.size) << srv_page_size_shift, true); + + ut_d(m_memo.for_each_block_in_reverse(CIterate<Shrink>(space))); + + m_memo.for_each_block_in_reverse(CIterate<const ReleaseBlocks> + (ReleaseBlocks(start_lsn, m_commit_lsn, + m_flush_observer))); + log_flush_order_mutex_exit(); + + mutex_enter(&fil_system.mutex); + ut_ad(space.is_being_truncated); + space.is_being_truncated= false; + space.set_stopping(false); + mutex_exit(&fil_system.mutex); + + m_memo.for_each_block_in_reverse(CIterate<ReleaseLatches>()); + srv_stats.log_write_requests.inc(); + + release_resources(); +} + /** Commit a mini-transaction that did not modify any pages, but generated some redo log on a higher level, such as MLOG_FILE_NAME records and a MLOG_CHECKPOINT marker. diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index 68f721248b9..811a08b6a58 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -1665,7 +1665,6 @@ row_fts_merge_insert( aux_table = dict_table_open_on_name(aux_table_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); ut_ad(aux_table != NULL); - dict_table_close(aux_table, FALSE, FALSE); aux_index = dict_table_get_first_index(aux_table); ut_ad(!aux_index->is_instant()); @@ -1792,6 +1791,8 @@ row_fts_merge_insert( } exit: + dict_table_close(aux_table, FALSE, FALSE); + fts_sql_commit(trx); trx->op_info = ""; diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index c8f471f1f7a..d300e3b54e3 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -1014,28 +1014,12 @@ not_found: rseg->needs_purge = false; } - mtr.commit(); - /* Write-ahead the redo log record. */ - log_write_up_to(mtr.commit_lsn(), true); - - /* Trim the file size. */ - os_file_truncate(file->name, file->handle, - os_offset_t(size) << srv_page_size_shift, true); + mtr.commit_shrink(*space); - /* This is only executed by the srv_purge_coordinator_thread. */ + /* No mutex; this is only updated by the purge coordinator. */ export_vars.innodb_undo_truncations++; - /* In MDEV-8319 (10.5) we will PUNCH_HOLE the garbage - (with write-ahead logging). */ - - mutex_enter(&fil_system.mutex); - ut_ad(space->is_being_truncated); - space->is_being_truncated = false; - space->set_stopping(false); - mutex_exit(&fil_system.mutex); - - if (purge_sys.rseg != NULL - && purge_sys.rseg->last_page_no == FIL_NULL) { + if (purge_sys.rseg && purge_sys.rseg->last_page_no == FIL_NULL) { /* If purge_sys.rseg is pointing to rseg that was recently truncated then move to next rseg element. Note: Ideally purge_sys.rseg should be NULL because purge |