diff options
33 files changed, 426 insertions, 213 deletions
diff --git a/cmake/os/Windows.cmake b/cmake/os/Windows.cmake index 22f1ff7f30d..37fd204ed72 100644 --- a/cmake/os/Windows.cmake +++ b/cmake/os/Windows.cmake @@ -139,8 +139,8 @@ IF(MSVC) ENDIF() #TODO: update the code and remove the disabled warnings - SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4800 /wd4805 /wd4996") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4800 /wd4805 /wd4996 /wd4291 /wd4577 /we4099") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4800 /wd4805 /wd4996 /we4700") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4800 /wd4805 /wd4996 /wd4291 /wd4577 /we4099 /we4700") ENDIF() diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index 7b78d1a4c39..8fe2e186919 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -113,6 +113,11 @@ xb_mysql_connect() (char *) &opt_secure_auth); } + if (xb_plugin_dir && *xb_plugin_dir){ + mysql_options(connection, MYSQL_PLUGIN_DIR, xb_plugin_dir); + } + mysql_options(connection, MYSQL_OPT_PROTOCOL, &opt_protocol); + msg_ts("Connecting to MySQL server host: %s, user: %s, password: %s, " "port: %s, socket: %s\n", opt_host ? opt_host : "localhost", opt_user ? opt_user : "not set", diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 160d9df1df9..cb42a359896 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -119,6 +119,7 @@ my_bool xtrabackup_export; longlong xtrabackup_use_memory; +uint opt_protocol; long xtrabackup_throttle; /* 0:unlimited */ static lint io_ticket; static os_event_t wait_throttle; @@ -527,6 +528,7 @@ enum options_xtrabackup OPT_XTRA_TABLES_EXCLUDE, OPT_XTRA_DATABASES_EXCLUDE, + OPT_PROTOCOL }; struct my_option xb_client_options[] = @@ -759,6 +761,9 @@ struct my_option xb_client_options[] = 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"protocol", OPT_PROTOCOL, "The protocol to use for connection (tcp, socket, pipe, memory).", + 0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"socket", 'S', "This option specifies the socket to use when " "connecting to the local database server with a UNIX domain socket. " "The option accepts a string argument. See mysql --help for details.", @@ -1290,8 +1295,13 @@ xb_get_one_option(int optid, start[1]=0 ; } break; - - + case OPT_PROTOCOL: + if (argument) + { + opt_protocol= find_type_or_exit(argument, &sql_protocol_typelib, + opt->name); + } + break; #include "sslopt-case.h" case '?': diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index d1d1e8df29a..b226a9d1d83 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -43,6 +43,8 @@ extern char *xtrabackup_incremental_dir; extern char *xtrabackup_incremental_basedir; extern char *innobase_data_home_dir; extern char *innobase_buffer_pool_filename; +extern char *xb_plugin_dir; +extern uint opt_protocol; extern ds_ctxt_t *ds_meta; extern ds_ctxt_t *ds_data; diff --git a/mysql-test/include/innodb_undo_tablespaces.combinations b/mysql-test/include/innodb_undo_tablespaces.combinations new file mode 100644 index 00000000000..0ac7a6e2d05 --- /dev/null +++ b/mysql-test/include/innodb_undo_tablespaces.combinations @@ -0,0 +1,5 @@ +[undo0] +innodb-undo-tablespaces=0 + +[undo3] +innodb-undo-tablespaces=3 diff --git a/mysql-test/include/innodb_undo_tablespaces.inc b/mysql-test/include/innodb_undo_tablespaces.inc new file mode 100644 index 00000000000..b109fcc4f0e --- /dev/null +++ b/mysql-test/include/innodb_undo_tablespaces.inc @@ -0,0 +1,3 @@ +# The goal of including this file is to enable innodb_undo_tablespaces combinations +# (see include/innodb_undo_tablespaces.combinations) + diff --git a/mysql-test/r/alter_table.result b/mysql-test/r/alter_table.result index 92df7bc577e..f7f87b810ce 100644 --- a/mysql-test/r/alter_table.result +++ b/mysql-test/r/alter_table.result @@ -2125,6 +2125,58 @@ t1 CREATE TABLE `t1` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8 DROP TABLE t1; # +# MDEV-8960 Can't refer the same column twice in one ALTER TABLE +# +CREATE TABLE t1 ( +`a` int(11) DEFAULT NULL +) DEFAULT CHARSET=utf8; +ALTER TABLE t1 ADD COLUMN `consultant_id` integer NOT NULL, +ALTER COLUMN `consultant_id` DROP DEFAULT; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `consultant_id` int(11) NOT NULL +) ENGINE=MyISAM DEFAULT CHARSET=utf8 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` int(11) DEFAULT NULL +) DEFAULT CHARSET=utf8; +ALTER TABLE t1 ADD COLUMN `consultant_id` integer NOT NULL, +ALTER COLUMN `consultant_id` SET DEFAULT 2; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `consultant_id` int(11) NOT NULL DEFAULT 2 +) ENGINE=MyISAM DEFAULT CHARSET=utf8 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` int(11) DEFAULT NULL +) DEFAULT CHARSET=utf8; +ALTER TABLE t1 ADD COLUMN `consultant_id` integer NOT NULL DEFAULT 2, +ALTER COLUMN `consultant_id` DROP DEFAULT; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `consultant_id` int(11) NOT NULL +) ENGINE=MyISAM DEFAULT CHARSET=utf8 +DROP TABLE t1; +CREATE TABLE t1 ( +`a` int(11) DEFAULT NULL +) DEFAULT CHARSET=utf8; +ALTER TABLE t1 ADD COLUMN `consultant_id` integer NOT NULL DEFAULT 2, +ALTER COLUMN `consultant_id` DROP DEFAULT, +MODIFY COLUMN `consultant_id` BIGINT; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `consultant_id` bigint(20) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=utf8 +DROP TABLE t1; +# # Start of 10.1 tests # # diff --git a/mysql-test/suite/encryption/r/innodb_encryption.result b/mysql-test/suite/encryption/r/innodb_encryption.result index 26c77499d25..ce494098d44 100644 --- a/mysql-test/suite/encryption/r/innodb_encryption.result +++ b/mysql-test/suite/encryption/r/innodb_encryption.result @@ -1,3 +1,5 @@ +call mtr.add_suppression("InnoDB: New log files created"); +call mtr.add_suppression("InnoDB: Creating foreign key constraint system tables."); SET @start_global_value = @@global.innodb_encryption_threads; SHOW VARIABLES LIKE 'innodb_encrypt%'; Variable_name Value diff --git a/mysql-test/suite/encryption/t/innodb_encryption.test b/mysql-test/suite/encryption/t/innodb_encryption.test index 50aca2a7260..6e9f80aac0c 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption.test +++ b/mysql-test/suite/encryption/t/innodb_encryption.test @@ -3,10 +3,14 @@ # -- source include/have_innodb.inc -- source include/have_example_key_management_plugin.inc +-- source include/innodb_undo_tablespaces.inc # embedded does not support restart -- source include/not_embedded.inc +call mtr.add_suppression("InnoDB: New log files created"); +call mtr.add_suppression("InnoDB: Creating foreign key constraint system tables."); + SET @start_global_value = @@global.innodb_encryption_threads; SHOW VARIABLES LIKE 'innodb_encrypt%'; diff --git a/mysql-test/suite/innodb/t/xa_recovery.test b/mysql-test/suite/innodb/t/xa_recovery.test index 957b758d05c..20bb52c22f2 100644 --- a/mysql-test/suite/innodb/t/xa_recovery.test +++ b/mysql-test/suite/innodb/t/xa_recovery.test @@ -15,8 +15,14 @@ connect (con1,localhost,root); XA START 'x'; UPDATE t1 set a=2; XA END 'x'; XA PREPARE 'x'; connection default; +# innodb_force_recovery=2 prevents the purge and tests that the fix of +# MDEV-13606 XA PREPARE transactions should survive innodb_force_recovery=1 or 2 +# is present. +--let $restart_parameters= --innodb-force-recovery=2 --let $shutdown_timeout=0 --source include/restart_mysqld.inc +--let $restart_parameters= +--let $shutdown_timeout= disconnect con1; connect (con1,localhost,root); diff --git a/mysql-test/suite/mariabackup/auth_plugin_win.opt b/mysql-test/suite/mariabackup/auth_plugin_win.opt new file mode 100644 index 00000000000..e534ae1eae5 --- /dev/null +++ b/mysql-test/suite/mariabackup/auth_plugin_win.opt @@ -0,0 +1 @@ +--loose-enable-named-pipe diff --git a/mysql-test/suite/mariabackup/auth_plugin_win.result b/mysql-test/suite/mariabackup/auth_plugin_win.result new file mode 100644 index 00000000000..7a623be147f --- /dev/null +++ b/mysql-test/suite/mariabackup/auth_plugin_win.result @@ -0,0 +1,5 @@ +INSTALL SONAME 'auth_named_pipe'; +CREATE USER 'USERNAME' IDENTIFIED WITH named_pipe; +GRANT ALL PRIVILEGES ON *.* to USERNAME; +DROP USER 'USERNAME'; +UNINSTALL SONAME 'auth_named_pipe'; diff --git a/mysql-test/suite/mariabackup/auth_plugin_win.test b/mysql-test/suite/mariabackup/auth_plugin_win.test new file mode 100644 index 00000000000..9c8cd5ad411 --- /dev/null +++ b/mysql-test/suite/mariabackup/auth_plugin_win.test @@ -0,0 +1,31 @@ +--source include/windows.inc +--source include/not_embedded.inc + +if (!$AUTH_NAMED_PIPE_SO) { + skip No named pipe plugin; +} + +if (!$USERNAME) { + skip USERNAME variable is undefined; +} + +if (`SELECT count(*) <> 0 FROM mysql.user WHERE user = '$USERNAME'`) { + skip \$USER=$USER which exists in mysql.user; +} + +INSTALL SONAME 'auth_named_pipe'; + +--replace_result $USERNAME USERNAME +eval CREATE USER '$USERNAME' IDENTIFIED WITH named_pipe; +--replace_result $USERNAME USERNAME +eval GRANT ALL PRIVILEGES ON *.* to $USERNAME; + +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf -u $USERNAME --backup --protocol=pipe --target-dir=$targetdir; +--enable_result_log +--replace_result $USERNAME USERNAME +eval DROP USER '$USERNAME'; +rmdir $targetdir; +UNINSTALL SONAME 'auth_named_pipe'; + diff --git a/mysql-test/t/alter_table.test b/mysql-test/t/alter_table.test index 93fd23b7fea..ca2f73db5ca 100644 --- a/mysql-test/t/alter_table.test +++ b/mysql-test/t/alter_table.test @@ -1766,6 +1766,48 @@ SHOW CREATE TABLE t1; DROP TABLE t1; --echo # +--echo # MDEV-8960 Can't refer the same column twice in one ALTER TABLE +--echo # + +CREATE TABLE t1 ( + `a` int(11) DEFAULT NULL +) DEFAULT CHARSET=utf8; + +ALTER TABLE t1 ADD COLUMN `consultant_id` integer NOT NULL, +ALTER COLUMN `consultant_id` DROP DEFAULT; + +SHOW CREATE TABLE t1; +DROP TABLE t1; + +CREATE TABLE t1 ( + `a` int(11) DEFAULT NULL +) DEFAULT CHARSET=utf8; + +ALTER TABLE t1 ADD COLUMN `consultant_id` integer NOT NULL, +ALTER COLUMN `consultant_id` SET DEFAULT 2; +SHOW CREATE TABLE t1; +DROP TABLE t1; + +CREATE TABLE t1 ( + `a` int(11) DEFAULT NULL +) DEFAULT CHARSET=utf8; + +ALTER TABLE t1 ADD COLUMN `consultant_id` integer NOT NULL DEFAULT 2, +ALTER COLUMN `consultant_id` DROP DEFAULT; +SHOW CREATE TABLE t1; +DROP TABLE t1; + +CREATE TABLE t1 ( + `a` int(11) DEFAULT NULL +) DEFAULT CHARSET=utf8; + +ALTER TABLE t1 ADD COLUMN `consultant_id` integer NOT NULL DEFAULT 2, +ALTER COLUMN `consultant_id` DROP DEFAULT, +MODIFY COLUMN `consultant_id` BIGINT; +SHOW CREATE TABLE t1; +DROP TABLE t1; + +--echo # --echo # Start of 10.1 tests --echo # diff --git a/mysys/my_init.c b/mysys/my_init.c index 7f0f7a8cc44..84489a994e3 100644 --- a/mysys/my_init.c +++ b/mysys/my_init.c @@ -200,7 +200,6 @@ Voluntary context switches %ld, Involuntary context switches %ld\n", _CrtSetReportMode( _CRT_ASSERT, _CRTDBG_MODE_FILE ); _CrtSetReportFile( _CRT_ASSERT, _CRTDBG_FILE_STDERR ); _CrtCheckMemory(); - _CrtDumpMemoryLeaks(); #endif } diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 68d086eccc5..8191782f24d 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -7702,9 +7702,25 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, { if (def->change && ! def->field) { - my_error(ER_BAD_FIELD_ERROR, MYF(0), def->change, - table->s->table_name.str); - goto err; + /* + Check if there is modify for newly added field. + */ + Create_field *find; + find_it.rewind(); + while((find=find_it++)) + { + if (!my_strcasecmp(system_charset_info,find->field_name, def->field_name)) + break; + } + + if (find && !find->field) + find_it.remove(); + else + { + my_error(ER_BAD_FIELD_ERROR, MYF(0), def->change, + table->s->table_name.str); + goto err; + } } /* Check that the DATE/DATETIME not null field we are going to add is @@ -7770,6 +7786,29 @@ mysql_prepare_alter_table(THD *thd, TABLE *table, find_it.after(def); // Put column after this } } + /* + Check if there is alter for newly added field. + */ + alter_it.rewind(); + Alter_column *alter; + while ((alter=alter_it++)) + { + if (!my_strcasecmp(system_charset_info,def->field_name, alter->name)) + break; + } + if (alter) + { + if (def->sql_type == MYSQL_TYPE_BLOB) + { + my_error(ER_BLOB_CANT_HAVE_DEFAULT, MYF(0), def->change); + goto err; + } + if ((def->default_value= alter->default_value)) // Use new default + def->flags&= ~NO_DEFAULT_VALUE_FLAG; + else + def->flags|= NO_DEFAULT_VALUE_FLAG; + alter_it.remove(); + } } if (alter_info->alter_list.elements) { diff --git a/storage/connect/CMakeLists.txt b/storage/connect/CMakeLists.txt index a5ec515011c..2fabd12dcd0 100644 --- a/storage/connect/CMakeLists.txt +++ b/storage/connect/CMakeLists.txt @@ -264,13 +264,6 @@ IF(CONNECT_WITH_JDBC) JdbcInterface.java ApacheInterface.java MariadbInterface.java MysqlInterface.java OracleInterface.java PostgresqlInterface.java JavaWrappers.jar) - # TODO: Find how to compile and install the java wrapper classes - # Find required libraries and include directories - SET (JAVA_SOURCES JdbcInterface.java) - add_jar(JdbcInterface ${JAVA_SOURCES}) - INSTALL(FILES ${CMAKE_CURRENT_SOURCE_DIR}/JavaWrappers.jar - ${CMAKE_CURRENT_BINARY_DIR}/JdbcInterface.jar - DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) add_definitions(-DJDBC_SUPPORT) IF(CONNECT_WITH_MONGO) SET(CONNECT_SOURCES ${CONNECT_SOURCES} @@ -348,4 +341,31 @@ MYSQL_ADD_PLUGIN(connect ${CONNECT_SOURCES} LINK_LIBRARIES ${ZLIB_LIBRARY} ${XML_LIBRARY} ${ICONV_LIBRARY} ${ODBC_LIBRARY} ${JDBC_LIBRARY} ${IPHLPAPI_LIBRARY}) +IF(NOT TARGET connect) + RETURN() +ENDIF() + +# Install some extra files that belong to connect engine +IF(WIN32) + # install ha_connect.lib + GET_TARGET_PROPERTY(CONNECT_LOCATION connect LOCATION) + STRING(REPLACE "dll" "lib" CONNECT_LIB ${CONNECT_LOCATION}) + IF(CMAKE_CONFIGURATION_TYPES) + STRING(REPLACE "${CMAKE_CFG_INTDIR}" "\${CMAKE_INSTALL_CONFIG_NAME}" + CONNECT_LIB ${CONNECT_LIB}) + ENDIF() + INSTALL(FILES ${CONNECT_LIB} + DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) +ENDIF(WIN32) + +IF(CONNECT_WITH_JDBC AND JAVA_FOUND AND JNI_FOUND) + # TODO: Find how to compile and install the java wrapper classes + # Find required libraries and include directories + SET (JAVA_SOURCES JdbcInterface.java) + add_jar(JdbcInterface ${JAVA_SOURCES}) + INSTALL(FILES + ${CMAKE_CURRENT_SOURCE_DIR}/JavaWrappers.jar + ${CMAKE_CURRENT_BINARY_DIR}/JdbcInterface.jar + DESTINATION ${INSTALL_PLUGINDIR} COMPONENT connect-engine) +ENDIF() diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index b8b7643702e..f3b701816e6 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -478,13 +478,15 @@ buf_read_page_background( << " in the background" " in a non-existing or being-dropped tablespace"; break; + case DB_PAGE_CORRUPTED: case DB_DECRYPTION_FAILED: ib::error() - << "Background Page read failed to decrypt page " - << page_id; + << "Background Page read failed to " + "read or decrypt " << page_id; break; default: - ut_error; + ib::fatal() << "Error " << err << " in background read of " + << page_id; } srv_stats.buf_pool_reads.add(count); @@ -755,9 +757,10 @@ buf_read_ahead_linear( case DB_TABLESPACE_DELETED: case DB_ERROR: break; + case DB_PAGE_CORRUPTED: case DB_DECRYPTION_FAILED: ib::error() << "linear readahead failed to" - " decrypt page " + " read or decrypt " << page_id_t(page_id.space(), i); break; default: @@ -853,8 +856,9 @@ tablespace_deleted: break; case DB_TABLESPACE_DELETED: goto tablespace_deleted; + case DB_PAGE_CORRUPTED: case DB_DECRYPTION_FAILED: - ib::error() << "Failed to decrypt page " << page_id + ib::error() << "Failed to read or decrypt " << page_id << " for change buffer merge"; break; default: @@ -936,8 +940,8 @@ buf_read_recv_pages( cur_page_id, page_size, true); } - if (err == DB_DECRYPTION_FAILED) { - ib::error() << "Recovery failed to decrypt page " + if (err == DB_DECRYPTION_FAILED || err == DB_PAGE_CORRUPTED) { + ib::error() << "Recovery failed to read or decrypt " << cur_page_id; } } diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index e1b5bcbc325..faf57a33cab 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -168,7 +168,12 @@ fil_crypt_get_latest_key_version( crypt_data->min_key_version, key_version, srv_fil_crypt_rotate_key_age)) { - os_event_set(fil_crypt_threads_event); + /* Below event seen as NULL-pointer at startup + when new database was created and we create a + checkpoint. Only seen when debugging. */ + if (fil_crypt_threads_inited) { + os_event_set(fil_crypt_threads_event); + } } } diff --git a/storage/innobase/row/row0umod.cc b/storage/innobase/row/row0umod.cc index 3df68ecb94d..0a2a68d1fef 100644 --- a/storage/innobase/row/row0umod.cc +++ b/storage/innobase/row/row0umod.cc @@ -34,6 +34,7 @@ Created 2/27/1997 Heikki Tuuri #include "trx0roll.h" #include "btr0btr.h" #include "mach0data.h" +#include "ibuf0ibuf.h" #include "row0undo.h" #include "row0vers.h" #include "row0log.h" diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 31e70a5aaa6..ac23234179c 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -843,18 +843,9 @@ trx_resurrect_insert( << trx_get_id_for_print(trx) << " was in the XA prepared state."; - if (srv_force_recovery == 0) { - - trx->state = TRX_STATE_PREPARED; - ++trx_sys->n_prepared_trx; - ++trx_sys->n_prepared_recovered_trx; - } else { - - ib::info() << "Since innodb_force_recovery" - " > 0, we will force a rollback."; - - trx->state = TRX_STATE_ACTIVE; - } + trx->state = TRX_STATE_PREPARED; + trx_sys->n_prepared_trx++; + trx_sys->n_prepared_recovered_trx++; } else { trx->state = TRX_STATE_COMMITTED_IN_MEMORY; } @@ -914,24 +905,14 @@ trx_resurrect_update_in_prepared_state( ib::info() << "Transaction " << trx_get_id_for_print(trx) << " was in the XA prepared state."; - if (srv_force_recovery == 0) { - - ut_ad(trx->state != TRX_STATE_FORCED_ROLLBACK); - - if (trx_state_eq(trx, TRX_STATE_NOT_STARTED)) { - ++trx_sys->n_prepared_trx; - ++trx_sys->n_prepared_recovered_trx; - } else { - ut_ad(trx_state_eq(trx, TRX_STATE_PREPARED)); - } - - trx->state = TRX_STATE_PREPARED; + if (trx_state_eq(trx, TRX_STATE_NOT_STARTED)) { + trx_sys->n_prepared_trx++; + trx_sys->n_prepared_recovered_trx++; } else { - ib::info() << "Since innodb_force_recovery > 0, we" - " will rollback it anyway."; - - trx->state = TRX_STATE_ACTIVE; + ut_ad(trx_state_eq(trx, TRX_STATE_PREPARED)); } + + trx->state = TRX_STATE_PREPARED; } else { trx->state = TRX_STATE_COMMITTED_IN_MEMORY; } diff --git a/storage/xtradb/buf/buf0rea.cc b/storage/xtradb/buf/buf0rea.cc index b2b737b8d40..76b71550710 100644 --- a/storage/xtradb/buf/buf0rea.cc +++ b/storage/xtradb/buf/buf0rea.cc @@ -428,13 +428,14 @@ read_ahead: space, i); break; case DB_DECRYPTION_FAILED: + case DB_PAGE_CORRUPTED: ib_logf(IB_LOG_LEVEL_ERROR, - "Random readahead failed to decrypt page " + "Random readahead failed to decrypt page or page corrupted " ULINTPF ":" ULINTPF ".", i, space); break; default: - ut_error; + ib_logf(IB_LOG_LEVEL_FATAL, "Error %u (%s) in random readahead", err, ut_strerr(err)); } } } @@ -570,13 +571,14 @@ buf_read_page_async( break; case DB_DECRYPTION_FAILED: + case DB_PAGE_CORRUPTED: ib_logf(IB_LOG_LEVEL_ERROR, - "Async page read failed to decrypt page " + "Async page read failed to decrypt page or page corrupted " ULINTPF ":" ULINTPF ".", space, offset); break; default: - ut_error; + ib_logf(IB_LOG_LEVEL_FATAL, "Error %u (%s) in async page read", err, ut_strerr(err)); } srv_stats.buf_pool_reads.add(count); @@ -863,13 +865,14 @@ buf_read_ahead_linear( break; case DB_DECRYPTION_FAILED: + case DB_PAGE_CORRUPTED: ib_logf(IB_LOG_LEVEL_ERROR, - "Linear readahead failed to decrypt page " + "Linear readahead failed to decrypt page or page corrupted" ULINTPF ":" ULINTPF ".", i, space); break; default: - ut_error; + ib_logf(IB_LOG_LEVEL_FATAL, "Error %u (%s) in linear readahead", err, ut_strerr(err)); } } } @@ -963,13 +966,14 @@ tablespace_deleted: ibuf_delete_for_discarded_space(space_ids[i]); break; case DB_DECRYPTION_FAILED: + case DB_PAGE_CORRUPTED: ib_logf(IB_LOG_LEVEL_ERROR, - "Failed to decrypt insert buffer page " + "Failed to decrypt insert buffer page or page corrupted " ULINTPF ":" ULINTPF ".", space_ids[i], page_nos[i]); break; default: - ut_error; + ib_logf(IB_LOG_LEVEL_FATAL, "Error %u (%s) in insert buffer read", err, ut_strerr(err)); } } diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index fa4d105e7dc..2be598cc18f 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -191,7 +191,12 @@ fil_crypt_get_latest_key_version( crypt_data->min_key_version, key_version, srv_fil_crypt_rotate_key_age)) { - os_event_set(fil_crypt_threads_event); + /* Below event seen as NULL-pointer at startup + when new database was created and we create a + checkpoint. Only seen when debugging. */ + if (fil_crypt_threads_inited) { + os_event_set(fil_crypt_threads_event); + } } } @@ -1649,20 +1654,6 @@ fil_crypt_find_page_to_rotate( return found; } -/*********************************************************************** -Check if a page is uninitialized (doesn't need to be rotated) -@param[in] frame Page to check -@param[in] zip_size zip_size or 0 -@return true if page is uninitialized, false if not. */ -static inline -bool -fil_crypt_is_page_uninitialized( - const byte *frame, - uint zip_size) -{ - return (buf_page_is_zeroes(frame, zip_size)); -} - #define fil_crypt_get_page_throttle(state,offset,mtr,sleeptime_ms) \ fil_crypt_get_page_throttle_func(state, offset, mtr, \ sleeptime_ms, __FILE__, __LINE__) @@ -1823,6 +1814,7 @@ fil_crypt_rotate_page( fil_space_crypt_t *crypt_data = space->crypt_data; ut_ad(space->n_pending_ops > 0); + ut_ad(offset > 0); /* In fil_crypt_thread where key rotation is done we have acquired space and checked that this space is not yet @@ -1851,31 +1843,40 @@ fil_crypt_rotate_page( byte* frame = buf_block_get_frame(block); uint kv = mach_read_from_4(frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - /* check if tablespace is closing after reading page */ - if (!space->is_stopping()) { - - if (kv == 0 && - fil_crypt_is_page_uninitialized(frame, zip_size)) { - ; - } else if (fil_crypt_needs_rotation( - crypt_data->encryption, - kv, key_state->key_version, - key_state->rotate_key_age)) { - - modified = true; - - /* force rotation by dummy updating page */ - mlog_write_ulint(frame + - FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, - space_id, MLOG_4BYTES, &mtr); - - /* statistics */ - state->crypt_stat.pages_modified++; - } else { - if (crypt_data->is_encrypted()) { - if (kv < state->min_key_version_found) { - state->min_key_version_found = kv; - } + if (space->is_stopping()) { + /* The tablespace is closing (in DROP TABLE or + TRUNCATE TABLE or similar): avoid further access */ + } else if (!*reinterpret_cast<uint32_t*>(FIL_PAGE_OFFSET + + frame)) { + /* It looks like this page was never + allocated. Because key rotation is accessing + pages in a pattern that is unlike the normal + B-tree and undo log access pattern, we cannot + invoke fseg_page_is_free() here, because that + could result in a deadlock. If we invoked + fseg_page_is_free() and released the + tablespace latch before acquiring block->lock, + then the fseg_page_is_free() information + could be stale already. */ + ut_ad(kv == 0); + ut_ad(page_get_space_id(frame) == 0); + } else if (fil_crypt_needs_rotation( + crypt_data->encryption, + kv, key_state->key_version, + key_state->rotate_key_age)) { + + modified = true; + + /* force rotation by dummy updating page */ + mlog_write_ulint(frame + FIL_PAGE_SPACE_ID, + space_id, MLOG_4BYTES, &mtr); + + /* statistics */ + state->crypt_stat.pages_modified++; + } else { + if (crypt_data->is_encrypted()) { + if (kv < state->min_key_version_found) { + state->min_key_version_found = kv; } } diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index fdd09a6034e..d0d98b4b77d 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -2372,8 +2372,10 @@ the first page of a first data file at database startup. @param[out] space_id tablepspace ID @param[out] flushed_lsn flushed lsn value @param[out] crypt_data encryption crypt data -@retval NULL on success, or if innodb_force_recovery is set -@return pointer to an error message string */ +@param[in] check_first_page true if first page contents + should be checked +@return NULL on success, or if innodb_force_recovery is set +@retval pointer to an error message string */ UNIV_INTERN const char* fil_read_first_page( @@ -2382,7 +2384,8 @@ fil_read_first_page( ulint* flags, ulint* space_id, lsn_t* flushed_lsn, - fil_space_crypt_t** crypt_data) + fil_space_crypt_t** crypt_data, + bool check_first_page) { byte* buf; byte* page; @@ -2418,28 +2421,32 @@ fil_read_first_page( *flags and *space_id as they were read from the first file and do not validate the first page. */ if (!one_read_already) { - *space_id = fsp_header_get_space_id(page); - *flags = fsp_header_get_flags(page); - - if (flushed_lsn) { - *flushed_lsn = mach_read_from_8(page + + /* Undo tablespace does not contain correct FSP_HEADER, + and actually we really need to read only crypt_data. */ + if (check_first_page) { + *space_id = fsp_header_get_space_id(page); + *flags = fsp_header_get_flags(page); + + if (flushed_lsn) { + *flushed_lsn = mach_read_from_8(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - } + } - if (!fsp_flags_is_valid(*flags)) { - ulint cflags = fsp_flags_convert_from_101(*flags); - if (cflags == ULINT_UNDEFINED) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Invalid flags 0x%x in tablespace %u", - unsigned(*flags), unsigned(*space_id)); - return "invalid tablespace flags"; - } else { - *flags = cflags; + if (!fsp_flags_is_valid(*flags)) { + ulint cflags = fsp_flags_convert_from_101(*flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Invalid flags 0x%x in tablespace %u", + unsigned(*flags), unsigned(*space_id)); + return "invalid tablespace flags"; + } else { + *flags = cflags; + } } - } - if (!(IS_XTRABACKUP() && srv_backup_mode)) { - check_msg = fil_check_first_page(page, *space_id, *flags); + if (!(IS_XTRABACKUP() && srv_backup_mode)) { + check_msg = fil_check_first_page(page, *space_id, *flags); + } } /* Possible encryption crypt data is also stored only to first page diff --git a/storage/xtradb/fsp/fsp0fsp.cc b/storage/xtradb/fsp/fsp0fsp.cc index df8c6ffe222..a2c11224ffd 100644 --- a/storage/xtradb/fsp/fsp0fsp.cc +++ b/storage/xtradb/fsp/fsp0fsp.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -2044,15 +2045,6 @@ fseg_create_general( mtr_x_lock(latch, mtr); - if (rw_lock_get_x_lock_count(latch) == 1) { - /* This thread did not own the latch before this call: free - excess pages from the insert buffer free list */ - - if (space == IBUF_SPACE_ID) { - ibuf_free_excess_pages(); - } - } - if (!has_done_reservation) { success = fsp_reserve_free_extents(&n_reserved, space, 2, FSP_NORMAL, mtr); @@ -2623,15 +2615,6 @@ fseg_alloc_free_page_general( mtr_x_lock(latch, mtr); - if (rw_lock_get_x_lock_count(latch) == 1) { - /* This thread did not own the latch before this call: free - excess pages from the insert buffer free list */ - - if (space == IBUF_SPACE_ID) { - ibuf_free_excess_pages(); - } - } - inode = fseg_inode_get(seg_header, space, zip_size, mtr); if (!has_done_reservation diff --git a/storage/xtradb/ibuf/ibuf0ibuf.cc b/storage/xtradb/ibuf/ibuf0ibuf.cc index 0445bb557e1..b920c2ae594 100644 --- a/storage/xtradb/ibuf/ibuf0ibuf.cc +++ b/storage/xtradb/ibuf/ibuf0ibuf.cc @@ -2216,6 +2216,8 @@ ibuf_remove_free_page(void) page_t* root; page_t* bitmap_page; + log_free_check(); + mtr_start(&mtr); /* Acquire the fsp latch before the ibuf header, obeying the latching @@ -2327,22 +2329,7 @@ ibuf_free_excess_pages(void) { ulint i; -#ifdef UNIV_SYNC_DEBUG - ut_ad(rw_lock_own(fil_space_get_latch(IBUF_SPACE_ID, NULL), - RW_LOCK_EX)); -#endif /* UNIV_SYNC_DEBUG */ - - ut_ad(rw_lock_get_x_lock_count( - fil_space_get_latch(IBUF_SPACE_ID, NULL)) == 1); - - /* NOTE: We require that the thread did not own the latch before, - because then we know that we can obey the correct latching order - for ibuf latches */ - - if (!ibuf) { - /* Not yet initialized; not sure if this is possible, but - does no harm to check for it. */ - + if (srv_force_recovery >= SRV_FORCE_NO_IBUF_MERGE) { return; } diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 6eab5db6883..a33cec65ed5 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -804,8 +804,10 @@ the first page of a first data file at database startup. @param[out] space_id tablepspace ID @param[out] flushed_lsn flushed lsn value @param[out] crypt_data encryption crypt data -@retval NULL on success, or if innodb_force_recovery is set -@return pointer to an error message string */ +@param[in] check_first_page true if first page contents + should be checked +@return NULL on success, or if innodb_force_recovery is set +@retval pointer to an error message string */ UNIV_INTERN const char* fil_read_first_page( @@ -814,7 +816,8 @@ fil_read_first_page( ulint* flags, ulint* space_id, lsn_t* flushed_lsn, - fil_space_crypt_t** crypt_data) + fil_space_crypt_t** crypt_data, + bool check_first_page=true) MY_ATTRIBUTE((warn_unused_result)); #endif /* !UNIV_HOTBACKUP */ diff --git a/storage/xtradb/row/row0ins.cc b/storage/xtradb/row/row0ins.cc index 00b8ce17b26..3a9aa91d52e 100644 --- a/storage/xtradb/row/row0ins.cc +++ b/storage/xtradb/row/row0ins.cc @@ -38,6 +38,7 @@ Created 4/20/1996 Heikki Tuuri #include "btr0btr.h" #include "btr0cur.h" #include "mach0data.h" +#include "ibuf0ibuf.h" #include "que0que.h" #include "row0upd.h" #include "row0sel.h" @@ -3064,6 +3065,11 @@ row_ins_sec_index_entry( if (err == DB_FAIL) { mem_heap_empty(heap); + if (index->space == IBUF_SPACE_ID + && !dict_index_is_unique(index)) { + ibuf_free_excess_pages(); + } + /* Try then pessimistic descent to the B-tree */ log_free_check(); diff --git a/storage/xtradb/row/row0uins.cc b/storage/xtradb/row/row0uins.cc index f14a4ef9bcf..29660055fad 100644 --- a/storage/xtradb/row/row0uins.cc +++ b/storage/xtradb/row/row0uins.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -201,6 +202,10 @@ row_undo_ins_remove_sec_low( mtr_s_lock(dict_index_get_lock(index), &mtr); } else { ut_ad(mode == BTR_MODIFY_TREE); + if (index->space == IBUF_SPACE_ID + && !dict_index_is_unique(index)) { + ibuf_free_excess_pages(); + } mtr_x_lock(dict_index_get_lock(index), &mtr); } diff --git a/storage/xtradb/row/row0umod.cc b/storage/xtradb/row/row0umod.cc index 8deba4f00a5..f23d7e9dc68 100644 --- a/storage/xtradb/row/row0umod.cc +++ b/storage/xtradb/row/row0umod.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -35,6 +36,7 @@ Created 2/27/1997 Heikki Tuuri #include "trx0roll.h" #include "btr0btr.h" #include "mach0data.h" +#include "ibuf0ibuf.h" #include "row0undo.h" #include "row0vers.h" #include "row0log.h" @@ -409,6 +411,11 @@ row_undo_mod_del_mark_or_remove_sec_low( log_free_check(); mtr_start_trx(&mtr, thr_get_trx(thr)); + if (mode == BTR_MODIFY_TREE + && index->space == IBUF_SPACE_ID + && !dict_index_is_unique(index)) { + ibuf_free_excess_pages(); + } if (*index->name == TEMP_INDEX_PREFIX) { /* The index->online_status may change if the @@ -581,6 +588,11 @@ row_undo_mod_del_unmark_sec_and_undo_update( log_free_check(); mtr_start_trx(&mtr, thr_get_trx(thr)); + if (mode == BTR_MODIFY_TREE + && index->space == IBUF_SPACE_ID + && !dict_index_is_unique(index)) { + ibuf_free_excess_pages(); + } if (*index->name == TEMP_INDEX_PREFIX) { /* The index->online_status may change if the diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index fd129c3e55f..fadbeacbc97 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -891,7 +891,7 @@ open_or_create_data_files( bool one_created = false; os_offset_t size; ulint flags; - ulint space; + ulint space = 0; ulint rounded_size_pages; char name[10000]; fil_space_crypt_t* crypt_data=NULL; @@ -1369,11 +1369,30 @@ srv_undo_tablespace_open( size = os_file_get_size(fh); ut_a(size != (os_offset_t) -1); + /* Load the tablespace into InnoDB's internal + data structures. */ + + const char* check_msg; + fil_space_crypt_t* crypt_data = NULL; + + /* Set the compressed page size to 0 (non-compressed) */ + flags = FSP_FLAGS_PAGE_SSIZE(); + + /* Read first page to find out does the crypt_info + exists on undo tablespace. */ + check_msg = fil_read_first_page( + fh, FALSE, &flags, &space, + NULL, &crypt_data, false); + ret = os_file_close(fh); ut_a(ret); - /* Load the tablespace into InnoDB's internal - data structures. */ + if (check_msg) { + ib_logf(IB_LOG_LEVEL_ERROR, + "%s in data file %s", + check_msg, name); + return (err); + } /* We set the biggest space id to the undo tablespace because InnoDB hasn't opened any other tablespace apart @@ -1381,10 +1400,8 @@ srv_undo_tablespace_open( fil_set_max_space_id_if_bigger(space); - /* Set the compressed page size to 0 (non-compressed) */ - flags = FSP_FLAGS_PAGE_SSIZE(); fil_space_create(name, space, flags, FIL_TABLESPACE, - NULL /* no encryption */, + crypt_data, true /* create */); ut_a(fil_validate()); @@ -1491,6 +1508,18 @@ srv_undo_tablespaces_init( n_undo_tablespaces = n_conf_tablespaces; undo_tablespace_ids[n_conf_tablespaces] = ULINT_UNDEFINED; + + if (backup_mode) { + ut_ad(!create_new_db); + /* MDEV-13561 FIXME: Determine srv_undo_space_id_start + from the undo001 file. */ + srv_undo_space_id_start = 1; + + for (i = 0; i < n_undo_tablespaces; i++) { + undo_tablespace_ids[i] + = i + srv_undo_space_id_start; + } + } } /* Open all the undo tablespaces that are currently in use. If we diff --git a/storage/xtradb/trx/trx0trx.cc b/storage/xtradb/trx/trx0trx.cc index 1d2f7ada54e..cfc1f83053a 100644 --- a/storage/xtradb/trx/trx0trx.cc +++ b/storage/xtradb/trx/trx0trx.cc @@ -720,25 +720,9 @@ trx_resurrect_insert( "InnoDB: Transaction " TRX_ID_FMT " was in the" " XA prepared state.\n", trx->id); - if (srv_force_recovery == 0) { - - /* XtraBackup should rollback prepared XA - transactions */ - if (IS_XTRABACKUP()) { - trx->state = TRX_STATE_ACTIVE; - } - else { - trx->state = TRX_STATE_PREPARED; - trx_sys->n_prepared_trx++; - trx_sys->n_prepared_recovered_trx++; - } - } else { - fprintf(stderr, - "InnoDB: Since innodb_force_recovery" - " > 0, we will rollback it anyway.\n"); - - trx->state = TRX_STATE_ACTIVE; - } + trx->state = TRX_STATE_PREPARED; + trx_sys->n_prepared_trx++; + trx_sys->n_prepared_recovered_trx++; } else { trx->state = TRX_STATE_COMMITTED_IN_MEMORY; } @@ -796,25 +780,14 @@ trx_resurrect_update_in_prepared_state( "InnoDB: Transaction " TRX_ID_FMT " was in the XA prepared state.\n", trx->id); - if (srv_force_recovery == 0) { - if (trx_state_eq(trx, TRX_STATE_NOT_STARTED)) { - if (!IS_XTRABACKUP()) { - trx_sys->n_prepared_trx++; - trx_sys->n_prepared_recovered_trx++; - } - } else { - ut_ad(trx_state_eq(trx, TRX_STATE_PREPARED)); - } - /* XtraBackup should rollback prepared XA - transactions */ - trx->state = IS_XTRABACKUP()?TRX_STATE_ACTIVE: TRX_STATE_PREPARED; + if (trx_state_eq(trx, TRX_STATE_NOT_STARTED)) { + trx_sys->n_prepared_trx++; + trx_sys->n_prepared_recovered_trx++; } else { - fprintf(stderr, - "InnoDB: Since innodb_force_recovery" - " > 0, we will rollback it anyway.\n"); - - trx->state = TRX_STATE_ACTIVE; + ut_ad(trx_state_eq(trx, TRX_STATE_PREPARED)); } + + trx->state = TRX_STATE_PREPARED; } else { trx->state = TRX_STATE_COMMITTED_IN_MEMORY; } diff --git a/win/packaging/create_msi.cmake.in b/win/packaging/create_msi.cmake.in index 4b05e61decd..c37f7ca9d1d 100644 --- a/win/packaging/create_msi.cmake.in +++ b/win/packaging/create_msi.cmake.in @@ -78,13 +78,6 @@ ELSE() ENDIF() SET(ENV{VS_UNICODE_OUTPUT}) -# Workaround for CMake bug#11452 -# Switch off the monolithic install -EXECUTE_PROCESS( - COMMAND ${CMAKE_COMMAND} -DCPACK_MONOLITHIC_INSTALL=0 ${CMAKE_BINARY_DIR} - OUTPUT_QUIET -) - INCLUDE(${CMAKE_BINARY_DIR}/CPackConfig.cmake) @@ -441,11 +434,4 @@ ENDIF() CONFIGURE_FILE(${CPACK_PACKAGE_FILE_NAME}.msi ${CMAKE_BINARY_DIR}/${CPACK_PACKAGE_FILE_NAME}.msi COPYONLY) - -# Workaround for CMake bug#11452 -# Switch monolithic install on again -EXECUTE_PROCESS( - COMMAND ${CMAKE_COMMAND} -DCPACK_MONOLITHIC_INSTALL=1 ${CMAKE_BINARY_DIR} - OUTPUT_QUIET -) |