diff options
51 files changed, 523 insertions, 798 deletions
diff --git a/client/mysql.cc b/client/mysql.cc index e4d110ba9eb..4b3a39fe302 100644 --- a/client/mysql.cc +++ b/client/mysql.cc @@ -1094,7 +1094,7 @@ inline bool is_delimiter_command(char *name, ulong len) only name(first DELIMITER_NAME_LEN bytes) is checked. */ return (len >= DELIMITER_NAME_LEN && - !my_strnncoll(charset_info, (uchar*) name, DELIMITER_NAME_LEN, + !my_strnncoll(&my_charset_latin1, (uchar*) name, DELIMITER_NAME_LEN, (uchar *) DELIMITER_NAME, DELIMITER_NAME_LEN)); } diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index bc37f347532..0b501970efa 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -46,6 +46,7 @@ Place, Suite 330, Boston, MA 02111-1307 USA #include <ut0mem.h> #include <srv0start.h> #include <fil0fil.h> +#include <trx0sys.h> #include <set> #include <string> #include <mysqld.h> @@ -1680,26 +1681,30 @@ copy_back() ut_crc32_init(); /* copy undo tablespaces */ - if (srv_undo_tablespaces > 0) { - dst_dir = (srv_undo_dir && *srv_undo_dir) - ? srv_undo_dir : mysql_data_home; - ds_data = ds_create(dst_dir, DS_TYPE_LOCAL); + dst_dir = (srv_undo_dir && *srv_undo_dir) + ? srv_undo_dir : mysql_data_home; - for (ulong i = 1; i <= srv_undo_tablespaces; i++) { - char filename[20]; - sprintf(filename, "undo%03lu", i); - if (!(ret = copy_or_move_file(filename, filename, - dst_dir, 1))) { - goto cleanup; - } - } + ds_data = ds_create(dst_dir, DS_TYPE_LOCAL); - ds_destroy(ds_data); - ds_data = NULL; + for (uint i = 1; i <= TRX_SYS_MAX_UNDO_SPACES; i++) { + char filename[20]; + sprintf(filename, "undo%03u", i); + if (!file_exists(filename)) { + break; + } + if (!(ret = copy_or_move_file(filename, filename, + dst_dir, 1))) { + goto cleanup; + } } + ds_destroy(ds_data); + ds_data = NULL; + + /* copy redo logs */ + dst_dir = (srv_log_group_home_dir && *srv_log_group_home_dir) ? srv_log_group_home_dir : mysql_data_home; @@ -1825,7 +1830,7 @@ copy_back() } } - /* copy buufer pool dump */ + /* copy buffer pool dump */ if (innobase_buffer_pool_filename) { const char *src_name; diff --git a/mysql-test/include/check-testcase.test b/mysql-test/include/check-testcase.test index 3b2c2a46590..a282201857e 100644 --- a/mysql-test/include/check-testcase.test +++ b/mysql-test/include/check-testcase.test @@ -82,6 +82,8 @@ call mtr.check_testcase(); let $datadir=`select @@datadir`; list_files $datadir mysql_upgrade_info; +list_files $datadir/test #sql*; +list_files $datadir/mysql #sql*; --enable_query_log diff --git a/mysql-test/r/delimiter_command_case_sensitivity.result b/mysql-test/r/delimiter_command_case_sensitivity.result new file mode 100644 index 00000000000..6ed281c757a --- /dev/null +++ b/mysql-test/r/delimiter_command_case_sensitivity.result @@ -0,0 +1,2 @@ +1 +1 diff --git a/mysql-test/r/type_date.result b/mysql-test/r/type_date.result index 4b0e4a61c64..a9e81b8071d 100644 --- a/mysql-test/r/type_date.result +++ b/mysql-test/r/type_date.result @@ -843,5 +843,16 @@ Warning 1292 Incorrect datetime value: '1' Warning 1292 Incorrect datetime value: '1' DROP TABLE t1; # +# MDEV-14221 Assertion `0' failed in Item::field_type_for_temporal_comparison +# +CREATE TABLE t1 (d DATE); +INSERT INTO t1 VALUES ('1985-05-13'),('1989-12-24'); +SELECT d, COUNT(*) FROM t1 GROUP BY d WITH ROLLUP HAVING CASE d WHEN '2017-05-25' THEN 0 ELSE 1 END; +d COUNT(*) +1985-05-13 1 +1989-12-24 1 +NULL 2 +DROP TABLE t1; +# # End of 10.1 tests # diff --git a/mysql-test/suite/plugins/r/server_audit.result b/mysql-test/suite/plugins/r/server_audit.result index 45206c8bcb4..0d02ae47586 100644 --- a/mysql-test/suite/plugins/r/server_audit.result +++ b/mysql-test/suite/plugins/r/server_audit.result @@ -52,6 +52,7 @@ alter table t1 rename renamed_t1; set global server_audit_events='connect,query'; select 1, 2, +# comment 3; 1 2 3 1 2 3 @@ -170,7 +171,9 @@ id 2 CREATE USER u1 IDENTIFIED BY 'pwd-123'; GRANT ALL ON sa_db TO u2 IDENTIFIED BY "pwd-321"; -SET PASSWORD FOR u1 = PASSWORD('pwd 098'); +SET PASSWORD +# comment +FOR u1 = PASSWORD('pwd 098'); SET PASSWORD FOR u1=<secret>; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '<secret>' at line 1 CREATE USER u3 IDENTIFIED BY ''; @@ -262,7 +265,7 @@ TIME,HOSTNAME,root,localhost,ID,ID,WRITE,mysql,index_stats, TIME,HOSTNAME,root,localhost,ID,ID,RENAME,test,t1|test.renamed_t1, TIME,HOSTNAME,root,localhost,ID,ID,QUERY,test,'alter table t1 rename renamed_t1',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,test,'set global server_audit_events=\'connect,query\'',0 -TIME,HOSTNAME,root,localhost,ID,ID,QUERY,test,'select 1, 2, 3',0 +TIME,HOSTNAME,root,localhost,ID,ID,QUERY,test,'select 1,\n2,\n# comment\n3',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,test,'insert into t2 values (1), (2)',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,test,'select * from t2',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,test,'select * from t_doesnt_exist',ID @@ -345,7 +348,7 @@ TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'/*! select 2*/',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'/*comment*/ select 2',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'CREATE USER u1 IDENTIFIED BY *****',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'GRANT ALL ON sa_db TO u2 IDENTIFIED BY *****',0 -TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'SET PASSWORD FOR u1 = PASSWORD(*****)',0 +TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'SET PASSWORD \n# comment\nFOR u1 = PASSWORD(*****)',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'SET PASSWORD FOR u1=<secret>',ID TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'CREATE USER u3 IDENTIFIED BY *****',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'drop user u1, u2, u3',0 diff --git a/mysql-test/suite/plugins/r/thread_pool_server_audit.result b/mysql-test/suite/plugins/r/thread_pool_server_audit.result index 45206c8bcb4..cf09ccb3a51 100644 --- a/mysql-test/suite/plugins/r/thread_pool_server_audit.result +++ b/mysql-test/suite/plugins/r/thread_pool_server_audit.result @@ -52,6 +52,7 @@ alter table t1 rename renamed_t1; set global server_audit_events='connect,query'; select 1, 2, +# comment 3; 1 2 3 1 2 3 @@ -170,7 +171,9 @@ id 2 CREATE USER u1 IDENTIFIED BY 'pwd-123'; GRANT ALL ON sa_db TO u2 IDENTIFIED BY "pwd-321"; -SET PASSWORD FOR u1 = PASSWORD('pwd 098'); +SET PASSWORD +# comment +FOR u1 = PASSWORD('pwd 098'); SET PASSWORD FOR u1=<secret>; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '<secret>' at line 1 CREATE USER u3 IDENTIFIED BY ''; @@ -262,7 +265,7 @@ TIME,HOSTNAME,root,localhost,ID,ID,WRITE,mysql,index_stats, TIME,HOSTNAME,root,localhost,ID,ID,RENAME,test,t1|test.renamed_t1, TIME,HOSTNAME,root,localhost,ID,ID,QUERY,test,'alter table t1 rename renamed_t1',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,test,'set global server_audit_events=\'connect,query\'',0 -TIME,HOSTNAME,root,localhost,ID,ID,QUERY,test,'select 1, 2, 3',0 +TIME,HOSTNAME,root,localhost,ID,ID,QUERY,test,'select 1,\n2,\n# comment\n3',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,test,'insert into t2 values (1), (2)',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,test,'select * from t2',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,test,'select * from t_doesnt_exist',ID @@ -345,7 +348,7 @@ TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'/*! select 2*/',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'/*comment*/ select 2',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'CREATE USER u1 IDENTIFIED BY *****',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'GRANT ALL ON sa_db TO u2 IDENTIFIED BY *****',0 -TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'SET PASSWORD FOR u1 = PASSWORD(*****)',0 +TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'SET PASSWORD\n# comment\nFOR u1 = PASSWORD(*****)',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'SET PASSWORD FOR u1=<secret>',ID TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'CREATE USER u3 IDENTIFIED BY *****',0 TIME,HOSTNAME,root,localhost,ID,ID,QUERY,sa_db,'drop user u1, u2, u3',0 diff --git a/mysql-test/suite/plugins/t/server_audit.test b/mysql-test/suite/plugins/t/server_audit.test index 52428909c3b..9be0d5556f0 100644 --- a/mysql-test/suite/plugins/t/server_audit.test +++ b/mysql-test/suite/plugins/t/server_audit.test @@ -38,6 +38,7 @@ alter table t1 rename renamed_t1; set global server_audit_events='connect,query'; select 1, 2, +# comment 3; insert into t2 values (1), (2); select * from t2; @@ -106,7 +107,9 @@ insert into t1 values (1), (2); select * from t1; CREATE USER u1 IDENTIFIED BY 'pwd-123'; GRANT ALL ON sa_db TO u2 IDENTIFIED BY "pwd-321"; -SET PASSWORD FOR u1 = PASSWORD('pwd 098'); +SET PASSWORD +# comment +FOR u1 = PASSWORD('pwd 098'); --error 1064 SET PASSWORD FOR u1=<secret>; CREATE USER u3 IDENTIFIED BY ''; diff --git a/mysql-test/suite/plugins/t/thread_pool_server_audit.test b/mysql-test/suite/plugins/t/thread_pool_server_audit.test index 626d4136c47..724000c9789 100644 --- a/mysql-test/suite/plugins/t/thread_pool_server_audit.test +++ b/mysql-test/suite/plugins/t/thread_pool_server_audit.test @@ -38,6 +38,7 @@ alter table t1 rename renamed_t1; set global server_audit_events='connect,query'; select 1, 2, +# comment 3; insert into t2 values (1), (2); select * from t2; @@ -106,7 +107,9 @@ insert into t1 values (1), (2); select * from t1; CREATE USER u1 IDENTIFIED BY 'pwd-123'; GRANT ALL ON sa_db TO u2 IDENTIFIED BY "pwd-321"; -SET PASSWORD FOR u1 = PASSWORD('pwd 098'); +SET PASSWORD +# comment +FOR u1 = PASSWORD('pwd 098'); --error 1064 SET PASSWORD FOR u1=<secret>; CREATE USER u3 IDENTIFIED BY ''; diff --git a/mysql-test/suite/sys_vars/r/innodb_buffer_pool_dump_now_basic.result b/mysql-test/suite/sys_vars/r/innodb_buffer_pool_dump_now_basic.result index 9c3a37f892b..522d5731a6d 100644 --- a/mysql-test/suite/sys_vars/r/innodb_buffer_pool_dump_now_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_buffer_pool_dump_now_basic.result @@ -1,7 +1,8 @@ -SET @orig = @@global.innodb_buffer_pool_dump_now; -SELECT @orig; -@orig +SELECT @@global.innodb_buffer_pool_dump_now; +@@global.innodb_buffer_pool_dump_now 0 +SELECT variable_value INTO @old_dump_status FROM information_schema.global_status +WHERE LOWER(variable_name) = 'innodb_buffer_pool_dump_status'; SET GLOBAL innodb_buffer_pool_dump_now = ON; SELECT @@global.innodb_buffer_pool_dump_now; @@global.innodb_buffer_pool_dump_now diff --git a/mysql-test/suite/sys_vars/r/innodb_buffer_pool_dump_pct_basic.result b/mysql-test/suite/sys_vars/r/innodb_buffer_pool_dump_pct_basic.result index 485136ccc4c..93a85ffbf43 100644 --- a/mysql-test/suite/sys_vars/r/innodb_buffer_pool_dump_pct_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_buffer_pool_dump_pct_basic.result @@ -1,7 +1,9 @@ SET @orig = @@global.innodb_buffer_pool_dump_pct; -SELECT @@global.innodb_buffer_pool_dump_pct; -@@global.innodb_buffer_pool_dump_pct +SELECT @orig; +@orig 25 +SET GLOBAL innodb_buffer_pool_dump_pct=3; +# Do the dump SET GLOBAL innodb_buffer_pool_dump_pct=20; SELECT @@global.innodb_buffer_pool_dump_pct; @@global.innodb_buffer_pool_dump_pct diff --git a/mysql-test/suite/sys_vars/r/innodb_buffer_pool_load_now_basic.result b/mysql-test/suite/sys_vars/r/innodb_buffer_pool_load_now_basic.result index 3185d1ca170..eebed4d0f4a 100644 --- a/mysql-test/suite/sys_vars/r/innodb_buffer_pool_load_now_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_buffer_pool_load_now_basic.result @@ -1,8 +1,6 @@ -SET @orig = @@global.innodb_buffer_pool_load_now; -SELECT @orig; -@orig +SELECT @@global.innodb_buffer_pool_load_now; +@@global.innodb_buffer_pool_load_now 0 -SET GLOBAL innodb_buffer_pool_dump_now = ON; SET GLOBAL innodb_buffer_pool_load_now = ON; SELECT variable_value FROM information_schema.global_status diff --git a/mysql-test/suite/sys_vars/t/innodb_buffer_pool_dump_now_basic.test b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_dump_now_basic.test index 0bae347428e..8c5f8fa7bf0 100644 --- a/mysql-test/suite/sys_vars/t/innodb_buffer_pool_dump_now_basic.test +++ b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_dump_now_basic.test @@ -5,8 +5,31 @@ -- source include/have_innodb.inc # Check the default value -SET @orig = @@global.innodb_buffer_pool_dump_now; -SELECT @orig; +SELECT @@global.innodb_buffer_pool_dump_now; + +-- let $file = `SELECT CONCAT(@@datadir, @@global.innodb_buffer_pool_filename)` +-- error 0,1 +-- remove_file $file + +SELECT variable_value INTO @old_dump_status FROM information_schema.global_status + WHERE LOWER(variable_name) = 'innodb_buffer_pool_dump_status'; + +# A previous test could have run buffer pool dump already; +# in this case we want to make sure that the current time is different +# from the timestamp in the status variable. +# We should have had a smart wait condition here, like the commented one below, +# let $wait_condition = +# SELECT TRIM(SUBSTR('$old_status', -8)) != DATE_FORMAT(CURTIME(), '%k:%i:%s'); +# -- source include/wait_condition.inc + +# ... but we can't because of MDEV-9867, so there will be just sleep instead. +# And it might be not enough to sleep one second, so we'll have to sleep two. + +if (`SELECT variable_value LIKE '%completed at%' FROM information_schema.global_status + WHERE LOWER(variable_name) = 'innodb_buffer_pool_dump_status'`) +{ + -- sleep 2 +} # Do the dump SET GLOBAL innodb_buffer_pool_dump_now = ON; @@ -15,11 +38,11 @@ SELECT @@global.innodb_buffer_pool_dump_now; # Wait for the dump to complete let $wait_condition = - SELECT SUBSTR(variable_value, 1, 33) = 'Buffer pool(s) dump completed at ' + SELECT variable_value != @old_dump_status + AND SUBSTR(variable_value, 1, 33) = 'Buffer pool(s) dump completed at ' FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_buffer_pool_dump_status'; -- source include/wait_condition.inc # Confirm that the dump file has been created --- let $file = `SELECT CONCAT(@@datadir, @@global.innodb_buffer_pool_filename)` -- file_exists $file diff --git a/mysql-test/suite/sys_vars/t/innodb_buffer_pool_dump_pct_basic.test b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_dump_pct_basic.test index 99d3f79fb27..9b4edafb17e 100644 --- a/mysql-test/suite/sys_vars/t/innodb_buffer_pool_dump_pct_basic.test +++ b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_dump_pct_basic.test @@ -11,9 +11,17 @@ # Save the default value SET @orig = @@global.innodb_buffer_pool_dump_pct; +SELECT @orig; -# Check the default value -SELECT @@global.innodb_buffer_pool_dump_pct; +SET GLOBAL innodb_buffer_pool_dump_pct=3; + +--echo # Do the dump + +--disable_query_log +--disable_result_log +--source innodb_buffer_pool_dump_now_basic.test +--enable_result_log +--enable_query_log # Set the valid value SET GLOBAL innodb_buffer_pool_dump_pct=20; diff --git a/mysql-test/suite/sys_vars/t/innodb_buffer_pool_load_now_basic.opt b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_load_now_basic.opt new file mode 100644 index 00000000000..e462be3c368 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_load_now_basic.opt @@ -0,0 +1 @@ +--innodb-buffer-pool-load-at-startup=off diff --git a/mysql-test/suite/sys_vars/t/innodb_buffer_pool_load_now_basic.test b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_load_now_basic.test index a260dc2516b..15536d338e1 100644 --- a/mysql-test/suite/sys_vars/t/innodb_buffer_pool_load_now_basic.test +++ b/mysql-test/suite/sys_vars/t/innodb_buffer_pool_load_now_basic.test @@ -5,44 +5,22 @@ -- source include/have_innodb.inc # Check the default value -SET @orig = @@global.innodb_buffer_pool_load_now; -SELECT @orig; +SELECT @@global.innodb_buffer_pool_load_now; -let $old_status= `SELECT variable_value FROM information_schema.global_status - WHERE LOWER(variable_name) = 'innodb_buffer_pool_dump_status'`; +# Make sure there is a dump file to load -# A previous test could have run buffer pool dump already; -# in this case we want to make sure that the current time is different -# from the timestamp in the status variable. -# We should have had a smart wait condition here, like the commented one below, -# but we can't because of MDEV-9867, so there will be just sleep instead. -# And it might be not enough to sleep one second, so we'll have to sleep two. -# let $wait_condition = -# SELECT TRIM(SUBSTR('$old_status', -8)) != DATE_FORMAT(CURTIME(), '%k:%i:%s'); -# -- source include/wait_condition.inc -if (`SELECT count(*) > 0 FROM information_schema.global_status - WHERE (LOWER(variable_name) = 'innodb_buffer_pool_dump_status' or - LOWER(variable_name) = 'innodb_buffer_pool_load_status') - and variable_value LIKE '%completed at%'`) -{ - -- sleep 2 -} -# Do the dump -SET GLOBAL innodb_buffer_pool_dump_now = ON; - -# Wait for the dump to complete -let $wait_condition = - SELECT variable_value != '$old_status' - AND SUBSTR(variable_value, 1, 33) = 'Buffer pool(s) dump completed at ' - FROM information_schema.global_status - WHERE LOWER(variable_name) = 'innodb_buffer_pool_dump_status'; ---disable_warnings --- source include/wait_condition.inc ---enable_warnings - -# Confirm the file is really created -- let $file = `SELECT CONCAT(@@datadir, @@global.innodb_buffer_pool_filename)` +-- error 0,1 -- file_exists $file +if ($errno) +{ + # Dump file does not exist, get it created + --disable_query_log + --disable_result_log + --source innodb_buffer_pool_dump_now_basic.test + --enable_result_log + --enable_query_log +} let $old_load_status= `SELECT variable_value FROM information_schema.global_status diff --git a/mysql-test/t/delimiter_case_mdev_10728.sql b/mysql-test/t/delimiter_case_mdev_10728.sql new file mode 100644 index 00000000000..72a1dcd9a9e --- /dev/null +++ b/mysql-test/t/delimiter_case_mdev_10728.sql @@ -0,0 +1,3 @@ +DeLiMiTeR A; +SELECT 1 A; +delimiter ; diff --git a/mysql-test/t/delimiter_command_case_sensitivity.test b/mysql-test/t/delimiter_command_case_sensitivity.test new file mode 100644 index 00000000000..2b06a35c723 --- /dev/null +++ b/mysql-test/t/delimiter_command_case_sensitivity.test @@ -0,0 +1,2 @@ +# MDEV-10728 +--exec $MYSQL --default-character-set=binary < "t/delimiter_case_mdev_10728.sql" diff --git a/mysql-test/t/type_date.test b/mysql-test/t/type_date.test index 11924f696db..b7fdf7b93b7 100644 --- a/mysql-test/t/type_date.test +++ b/mysql-test/t/type_date.test @@ -576,6 +576,15 @@ SELECT DATE(a), DATE(b), DATE(c) FROM t1; SELECT DATE(COALESCE(a)), DATE(COALESCE(b)), DATE(COALESCE(c)) FROM t1; DROP TABLE t1; +--echo # +--echo # MDEV-14221 Assertion `0' failed in Item::field_type_for_temporal_comparison +--echo # + +CREATE TABLE t1 (d DATE); +INSERT INTO t1 VALUES ('1985-05-13'),('1989-12-24'); +SELECT d, COUNT(*) FROM t1 GROUP BY d WITH ROLLUP HAVING CASE d WHEN '2017-05-25' THEN 0 ELSE 1 END; +DROP TABLE t1; + --echo # --echo # End of 10.1 tests diff --git a/mysql-test/t/view.test b/mysql-test/t/view.test index a9764bf0668..9e8e483929f 100644 --- a/mysql-test/t/view.test +++ b/mysql-test/t/view.test @@ -1,3 +1,4 @@ +--source include/have_partition.inc # Save the initial number of concurrent sessions. --source include/count_sessions.inc diff --git a/plugin/server_audit/server_audit.c b/plugin/server_audit/server_audit.c index 87a18e47fc5..b75f6b9a863 100644 --- a/plugin/server_audit/server_audit.c +++ b/plugin/server_audit/server_audit.c @@ -15,7 +15,7 @@ #define PLUGIN_VERSION 0x104 -#define PLUGIN_STR_VERSION "1.4.2" +#define PLUGIN_STR_VERSION "1.4.3" #define _my_thread_var loc_thread_var @@ -1118,6 +1118,21 @@ do { \ } while(0) +#define ESC_MAP_SIZE 0x60 +static const char esc_map[ESC_MAP_SIZE]= +{ + 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, '\'', 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '\\', 0, 0, 0 +}; + +static char escaped_char(char c) +{ + return ((unsigned char ) c) >= ESC_MAP_SIZE ? 0 : esc_map[(unsigned char) c]; +} static void setup_connection_initdb(struct connection_info *cn, @@ -1324,21 +1339,16 @@ static size_t escape_string(const char *str, unsigned int len, const char *res_end= result + result_len - 2; while (len) { + char esc_c; + if (result >= res_end) break; - if (*str == '\'') - { - if (result+1 >= res_end) - break; - *(result++)= '\\'; - *(result++)= '\''; - } - else if (*str == '\\') + if ((esc_c= escaped_char(*str))) { if (result+1 >= res_end) break; *(result++)= '\\'; - *(result++)= '\\'; + *(result++)= esc_c; } else if (is_space(*str)) *(result++)= ' '; @@ -1427,19 +1437,12 @@ static size_t escape_string_hide_passwords(const char *str, unsigned int len, no_password: if (result >= res_end) break; - if (*str == '\'') + if ((b_char= escaped_char(*str))) { if (result+1 >= res_end) break; *(result++)= '\\'; - *(result++)= '\''; - } - else if (*str == '\\') - { - if (result+1 >= res_end) - break; - *(result++)= '\\'; - *(result++)= '\\'; + *(result++)= b_char; } else if (is_space(*str)) *(result++)= ' '; diff --git a/sql/datadict.cc b/sql/datadict.cc index 103a33214ae..ec3d65f0113 100644 --- a/sql/datadict.cc +++ b/sql/datadict.cc @@ -45,6 +45,8 @@ static int read_string(File file, uchar**to, size_t length) engine_name is a LEX_STRING, where engine_name->str must point to a buffer of at least NAME_CHAR_LEN+1 bytes. + If engine_name is 0, then the function will only test if the file is a + view or not @retval FRMTYPE_ERROR error @retval FRMTYPE_TABLE table @@ -72,12 +74,23 @@ frm_type_enum dd_frm_type(THD *thd, char *path, LEX_STRING *engine_name) goto err; } + /* + We return FRMTYPE_TABLE if we can read the .frm file. This allows us + to drop a bad .frm file with DROP TABLE + */ type= FRMTYPE_TABLE; - if (!is_binary_frm_header(header) || !engine_name) + /* engine_name is 0 if we only want to know if table is view or not */ + if (!engine_name) goto err; + /* Initialize engine name in case we are not able to find it out */ engine_name->length= 0; + engine_name->str[0]= 0; + + if (!is_binary_frm_header(header)) + goto err; + dbt= header[3]; /* cannot use ha_resolve_by_legacy_type without a THD */ diff --git a/sql/handler.cc b/sql/handler.cc index 1f3df447f24..7eed722a971 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -5072,10 +5072,15 @@ bool ha_table_exists(THD *thd, const char *db, const char *table_name, { char engine_buf[NAME_CHAR_LEN + 1]; LEX_STRING engine= { engine_buf, 0 }; + frm_type_enum type; - if (dd_frm_type(thd, path, &engine) != FRMTYPE_VIEW) + if ((type= dd_frm_type(thd, path, &engine)) == FRMTYPE_ERROR) + DBUG_RETURN(0); + + if (type != FRMTYPE_VIEW) { - plugin_ref p= plugin_lock_by_name(thd, &engine, MYSQL_STORAGE_ENGINE_PLUGIN); + plugin_ref p= plugin_lock_by_name(thd, &engine, + MYSQL_STORAGE_ENGINE_PLUGIN); *hton= p ? plugin_hton(p) : NULL; if (*hton) // verify that the table really exists diff --git a/sql/item.h b/sql/item.h index 9b94a08c34e..d7daea4853d 100644 --- a/sql/item.h +++ b/sql/item.h @@ -2768,6 +2768,17 @@ public: Field *result_field; Item_null_result(THD *thd): Item_null(thd), result_field(0) {} bool is_result_field() { return result_field != 0; } +#if MARIADB_VERSION_ID < 100300 + enum_field_types field_type() const + { + return result_field->type(); + } +#else + const Type_handler *type_handler() const + { + return result_field->type_handler(); + } +#endif void save_in_result_field(bool no_conversions) { save_in_field(result_field, no_conversions); diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 684f3fe0b36..3a201f5cef4 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -4168,7 +4168,8 @@ buf_page_get_gen( ulint retries = 0; buf_pool_t* buf_pool = buf_pool_get(page_id); - ut_ad(mtr->is_active()); + ut_ad((mtr == NULL) == (mode == BUF_EVICT_IF_IN_POOL)); + ut_ad(!mtr || mtr->is_active()); ut_ad((rw_latch == RW_S_LATCH) || (rw_latch == RW_X_LATCH) || (rw_latch == RW_SX_LATCH) @@ -4180,29 +4181,31 @@ buf_page_get_gen( #ifdef UNIV_DEBUG switch (mode) { + case BUF_EVICT_IF_IN_POOL: + /* After DISCARD TABLESPACE, the tablespace would not exist, + but in IMPORT TABLESPACE, PageConverter::operator() must + replace any old pages, which were not evicted during DISCARD. + Skip the assertion on space_page_size. */ + break; + default: + ut_error; case BUF_GET_NO_LATCH: ut_ad(rw_latch == RW_NO_LATCH); - break; + /* fall through */ case BUF_GET: case BUF_GET_IF_IN_POOL: case BUF_PEEK_IF_IN_POOL: case BUF_GET_IF_IN_POOL_OR_WATCH: case BUF_GET_POSSIBLY_FREED: - break; - default: - ut_error; + bool found; + const page_size_t& space_page_size + = fil_space_get_page_size(page_id.space(), &found); + ut_ad(found); + ut_ad(page_size.equals_to(space_page_size)); } - - bool found; - const page_size_t& space_page_size - = fil_space_get_page_size(page_id.space(), &found); - - ut_ad(found); - - ut_ad(page_size.equals_to(space_page_size)); #endif /* UNIV_DEBUG */ - ut_ad(!ibuf_inside(mtr) + ut_ad(!mtr || !ibuf_inside(mtr) || ibuf_page_low(page_id, page_size, FALSE, file, line, NULL)); buf_pool->stat.n_page_gets++; @@ -4263,7 +4266,24 @@ loop: sure that no state change takes place. */ fix_block = block; - buf_block_fix(fix_block); + if (fsp_is_system_temporary(page_id.space())) { + /* For temporary tablespace, + the mutex is being used for + synchronization between user + thread and flush thread, + instead of block->lock. See + buf_flush_page() for the flush + thread counterpart. */ + + BPageMutex* fix_mutex + = buf_page_get_mutex( + &fix_block->page); + mutex_enter(fix_mutex); + buf_block_fix(fix_block); + mutex_exit(fix_mutex); + } else { + buf_block_fix(fix_block); + } /* Now safe to release page_hash mutex */ rw_lock_x_unlock(hash_lock); @@ -4273,13 +4293,15 @@ loop: rw_lock_x_unlock(hash_lock); } - if (mode == BUF_GET_IF_IN_POOL - || mode == BUF_PEEK_IF_IN_POOL - || mode == BUF_GET_IF_IN_POOL_OR_WATCH) { - + switch (mode) { + case BUF_GET_IF_IN_POOL: + case BUF_GET_IF_IN_POOL_OR_WATCH: + case BUF_PEEK_IF_IN_POOL: + case BUF_EVICT_IF_IN_POOL: +#ifdef UNIV_SYNC_DEBUG ut_ad(!rw_lock_own(hash_lock, RW_LOCK_X)); ut_ad(!rw_lock_own(hash_lock, RW_LOCK_S)); - +#endif /* UNIV_SYNC_DEBUG */ return(NULL); } @@ -4354,15 +4376,29 @@ loop: fix_block = block; } - buf_block_fix(fix_block); + if (fsp_is_system_temporary(page_id.space())) { + /* For temporary tablespace, the mutex is being used + for synchronization between user thread and flush + thread, instead of block->lock. See buf_flush_page() + for the flush thread counterpart. */ + BPageMutex* fix_mutex = buf_page_get_mutex( + &fix_block->page); + mutex_enter(fix_mutex); + buf_block_fix(fix_block); + mutex_exit(fix_mutex); + } else { + buf_block_fix(fix_block); + } /* Now safe to release page_hash mutex */ rw_lock_s_unlock(hash_lock); got_block: - if (mode == BUF_GET_IF_IN_POOL || mode == BUF_PEEK_IF_IN_POOL) { - + switch (mode) { + case BUF_GET_IF_IN_POOL: + case BUF_PEEK_IF_IN_POOL: + case BUF_EVICT_IF_IN_POOL: buf_page_t* fix_page = &fix_block->page; BPageMutex* fix_mutex = buf_page_get_mutex(fix_page); mutex_enter(fix_mutex); @@ -4394,6 +4430,20 @@ got_block: os_thread_sleep(WAIT_FOR_WRITE); goto loop; } + + if (UNIV_UNLIKELY(mode == BUF_EVICT_IF_IN_POOL)) { +evict_from_pool: + ut_ad(!fix_block->page.oldest_modification); + buf_pool_mutex_enter(buf_pool); + buf_block_unfix(fix_block); + + if (!buf_LRU_free_page(&fix_block->page, true)) { + ut_ad(0); + } + + buf_pool_mutex_exit(buf_pool); + return(NULL); + } break; case BUF_BLOCK_ZIP_PAGE: @@ -4426,6 +4476,10 @@ got_block: goto loop; } + if (UNIV_UNLIKELY(mode == BUF_EVICT_IF_IN_POOL)) { + goto evict_from_pool; + } + /* Buffer-fix the block so that it cannot be evicted or relocated while we are attempting to allocate an uncompressed page. */ diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 8fa9c0f3d20..4661a4e0d14 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -3830,8 +3830,7 @@ FlushObserver::flush() that will be freed by the clean-up of the ALTER operation. (Maybe, instead of buf_pool->flush_list, use a dedicated list for pages on which redo logging has been disabled.) */ - buf_LRU_flush_or_remove_pages( - m_space_id, BUF_REMOVE_FLUSH_WRITE, m_trx); + buf_LRU_flush_or_remove_pages(m_space_id, m_trx); /* Wait for all dirty pages were flushed. */ for (ulint i = 0; i < srv_buf_pool_instances; i++) { diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 2137760e815..9d0d9627d26 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -586,8 +586,8 @@ rescan: /* If flush observer is NULL, flush page for space id, or flush page for flush observer. */ - if ((observer != NULL && observer != bpage->flush_observer) - || (observer == NULL && id != bpage->id.space())) { + if (observer ? (observer != bpage->flush_observer) + : (id != bpage->id.space())) { /* Skip this block, as it does not belong to the target space. */ @@ -657,24 +657,27 @@ rescan: return(all_freed ? DB_SUCCESS : DB_FAIL); } -/******************************************************************//** -Remove or flush all the dirty pages that belong to a given tablespace +/** Remove or flush all the dirty pages that belong to a given tablespace inside a specific buffer pool instance. The pages will remain in the LRU list and will be evicted from the LRU list as they age and move towards -the tail of the LRU list. */ +the tail of the LRU list. +@param[in,out] buf_pool buffer pool +@param[in] id tablespace identifier +@param[in] observer flush observer, + or NULL if the files should not be written to +@param[in] trx transaction (to check for interrupt), + or NULL if the files should not be written to +*/ static void buf_flush_dirty_pages( -/*==================*/ - buf_pool_t* buf_pool, /*!< buffer pool instance */ - ulint id, /*!< in: space id */ - FlushObserver* observer, /*!< in: flush observer */ - bool flush, /*!< in: flush to disk if true otherwise - remove the pages without flushing */ - const trx_t* trx) /*!< to check if the operation must - be interrupted */ + buf_pool_t* buf_pool, + ulint id, + FlushObserver* observer, + const trx_t* trx) { dberr_t err; + bool flush = trx != NULL; do { buf_pool_mutex_enter(buf_pool); @@ -708,238 +711,30 @@ buf_flush_dirty_pages( || buf_pool_get_dirty_pages_count(buf_pool, id, observer) == 0); } -/******************************************************************//** -Remove all pages that belong to a given tablespace inside a specific -buffer pool instance when we are DISCARDing the tablespace. */ -static -void -buf_LRU_remove_all_pages( -/*=====================*/ - buf_pool_t* buf_pool, /*!< buffer pool instance */ - ulint id) /*!< in: space id */ -{ - buf_page_t* bpage; - ibool all_freed; - -scan_again: - buf_pool_mutex_enter(buf_pool); - - all_freed = TRUE; - - for (bpage = UT_LIST_GET_LAST(buf_pool->LRU); - bpage != NULL; - /* No op */) { - - rw_lock_t* hash_lock; - buf_page_t* prev_bpage; - BPageMutex* block_mutex; - - ut_a(buf_page_in_file(bpage)); - ut_ad(bpage->in_LRU_list); - - prev_bpage = UT_LIST_GET_PREV(LRU, bpage); - - /* bpage->id.space() and bpage->io_fix are protected by - buf_pool->mutex and the block_mutex. It is safe to check - them while holding buf_pool->mutex only. */ - - if (bpage->id.space() != id) { - /* Skip this block, as it does not belong to - the space that is being invalidated. */ - goto next_page; - } else if (buf_page_get_io_fix(bpage) != BUF_IO_NONE) { - /* We cannot remove this page during this scan - yet; maybe the system is currently reading it - in, or flushing the modifications to the file */ - - all_freed = FALSE; - goto next_page; - } else { - hash_lock = buf_page_hash_lock_get(buf_pool, bpage->id); - - rw_lock_x_lock(hash_lock); - - block_mutex = buf_page_get_mutex(bpage); - - mutex_enter(block_mutex); - - if (bpage->buf_fix_count > 0) { - - mutex_exit(block_mutex); - - rw_lock_x_unlock(hash_lock); - - /* We cannot remove this page during - this scan yet; maybe the system is - currently reading it in, or flushing - the modifications to the file */ - - all_freed = FALSE; - - goto next_page; - } - } - - ut_ad(mutex_own(block_mutex)); - - DBUG_PRINT("ib_buf", ("evict page %u:%u" - " state %u", - bpage->id.space(), - bpage->id.page_no(), - bpage->state)); -#ifdef BTR_CUR_HASH_ADAPT - if (buf_page_get_state(bpage) != BUF_BLOCK_FILE_PAGE) { - /* Do nothing, because the adaptive hash index - covers uncompressed pages only. */ - } else if (((buf_block_t*) bpage)->index) { - buf_pool_mutex_exit(buf_pool); - - rw_lock_x_unlock(hash_lock); - - mutex_exit(block_mutex); - - /* Note that the following call will acquire - and release block->lock X-latch. - Note that the table cannot be evicted during - the execution of ALTER TABLE...DISCARD TABLESPACE - because MySQL is keeping the table handle open. */ - - btr_search_drop_page_hash_when_freed( - bpage->id, bpage->size); - - goto scan_again; - } else { - /* This debug check uses a dirty read that could - theoretically cause false positives while - buf_pool_clear_hash_index() is executing, - if the writes to block->index=NULL and - block->n_pointers=0 are reordered. - (Other conflicting access paths to the adaptive hash - index should not be possible, because when a - tablespace is being discarded or dropped, there must - be no concurrect access to the contained tables.) */ - assert_block_ahi_empty((buf_block_t*) bpage); - } -#endif /* BTR_CUR_HASH_ADAPT */ - - if (bpage->oldest_modification != 0) { - - buf_flush_remove(bpage); - } - - ut_ad(!bpage->in_flush_list); - - /* Remove from the LRU list. */ - - if (buf_LRU_block_remove_hashed(bpage, true)) { - buf_LRU_block_free_hashed_page((buf_block_t*) bpage); - } else { - ut_ad(block_mutex == &buf_pool->zip_mutex); - } - - ut_ad(!mutex_own(block_mutex)); - - /* buf_LRU_block_remove_hashed() releases the hash_lock */ - ut_ad(!rw_lock_own(hash_lock, RW_LOCK_X)); - ut_ad(!rw_lock_own(hash_lock, RW_LOCK_S)); - -next_page: - bpage = prev_bpage; - } - - buf_pool_mutex_exit(buf_pool); - - if (!all_freed) { - os_thread_sleep(20000); - - goto scan_again; - } -} - -/******************************************************************//** -Remove pages belonging to a given tablespace inside a specific -buffer pool instance when we are deleting the data file(s) of that -tablespace. The pages still remain a part of LRU and are evicted from -the list as they age towards the tail of the LRU only if buf_remove -is BUF_REMOVE_FLUSH_NO_WRITE. */ -static +/** Empty the flush list for all pages belonging to a tablespace. +@param[in] id tablespace identifier +@param[in] trx transaction, for checking for user interrupt; + or NULL if nothing is to be written +@param[in] drop_ahi whether to drop the adaptive hash index */ void -buf_LRU_remove_pages( -/*=================*/ - buf_pool_t* buf_pool, /*!< buffer pool instance */ - ulint id, /*!< in: space id */ - buf_remove_t buf_remove, /*!< in: remove or flush strategy */ - const trx_t* trx) /*!< to check if the operation must - be interrupted */ +buf_LRU_flush_or_remove_pages(ulint id, const trx_t* trx, bool drop_ahi) { FlushObserver* observer = (trx == NULL) ? NULL : trx->flush_observer; + /* Pages in the system tablespace must never be discarded. */ + ut_ad(id || trx); - switch (buf_remove) { - case BUF_REMOVE_ALL_NO_WRITE: - buf_LRU_remove_all_pages(buf_pool, id); - break; - - case BUF_REMOVE_FLUSH_NO_WRITE: - /* Pass trx as NULL to avoid interruption check. */ - buf_flush_dirty_pages(buf_pool, id, observer, false, NULL); - break; - - case BUF_REMOVE_FLUSH_WRITE: - buf_flush_dirty_pages(buf_pool, id, observer, true, trx); - - if (observer == NULL) { - /* Ensure that all asynchronous IO is completed. */ - os_aio_wait_until_no_pending_writes(); - fil_flush(id); + for (ulint i = 0; i < srv_buf_pool_instances; i++) { + buf_pool_t* buf_pool = buf_pool_from_array(i); + if (drop_ahi) { + buf_LRU_drop_page_hash_for_tablespace(buf_pool, id); } - - break; + buf_flush_dirty_pages(buf_pool, id, observer, trx); } -} -/******************************************************************//** -Flushes all dirty pages or removes all pages belonging -to a given tablespace. A PROBLEM: if readahead is being started, what -guarantees that it will not try to read in pages after this operation -has completed? */ -void -buf_LRU_flush_or_remove_pages( -/*==========================*/ - ulint id, /*!< in: space id */ - buf_remove_t buf_remove, /*!< in: remove or flush strategy */ - const trx_t* trx) /*!< to check if the operation must - be interrupted */ -{ - ulint i; - - /* Before we attempt to drop pages one by one we first - attempt to drop page hash index entries in batches to make - it more efficient. The batching attempt is a best effort - attempt and does not guarantee that all pages hash entries - will be dropped. We get rid of remaining page hash entries - one by one below. */ - for (i = 0; i < srv_buf_pool_instances; i++) { - buf_pool_t* buf_pool; - - buf_pool = buf_pool_from_array(i); -#ifdef BTR_CUR_HASH_ADAPT - switch (buf_remove) { - case BUF_REMOVE_ALL_NO_WRITE: - buf_LRU_drop_page_hash_for_tablespace(buf_pool, id); - break; - - case BUF_REMOVE_FLUSH_NO_WRITE: - /* It is a DROP TABLE for a single table - tablespace. No AHI entries exist because - we already dealt with them when freeing up - extents. */ - case BUF_REMOVE_FLUSH_WRITE: - /* We allow read-only queries against the - table, there is no need to drop the AHI entries. */ - break; - } -#endif /* BTR_CUR_HASH_ADAPT */ - buf_LRU_remove_pages(buf_pool, id, buf_remove, trx); + if (trx && !observer && !trx_is_interrupted(trx)) { + /* Ensure that all asynchronous IO is completed. */ + os_aio_wait_until_no_pending_writes(); + fil_flush(id); } } diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 87334954cf8..137b88b15b3 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -1641,7 +1641,7 @@ dict_table_rename_in_cache( return(DB_OUT_OF_MEMORY); } - fil_delete_tablespace(table->space, BUF_REMOVE_ALL_NO_WRITE); + fil_delete_tablespace(table->space, true); /* Delete any temp file hanging around. */ if (os_file_status(filepath, &exists, &ftype) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 73132754fdf..d836e872265 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -2449,7 +2449,7 @@ fil_recreate_tablespace( /* Step-1: Invalidate buffer pool pages belonging to the tablespace to re-create. */ - buf_LRU_flush_or_remove_pages(space_id, BUF_REMOVE_ALL_NO_WRITE, 0); + buf_LRU_flush_or_remove_pages(space_id, NULL); /* Remove all insert buffer entries for the tablespace */ ibuf_delete_for_discarded_space(space_id); @@ -2907,7 +2907,7 @@ fil_close_tablespace( completely and permanently. The flag stop_new_ops also prevents fil_flush() from being applied to this tablespace. */ - buf_LRU_flush_or_remove_pages(id, BUF_REMOVE_FLUSH_WRITE, trx); + buf_LRU_flush_or_remove_pages(id, trx); /* If the free is successful, the X lock will be released before the space memory data structure is freed. */ @@ -2959,17 +2959,12 @@ fil_table_accessible(const dict_table_t* table) } } -/** Deletes an IBD tablespace, either general or single-table. -The tablespace must be cached in the memory cache. This will delete the -datafile, fil_space_t & fil_node_t entries from the file_system_t cache. -@param[in] space_id Tablespace id -@param[in] buf_remove Specify the action to take on the pages -for this table in the buffer pool. -@return DB_SUCCESS or error */ +/** Delete a tablespace and associated .ibd file. +@param[in] id tablespace identifier +@param[in] drop_ahi whether to drop the adaptive hash index +@return DB_SUCCESS or error */ dberr_t -fil_delete_tablespace( - ulint id, - buf_remove_t buf_remove) +fil_delete_tablespace(ulint id, bool drop_ahi) { char* path = 0; fil_space_t* space = 0; @@ -3012,7 +3007,7 @@ fil_delete_tablespace( To deal with potential read requests, we will check the ::stop_new_ops flag in fil_io(). */ - buf_LRU_flush_or_remove_pages(id, buf_remove, 0); + buf_LRU_flush_or_remove_pages(id, NULL, drop_ahi); /* If it is a delete then also delete any generated files, otherwise when we drop the database the remove directory will fail. */ @@ -3103,7 +3098,7 @@ fil_truncate_tablespace( /* Step-2: Invalidate buffer pool pages belonging to the tablespace to re-create. Remove all insert buffer entries for the tablespace */ - buf_LRU_flush_or_remove_pages(space_id, BUF_REMOVE_ALL_NO_WRITE, 0); + buf_LRU_flush_or_remove_pages(space_id, NULL); /* Step-3: Truncate the tablespace and accordingly update the fil_space_t handler that is used to access this tablespace. */ @@ -3199,7 +3194,7 @@ fil_reinit_space_header_for_table( from disabling AHI during the scan */ btr_search_s_lock_all(); DEBUG_SYNC_C("buffer_pool_scan"); - buf_LRU_flush_or_remove_pages(id, BUF_REMOVE_ALL_NO_WRITE, 0); + buf_LRU_flush_or_remove_pages(id, NULL); btr_search_s_unlock_all(); row_mysql_lock_data_dictionary(trx); @@ -3292,7 +3287,7 @@ fil_discard_tablespace( { dberr_t err; - switch (err = fil_delete_tablespace(id, BUF_REMOVE_ALL_NO_WRITE)) { + switch (err = fil_delete_tablespace(id, true)) { case DB_SUCCESS: break; diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index ea083381757..d26391a80f4 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -4962,21 +4962,36 @@ ibuf_check_bitmap_on_import( const trx_t* trx, /*!< in: transaction */ ulint space_id) /*!< in: tablespace identifier */ { - ulint size; ulint page_no; ut_ad(space_id); ut_ad(trx->mysql_thd); - bool found; - const page_size_t& page_size - = fil_space_get_page_size(space_id, &found); - - if (!found) { + FilSpace space(space_id); + if (!space()) { return(DB_TABLE_NOT_FOUND); } - size = fil_space_get_size(space_id); + const page_size_t page_size(space->flags); + /* fil_space_t::size and fil_space_t::free_limit would still be 0 + at this point. So, we will have to read page 0. */ + ut_ad(!space->free_limit); + ut_ad(!space->size); + + mtr_t mtr; + ulint size; + mtr.start(); + if (buf_block_t* sp = buf_page_get(page_id_t(space_id, 0), page_size, + RW_S_LATCH, &mtr)) { + size = std::min( + mach_read_from_4(FSP_HEADER_OFFSET + FSP_FREE_LIMIT + + sp->frame), + mach_read_from_4(FSP_HEADER_OFFSET + FSP_SIZE + + sp->frame)); + } else { + size = 0; + } + mtr.commit(); if (size == 0) { return(DB_TABLE_NOT_FOUND); @@ -4991,7 +5006,6 @@ ibuf_check_bitmap_on_import( the space, as usual. */ for (page_no = 0; page_no < size; page_no += page_size.physical()) { - mtr_t mtr; page_t* bitmap_page; ulint i; diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 516898066aa..4a54c30629b 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -65,6 +65,7 @@ struct fil_addr_t; #define BUF_GET_POSSIBLY_FREED 16 /*!< Like BUF_GET, but do not mind if the file page has been freed. */ +#define BUF_EVICT_IF_IN_POOL 20 /*!< evict a clean block if found */ /* @} */ /** @name Modes for buf_page_get_known_nowait */ /* @{ */ diff --git a/storage/innobase/include/buf0lru.h b/storage/innobase/include/buf0lru.h index 3cc01473da1..54c001ce478 100644 --- a/storage/innobase/include/buf0lru.h +++ b/storage/innobase/include/buf0lru.h @@ -50,18 +50,14 @@ These are low-level functions /** Minimum LRU list length for which the LRU_old pointer is defined */ #define BUF_LRU_OLD_MIN_LEN 512 /* 8 megabytes of 16k pages */ -/******************************************************************//** -Flushes all dirty pages or removes all pages belonging -to a given tablespace. A PROBLEM: if readahead is being started, what -guarantees that it will not try to read in pages after this operation -has completed? */ +/** Empty the flush list for all pages belonging to a tablespace. +@param[in] id tablespace identifier +@param[in] trx transaction, for checking for user interrupt; + or NULL if nothing is to be written +@param[in] drop_ahi whether to drop the adaptive hash index */ +UNIV_INTERN void -buf_LRU_flush_or_remove_pages( -/*==========================*/ - ulint id, /*!< in: space id */ - buf_remove_t buf_remove, /*!< in: remove or flush strategy */ - const trx_t* trx); /*!< to check if the operation must - be interrupted */ +buf_LRU_flush_or_remove_pages(ulint id, const trx_t* trx, bool drop_ahi=false); #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG /********************************************************************//** diff --git a/storage/innobase/include/buf0types.h b/storage/innobase/include/buf0types.h index 102b831ec61..719699f5ee2 100644 --- a/storage/innobase/include/buf0types.h +++ b/storage/innobase/include/buf0types.h @@ -59,17 +59,6 @@ enum buf_flush_t { BUF_FLUSH_N_TYPES /*!< index of last element + 1 */ }; -/** Algorithm to remove the pages for a tablespace from the buffer pool. -See buf_LRU_flush_or_remove_pages(). */ -enum buf_remove_t { - BUF_REMOVE_ALL_NO_WRITE, /*!< Remove all pages from the buffer - pool, don't write or sync to disk */ - BUF_REMOVE_FLUSH_NO_WRITE, /*!< Remove only, from the flush list, - don't write or sync to disk */ - BUF_REMOVE_FLUSH_WRITE /*!< Flush dirty pages to disk only - don't remove from the buffer pool */ -}; - /** Flags for io_fix types */ enum buf_io_fix { BUF_IO_NONE = 0, /**< no pending I/O */ diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index d3336c5f5b5..9fa507c2114 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -929,17 +929,12 @@ bool fil_table_accessible(const dict_table_t* table) MY_ATTRIBUTE((warn_unused_result, nonnull)); -/** Deletes an IBD tablespace, either general or single-table. -The tablespace must be cached in the memory cache. This will delete the -datafile, fil_space_t & fil_node_t entries from the file_system_t cache. -@param[in] space_id Tablespace id -@param[in] buf_remove Specify the action to take on the pages -for this table in the buffer pool. -@return true if success */ +/** Delete a tablespace and associated .ibd file. +@param[in] id tablespace identifier +@param[in] drop_ahi whether to drop the adaptive hash index +@return DB_SUCCESS or error */ dberr_t -fil_delete_tablespace( - ulint id, - buf_remove_t buf_remove); +fil_delete_tablespace(ulint id, bool drop_ahi = false); /** Truncate the tablespace to needed size. @param[in] space_id id of tablespace to truncate diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index 1804cba7065..f06bc878c3b 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -1537,18 +1537,16 @@ PageConverter::PageConverter( : AbstractCallback(trx), m_cfg(cfg), + m_index(cfg->m_indexes), + m_current_lsn(log_get_lsn()), m_page_zip_ptr(0), - m_heap(0) UNIV_NOTHROW + m_rec_iter(), + m_offsets_(), m_offsets(m_offsets_), + m_heap(0), + m_cluster_index(dict_table_get_first_index(cfg->m_table)) UNIV_NOTHROW { - m_index = m_cfg->m_indexes; - - m_current_lsn = log_get_lsn(); ut_a(m_current_lsn > 0); - - m_offsets = m_offsets_; rec_offs_init(m_offsets_); - - m_cluster_index = dict_table_get_first_index(m_cfg->m_table); } /** Adjust the BLOB reference for a single column that is externally stored @@ -2008,7 +2006,7 @@ PageConverter::operator() ( we can work on them */ if ((err = update_page(block, page_type)) != DB_SUCCESS) { - return(err); + break; } /* Note: For compressed pages this function will write to the @@ -2047,9 +2045,15 @@ PageConverter::operator() ( << " at offset " << offset << " looks corrupted in file " << m_filepath; - return(DB_CORRUPTION); + err = DB_CORRUPTION; } + /* If we already had and old page with matching number + in the buffer pool, evict it now, because + we no longer evict the pages on DISCARD TABLESPACE. */ + buf_page_get_gen(block->page.id, get_page_size(), + RW_NO_LATCH, NULL, BUF_EVICT_IF_IN_POOL, + __FILE__, __LINE__, NULL, NULL); return(err); } @@ -3649,8 +3653,7 @@ row_import_for_mysql( The only dirty pages generated should be from the pessimistic purge of delete marked records that couldn't be purged in Phase I. */ - buf_LRU_flush_or_remove_pages( - prebuilt->table->space, BUF_REMOVE_FLUSH_WRITE, trx); + buf_LRU_flush_or_remove_pages(prebuilt->table->space, trx); if (trx_is_interrupted(trx)) { ib::info() << "Phase III - Flush interrupted"; diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index d620bc94baa..75b4f4cbd25 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -2503,10 +2503,7 @@ err_exit: /* We already have .ibd file here. it should be deleted. */ if (dict_table_is_file_per_table(table) - && fil_delete_tablespace( - table->space, - BUF_REMOVE_FLUSH_NO_WRITE) - != DB_SUCCESS) { + && fil_delete_tablespace(table->space) != DB_SUCCESS) { ib::error() << "Not able to delete tablespace " << table->space << " of table " @@ -3173,9 +3170,6 @@ row_discard_tablespace( 4) FOREIGN KEY operations: if table->n_foreign_key_checks_running > 0, we do not allow the discard. */ - /* Play safe and remove all insert buffer entries, though we should - have removed them already when DISCARD TABLESPACE was called */ - ibuf_delete_for_discarded_space(table->space); table_id_t new_id; @@ -3540,8 +3534,7 @@ row_drop_single_table_tablespace( ib::info() << "Removed datafile " << filepath << " for table " << tablename; - } else if (fil_delete_tablespace(space_id, BUF_REMOVE_FLUSH_NO_WRITE) - != DB_SUCCESS) { + } else if (fil_delete_tablespace(space_id) != DB_SUCCESS) { ib::error() << "We removed the InnoDB internal data" " dictionary entry of table " << tablename diff --git a/storage/innobase/row/row0quiesce.cc b/storage/innobase/row/row0quiesce.cc index 54583956107..ccf58b9e73f 100644 --- a/storage/innobase/row/row0quiesce.cc +++ b/storage/innobase/row/row0quiesce.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -536,8 +537,7 @@ row_quiesce_table_start( } if (!trx_is_interrupted(trx)) { - buf_LRU_flush_or_remove_pages( - table->space, BUF_REMOVE_FLUSH_WRITE, trx); + buf_LRU_flush_or_remove_pages(table->space, trx); if (trx_is_interrupted(trx)) { diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index d6dd5805186..dc1b1eca0ef 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -1100,21 +1100,22 @@ srv_undo_tablespaces_init(bool create_new_db) mtr_commit(&mtr); /* Step-2: Flush the dirty pages from the buffer pool. */ + trx_t* trx = trx_allocate_for_background(); + for (undo::undo_spaces_t::const_iterator it = undo::Truncate::s_fix_up_spaces.begin(); it != undo::Truncate::s_fix_up_spaces.end(); ++it) { - buf_LRU_flush_or_remove_pages( - TRX_SYS_SPACE, BUF_REMOVE_FLUSH_WRITE, NULL); + buf_LRU_flush_or_remove_pages(TRX_SYS_SPACE, trx); - buf_LRU_flush_or_remove_pages( - *it, BUF_REMOVE_FLUSH_WRITE, NULL); + buf_LRU_flush_or_remove_pages(*it, trx); /* Remove the truncate redo log file. */ undo::Truncate undo_trunc; undo_trunc.done_logging(*it); } + trx_free_for_background(trx); } return(DB_SUCCESS); diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 7eb0474091e..51e2453a853 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -730,6 +730,8 @@ buf_page_is_corrupted( ulint zip_size, const fil_space_t* space) { + DBUG_EXECUTE_IF("buf_page_import_corrupt_failure", return(TRUE); ); + ulint checksum_field1; ulint checksum_field2; ulint space_id = mach_read_from_4( @@ -838,8 +840,6 @@ buf_page_is_corrupted( return(false); } - DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", return(true); ); - ulint page_no = mach_read_from_4(read_buf + FIL_PAGE_OFFSET); const srv_checksum_algorithm_t curr_algo = @@ -2956,8 +2956,8 @@ buf_page_get_gen( ib_mutex_t* fix_mutex = NULL; buf_pool_t* buf_pool = buf_pool_get(space, offset); - ut_ad(mtr); - ut_ad(mtr->state == MTR_ACTIVE); + ut_ad((mtr == NULL) == (mode == BUF_EVICT_IF_IN_POOL)); + ut_ad(!mtr || mtr->state == MTR_ACTIVE); ut_ad((rw_latch == RW_S_LATCH) || (rw_latch == RW_X_LATCH) || (rw_latch == RW_NO_LATCH)); @@ -2968,23 +2968,29 @@ buf_page_get_gen( #ifdef UNIV_DEBUG switch (mode) { + case BUF_EVICT_IF_IN_POOL: + /* After DISCARD TABLESPACE, the tablespace would not exist, + but in IMPORT TABLESPACE, PageConverter::operator() must + replace any old pages, which were not evicted during DISCARD. + Skip the assertion on zip_size. */ + break; case BUF_GET_NO_LATCH: ut_ad(rw_latch == RW_NO_LATCH); - break; + /* fall through */ case BUF_GET: case BUF_GET_IF_IN_POOL: case BUF_PEEK_IF_IN_POOL: case BUF_GET_IF_IN_POOL_OR_WATCH: case BUF_GET_POSSIBLY_FREED: + ut_ad(zip_size == fil_space_get_zip_size(space)); break; default: ut_error; } #endif /* UNIV_DEBUG */ - ut_ad(zip_size == fil_space_get_zip_size(space)); ut_ad(ut_is_2pow(zip_size)); #ifndef UNIV_LOG_DEBUG - ut_ad(!ibuf_inside(mtr) + ut_ad(!mtr || !ibuf_inside(mtr) || ibuf_page_low(space, zip_size, offset, FALSE, file, line, NULL)); #endif @@ -3054,9 +3060,11 @@ loop: rw_lock_x_unlock(hash_lock); } - if (mode == BUF_GET_IF_IN_POOL - || mode == BUF_PEEK_IF_IN_POOL - || mode == BUF_GET_IF_IN_POOL_OR_WATCH) { + switch (mode) { + case BUF_GET_IF_IN_POOL: + case BUF_GET_IF_IN_POOL_OR_WATCH: + case BUF_PEEK_IF_IN_POOL: + case BUF_EVICT_IF_IN_POOL: #ifdef UNIV_SYNC_DEBUG ut_ad(!rw_lock_own(hash_lock, RW_LOCK_EX)); ut_ad(!rw_lock_own(hash_lock, RW_LOCK_SHARED)); @@ -3145,8 +3153,10 @@ got_block: ut_ad(page_zip_get_size(&block->page.zip) == zip_size); - if (mode == BUF_GET_IF_IN_POOL || mode == BUF_PEEK_IF_IN_POOL) { - + switch (mode) { + case BUF_GET_IF_IN_POOL: + case BUF_PEEK_IF_IN_POOL: + case BUF_EVICT_IF_IN_POOL: bool must_read; { @@ -3184,6 +3194,22 @@ got_block: case BUF_BLOCK_FILE_PAGE: ut_ad(fix_mutex != &buf_pool->zip_mutex); + + if (UNIV_UNLIKELY(mode == BUF_EVICT_IF_IN_POOL)) { +evict_from_pool: + ut_ad(!fix_block->page.oldest_modification); + mutex_enter(&buf_pool->LRU_list_mutex); + buf_block_unfix(fix_block); + mutex_enter(fix_mutex); + + if (!buf_LRU_free_page(&fix_block->page, true)) { + ut_ad(0); + mutex_exit(&buf_pool->LRU_list_mutex); + } + + mutex_exit(fix_mutex); + return(NULL); + } break; case BUF_BLOCK_ZIP_PAGE: @@ -3218,6 +3244,10 @@ got_block: goto loop; } + if (UNIV_UNLIKELY(mode == BUF_EVICT_IF_IN_POOL)) { + goto evict_from_pool; + } + /* Buffer-fix the block so that it cannot be evicted or relocated while we are attempting to allocate an uncompressed page. */ @@ -4798,7 +4828,7 @@ database_corrupted: if (err != DB_SUCCESS) { /* Not a real corruption if it was triggered by error injection */ - DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", + DBUG_EXECUTE_IF("buf_page_import_corrupt_failure", if (bpage->space > TRX_SYS_SPACE) { buf_mark_space_corrupt(bpage); ib_logf(IB_LOG_LEVEL_INFO, @@ -4866,7 +4896,7 @@ database_corrupted: } } - DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", + DBUG_EXECUTE_IF("buf_page_import_corrupt_failure", page_not_corrupt: bpage = bpage; ); if (recv_recovery_is_on()) { diff --git a/storage/xtradb/buf/buf0lru.cc b/storage/xtradb/buf/buf0lru.cc index d979eb44a96..c71d45009e4 100644 --- a/storage/xtradb/buf/buf0lru.cc +++ b/storage/xtradb/buf/buf0lru.cc @@ -571,26 +571,20 @@ buf_flush_or_remove_page( return(processed); } -/******************************************************************//** -Remove all dirty pages belonging to a given tablespace inside a specific +/** Remove all dirty pages belonging to a given tablespace inside a specific buffer pool instance when we are deleting the data file(s) of that tablespace. The pages still remain a part of LRU and are evicted from the list as they age towards the tail of the LRU. +@param[in,out] buf_pool buffer pool +@param[in] id tablespace identifier +@param[in] trx transaction (to check for interrupt), + or NULL if the files should not be written to @retval DB_SUCCESS if all freed @retval DB_FAIL if not all freed @retval DB_INTERRUPTED if the transaction was interrupted */ static MY_ATTRIBUTE((nonnull(1), warn_unused_result)) dberr_t -buf_flush_or_remove_pages( -/*======================*/ - buf_pool_t* buf_pool, /*!< buffer pool instance */ - ulint id, /*!< in: target space id for which - to remove or flush pages */ - bool flush, /*!< in: flush to disk if true but - don't remove else remove without - flushing to disk */ - const trx_t* trx) /*!< to check if the operation must - be interrupted, can be 0 */ +buf_flush_or_remove_pages(buf_pool_t* buf_pool, ulint id, const trx_t* trx) { buf_page_t* prev; buf_page_t* bpage; @@ -621,7 +615,7 @@ rescan: /* Skip this block, as it does not belong to the target space. */ - } else if (!buf_flush_or_remove_page(buf_pool, bpage, flush, + } else if (!buf_flush_or_remove_page(buf_pool, bpage, trx, &must_restart)) { /* Remove was unsuccessful, we have to try again @@ -647,7 +641,7 @@ rescan: /* Cannot trust the prev pointer */ break; } - } else if (flush) { + } else if (trx) { /* The processing was successful. And during the processing we have released all the buf_pool mutexes @@ -674,19 +668,17 @@ rescan: break; } -#ifdef DBUG_OFF - if (flush) { + if (trx) { DBUG_EXECUTE_IF("ib_export_flush_crash", static ulint n_pages; if (++n_pages == 4) {DBUG_SUICIDE();}); - } -#endif /* DBUG_OFF */ - /* The check for trx is interrupted is expensive, we want - to check every N iterations. */ - if (!processed && trx && trx_is_interrupted(trx)) { - buf_flush_list_mutex_exit(buf_pool); - return(DB_INTERRUPTED); + /* The check for trx is interrupted is + expensive, we want to check every N iterations. */ + if (!processed && trx_is_interrupted(trx)) { + buf_flush_list_mutex_exit(buf_pool); + return(DB_INTERRUPTED); + } } } @@ -695,28 +687,25 @@ rescan: return(all_freed ? DB_SUCCESS : DB_FAIL); } -/******************************************************************//** -Remove or flush all the dirty pages that belong to a given tablespace +/** Remove or flush all the dirty pages that belong to a given tablespace inside a specific buffer pool instance. The pages will remain in the LRU list and will be evicted from the LRU list as they age and move towards -the tail of the LRU list. */ +the tail of the LRU list. +@param[in,out] buf_pool buffer pool +@param[in] id tablespace identifier +@param[in] trx transaction (to check for interrupt), + or NULL if the files should not be written to +*/ static MY_ATTRIBUTE((nonnull(1))) void -buf_flush_dirty_pages( -/*==================*/ - buf_pool_t* buf_pool, /*!< buffer pool instance */ - ulint id, /*!< in: space id */ - bool flush, /*!< in: flush to disk if true otherwise - remove the pages without flushing */ - const trx_t* trx) /*!< to check if the operation must - be interrupted */ +buf_flush_dirty_pages(buf_pool_t* buf_pool, ulint id, const trx_t* trx) { dberr_t err; do { mutex_enter(&buf_pool->LRU_list_mutex); - err = buf_flush_or_remove_pages(buf_pool, id, flush, trx); + err = buf_flush_or_remove_pages(buf_pool, id, trx); mutex_exit(&buf_pool->LRU_list_mutex); @@ -737,239 +726,27 @@ buf_flush_dirty_pages( || buf_pool_get_dirty_pages_count(buf_pool, id) == 0); } -/******************************************************************//** -Remove all pages that belong to a given tablespace inside a specific -buffer pool instance when we are DISCARDing the tablespace. */ -static MY_ATTRIBUTE((nonnull)) +/** Empty the flush list for all pages belonging to a tablespace. +@param[in] id tablespace identifier +@param[in] trx transaction, for checking for user interrupt; + or NULL if nothing is to be written +@param[in] drop_ahi whether to drop the adaptive hash index */ +UNIV_INTERN void -buf_LRU_remove_all_pages( -/*=====================*/ - buf_pool_t* buf_pool, /*!< buffer pool instance */ - ulint id) /*!< in: space id */ +buf_LRU_flush_or_remove_pages(ulint id, const trx_t* trx, bool drop_ahi) { - buf_page_t* bpage; - ibool all_freed; - -scan_again: - mutex_enter(&buf_pool->LRU_list_mutex); - - all_freed = TRUE; - - for (bpage = UT_LIST_GET_LAST(buf_pool->LRU); - bpage != NULL; - /* No op */) { - - prio_rw_lock_t* hash_lock; - buf_page_t* prev_bpage; - ib_mutex_t* block_mutex = NULL; - - ut_a(buf_page_in_file(bpage)); - ut_ad(bpage->in_LRU_list); - - prev_bpage = UT_LIST_GET_PREV(LRU, bpage); - - /* It is safe to check bpage->space and bpage->io_fix while - holding buf_pool->LRU_list_mutex only and later recheck - while holding the buf_page_get_mutex() mutex. */ - - if (buf_page_get_space(bpage) != id) { - /* Skip this block, as it does not belong to - the space that is being invalidated. */ - goto next_page; - } else if (UNIV_UNLIKELY(buf_page_get_io_fix_unlocked(bpage) - != BUF_IO_NONE)) { - /* We cannot remove this page during this scan - yet; maybe the system is currently reading it - in, or flushing the modifications to the file */ - - all_freed = FALSE; - goto next_page; - } else { - ulint fold = buf_page_address_fold( - bpage->space, bpage->offset); - - hash_lock = buf_page_hash_lock_get(buf_pool, fold); - - rw_lock_x_lock(hash_lock); - - block_mutex = buf_page_get_mutex(bpage); - mutex_enter(block_mutex); - - if (UNIV_UNLIKELY( - buf_page_get_space(bpage) != id - || bpage->buf_fix_count > 0 - || (buf_page_get_io_fix(bpage) - != BUF_IO_NONE))) { - - mutex_exit(block_mutex); - - rw_lock_x_unlock(hash_lock); - - /* We cannot remove this page during - this scan yet; maybe the system is - currently reading it in, or flushing - the modifications to the file */ - - all_freed = FALSE; - - goto next_page; - } - } - - ut_ad(mutex_own(block_mutex)); - -#ifdef UNIV_DEBUG - if (buf_debug_prints) { - fprintf(stderr, - "Dropping space %lu page %lu\n", - (ulong) buf_page_get_space(bpage), - (ulong) buf_page_get_page_no(bpage)); - } -#endif - if (buf_page_get_state(bpage) != BUF_BLOCK_FILE_PAGE) { - /* Do nothing, because the adaptive hash index - covers uncompressed pages only. */ - } else if (((buf_block_t*) bpage)->index) { - ulint page_no; - ulint zip_size; - - mutex_exit(&buf_pool->LRU_list_mutex); - - zip_size = buf_page_get_zip_size(bpage); - page_no = buf_page_get_page_no(bpage); - - mutex_exit(block_mutex); - - rw_lock_x_unlock(hash_lock); - - /* Note that the following call will acquire - and release block->lock X-latch. */ - - btr_search_drop_page_hash_when_freed( - id, zip_size, page_no); - - goto scan_again; - } - - if (bpage->oldest_modification != 0) { - - buf_flush_remove(bpage); - } - - ut_ad(!bpage->in_flush_list); - - /* Remove from the LRU list. */ - - if (buf_LRU_block_remove_hashed(bpage, true)) { - - mutex_enter(block_mutex); - buf_LRU_block_free_hashed_page((buf_block_t*) bpage); - mutex_exit(block_mutex); - } else { - ut_ad(block_mutex == &buf_pool->zip_mutex); + for (ulint i = 0; i < srv_buf_pool_instances; i++) { + buf_pool_t* buf_pool = buf_pool_from_array(i); + if (drop_ahi) { + buf_LRU_drop_page_hash_for_tablespace(buf_pool, id); } - - ut_ad(!mutex_own(block_mutex)); - -#ifdef UNIV_SYNC_DEBUG - /* buf_LRU_block_remove_hashed() releases the hash_lock */ - ut_ad(!rw_lock_own(hash_lock, RW_LOCK_EX)); - ut_ad(!rw_lock_own(hash_lock, RW_LOCK_SHARED)); -#endif /* UNIV_SYNC_DEBUG */ - -next_page: - bpage = prev_bpage; + buf_flush_dirty_pages(buf_pool, id, trx); } - mutex_exit(&buf_pool->LRU_list_mutex); - - if (!all_freed) { - os_thread_sleep(20000); - - goto scan_again; - } -} - -/******************************************************************//** -Remove pages belonging to a given tablespace inside a specific -buffer pool instance when we are deleting the data file(s) of that -tablespace. The pages still remain a part of LRU and are evicted from -the list as they age towards the tail of the LRU only if buf_remove -is BUF_REMOVE_FLUSH_NO_WRITE. */ -static MY_ATTRIBUTE((nonnull(1))) -void -buf_LRU_remove_pages( -/*=================*/ - buf_pool_t* buf_pool, /*!< buffer pool instance */ - ulint id, /*!< in: space id */ - buf_remove_t buf_remove, /*!< in: remove or flush strategy */ - const trx_t* trx) /*!< to check if the operation must - be interrupted */ -{ - switch (buf_remove) { - case BUF_REMOVE_ALL_NO_WRITE: - buf_LRU_remove_all_pages(buf_pool, id); - break; - - case BUF_REMOVE_FLUSH_NO_WRITE: - ut_a(trx == 0); - buf_flush_dirty_pages(buf_pool, id, false, NULL); - break; - - case BUF_REMOVE_FLUSH_WRITE: - ut_a(trx != 0); - buf_flush_dirty_pages(buf_pool, id, true, trx); + if (trx && !trx_is_interrupted(trx)) { /* Ensure that all asynchronous IO is completed. */ os_aio_wait_until_no_pending_writes(); fil_flush(id); - break; - } -} - -/******************************************************************//** -Flushes all dirty pages or removes all pages belonging -to a given tablespace. A PROBLEM: if readahead is being started, what -guarantees that it will not try to read in pages after this operation -has completed? */ -UNIV_INTERN -void -buf_LRU_flush_or_remove_pages( -/*==========================*/ - ulint id, /*!< in: space id */ - buf_remove_t buf_remove, /*!< in: remove or flush strategy */ - const trx_t* trx) /*!< to check if the operation must - be interrupted */ -{ - ulint i; - - /* Before we attempt to drop pages one by one we first - attempt to drop page hash index entries in batches to make - it more efficient. The batching attempt is a best effort - attempt and does not guarantee that all pages hash entries - will be dropped. We get rid of remaining page hash entries - one by one below. */ - for (i = 0; i < srv_buf_pool_instances; i++) { - buf_pool_t* buf_pool; - - buf_pool = buf_pool_from_array(i); - - switch (buf_remove) { - case BUF_REMOVE_ALL_NO_WRITE: - buf_LRU_drop_page_hash_for_tablespace(buf_pool, id); - break; - - case BUF_REMOVE_FLUSH_NO_WRITE: - /* It is a DROP TABLE for a single table - tablespace. No AHI entries exist because - we already dealt with them when freeing up - extents. */ - case BUF_REMOVE_FLUSH_WRITE: - /* We allow read-only queries against the - table, there is no need to drop the AHI entries. */ - break; - } - - buf_LRU_remove_pages(buf_pool, id, buf_remove, trx); } } diff --git a/storage/xtradb/dict/dict0dict.cc b/storage/xtradb/dict/dict0dict.cc index 1558c6d50ac..5af3a635a96 100644 --- a/storage/xtradb/dict/dict0dict.cc +++ b/storage/xtradb/dict/dict0dict.cc @@ -1679,7 +1679,7 @@ dict_table_rename_in_cache( filepath = fil_make_ibd_name(table->name, false); } - fil_delete_tablespace(table->space, BUF_REMOVE_ALL_NO_WRITE); + fil_delete_tablespace(table->space, true); /* Delete any temp file hanging around. */ if (os_file_status(filepath, &exists, &ftype) diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 3dd8aa8081b..2210505a2bd 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -2572,8 +2572,7 @@ fil_op_log_parse_or_replay( switch (type) { case MLOG_FILE_DELETE: if (fil_tablespace_exists_in_mem(space_id)) { - dberr_t err = fil_delete_tablespace( - space_id, BUF_REMOVE_FLUSH_NO_WRITE); + dberr_t err = fil_delete_tablespace(space_id); ut_a(err == DB_SUCCESS); } @@ -2851,7 +2850,7 @@ fil_close_tablespace( completely and permanently. The flag stop_new_ops also prevents fil_flush() from being applied to this tablespace. */ - buf_LRU_flush_or_remove_pages(id, BUF_REMOVE_FLUSH_WRITE, trx); + buf_LRU_flush_or_remove_pages(id, trx); #endif mutex_enter(&fil_system->mutex); @@ -2878,18 +2877,13 @@ fil_close_tablespace( return(err); } -/*******************************************************************//** -Deletes a single-table tablespace. The tablespace must be cached in the -memory cache. +/** Delete a tablespace and associated .ibd file. +@param[in] id tablespace identifier +@param[in] drop_ahi whether to drop the adaptive hash index @return DB_SUCCESS or error */ UNIV_INTERN dberr_t -fil_delete_tablespace( -/*==================*/ - ulint id, /*!< in: space id */ - buf_remove_t buf_remove) /*!< in: specify the action to take - on the tables pages in the buffer - pool */ +fil_delete_tablespace(ulint id, bool drop_ahi) { char* path = 0; fil_space_t* space = 0; @@ -2945,7 +2939,7 @@ fil_delete_tablespace( To deal with potential read requests by checking the ::stop_new_ops flag in fil_io() */ - buf_LRU_flush_or_remove_pages(id, buf_remove, 0); + buf_LRU_flush_or_remove_pages(id, NULL, drop_ahi); #endif /* !UNIV_HOTBACKUP */ @@ -3056,7 +3050,7 @@ fil_discard_tablespace( { dberr_t err; - switch (err = fil_delete_tablespace(id, BUF_REMOVE_ALL_NO_WRITE)) { + switch (err = fil_delete_tablespace(id, true)) { case DB_SUCCESS: break; diff --git a/storage/xtradb/ibuf/ibuf0ibuf.cc b/storage/xtradb/ibuf/ibuf0ibuf.cc index 17741b324c9..6925cd3abb5 100644 --- a/storage/xtradb/ibuf/ibuf0ibuf.cc +++ b/storage/xtradb/ibuf/ibuf0ibuf.cc @@ -5178,7 +5178,20 @@ ibuf_check_bitmap_on_import( return(DB_TABLE_NOT_FOUND); } - size = fil_space_get_size(space_id); + mtr_t mtr; + mtr_start(&mtr); + { + buf_block_t* sp = buf_page_get(space_id, zip_size, 0, + RW_S_LATCH, &mtr); + if (sp) { + size = mach_read_from_4( + FSP_HEADER_OFFSET + FSP_FREE_LIMIT + + sp->frame); + } else { + size = 0; + } + } + mtr_commit(&mtr); if (size == 0) { return(DB_TABLE_NOT_FOUND); @@ -5189,7 +5202,6 @@ ibuf_check_bitmap_on_import( page_size = zip_size ? zip_size : UNIV_PAGE_SIZE; for (page_no = 0; page_no < size; page_no += page_size) { - mtr_t mtr; page_t* bitmap_page; ulint i; diff --git a/storage/xtradb/include/buf0buf.h b/storage/xtradb/include/buf0buf.h index 88ee042e8c3..7661ba1785d 100644 --- a/storage/xtradb/include/buf0buf.h +++ b/storage/xtradb/include/buf0buf.h @@ -58,6 +58,7 @@ Created 11/5/1995 Heikki Tuuri #define BUF_GET_POSSIBLY_FREED 16 /*!< Like BUF_GET, but do not mind if the file page has been freed. */ +#define BUF_EVICT_IF_IN_POOL 20 /*!< evict a clean block if found */ /* @} */ /** @name Modes for buf_page_get_known_nowait */ /* @{ */ diff --git a/storage/xtradb/include/buf0lru.h b/storage/xtradb/include/buf0lru.h index f056c6c4116..1bc11937fa1 100644 --- a/storage/xtradb/include/buf0lru.h +++ b/storage/xtradb/include/buf0lru.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -53,19 +54,14 @@ These are low-level functions /** Minimum LRU list length for which the LRU_old pointer is defined */ #define BUF_LRU_OLD_MIN_LEN 512 /* 8 megabytes of 16k pages */ -/******************************************************************//** -Flushes all dirty pages or removes all pages belonging -to a given tablespace. A PROBLEM: if readahead is being started, what -guarantees that it will not try to read in pages after this operation -has completed? */ +/** Empty the flush list for all pages belonging to a tablespace. +@param[in] id tablespace identifier +@param[in] trx transaction, for checking for user interrupt; + or NULL if nothing is to be written +@param[in] drop_ahi whether to drop the adaptive hash index */ UNIV_INTERN void -buf_LRU_flush_or_remove_pages( -/*==========================*/ - ulint id, /*!< in: space id */ - buf_remove_t buf_remove, /*!< in: remove or flush strategy */ - const trx_t* trx); /*!< to check if the operation must - be interrupted */ +buf_LRU_flush_or_remove_pages(ulint id, const trx_t* trx, bool drop_ahi=false); #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG /********************************************************************//** diff --git a/storage/xtradb/include/buf0types.h b/storage/xtradb/include/buf0types.h index 4eb5ea18cef..6db3cb1238c 100644 --- a/storage/xtradb/include/buf0types.h +++ b/storage/xtradb/include/buf0types.h @@ -58,17 +58,6 @@ enum buf_flush_t { BUF_FLUSH_N_TYPES /*!< index of last element + 1 */ }; -/** Algorithm to remove the pages for a tablespace from the buffer pool. -See buf_LRU_flush_or_remove_pages(). */ -enum buf_remove_t { - BUF_REMOVE_ALL_NO_WRITE, /*!< Remove all pages from the buffer - pool, don't write or sync to disk */ - BUF_REMOVE_FLUSH_NO_WRITE, /*!< Remove only, from the flush list, - don't write or sync to disk */ - BUF_REMOVE_FLUSH_WRITE /*!< Flush dirty pages to disk only - don't remove from the buffer pool */ -}; - /** Flags for io_fix types */ enum buf_io_fix { BUF_IO_NONE = 0, /**< no pending I/O */ diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index a33cec65ed5..5fe2d20b4f0 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -849,18 +849,13 @@ fil_op_log_parse_or_replay( only be parsed but not replayed */ ulint log_flags); /*!< in: redo log flags (stored in the page number parameter) */ -/*******************************************************************//** -Deletes a single-table tablespace. The tablespace must be cached in the -memory cache. -@return TRUE if success */ +/** Delete a tablespace and associated .ibd file. +@param[in] id tablespace identifier +@param[in] drop_ahi whether to drop the adaptive hash index +@return DB_SUCCESS or error */ UNIV_INTERN dberr_t -fil_delete_tablespace( -/*==================*/ - ulint id, /*!< in: space id */ - buf_remove_t buf_remove); /*!< in: specify the action to take - on the tables pages in the buffer - pool */ +fil_delete_tablespace(ulint id, bool drop_ahi = false); /*******************************************************************//** Closes a single-table tablespace. The tablespace must be cached in the memory cache. Free all pages used by the tablespace. diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 183f65bcbd8..8b0fa059100 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -2570,8 +2570,8 @@ os_file_get_size( return(offset); #else - return((os_offset_t) lseek(file, 0, SEEK_END)); - + struct stat statbuf; + return fstat(file, &statbuf) ? os_offset_t(-1) : statbuf.st_size; #endif /* __WIN__ */ } @@ -2625,19 +2625,29 @@ os_file_set_size( if (srv_use_posix_fallocate) { int err; do { - err = posix_fallocate(file, 0, size); + os_offset_t current_size = os_file_get_size(file); + err = current_size >= size + ? 0 : posix_fallocate(file, current_size, + size - current_size); } while (err == EINTR && srv_shutdown_state == SRV_SHUTDOWN_NONE); - if (err) { + switch (err) { + case 0: + return true; + default: ib_logf(IB_LOG_LEVEL_ERROR, "preallocating " INT64PF " bytes for" "file %s failed with error %d", size, name, err); + /* fall through */ + case EINTR: + errno = err; + return false; + case EINVAL: + /* fall back to the code below */ + break; } - /* Set errno because posix_fallocate() does not do it.*/ - errno = err; - return(!err); } # endif @@ -2679,11 +2689,12 @@ os_file_set_size( } current_size += n_bytes; - } while (current_size < size); + } while (current_size < size + && srv_shutdown_state == SRV_SHUTDOWN_NONE); free(buf2); - return(ret && os_file_flush(file)); + return(ret && current_size >= size && os_file_flush(file)); #endif } diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index 130f50ec9ed..7689c362c0d 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -1602,18 +1602,16 @@ PageConverter::PageConverter( : AbstractCallback(trx), m_cfg(cfg), + m_index(cfg->m_indexes), + m_current_lsn(log_get_lsn()), m_page_zip_ptr(0), - m_heap(0) UNIV_NOTHROW + m_rec_iter(), + m_offsets_(), m_offsets(m_offsets_), + m_heap(0), + m_cluster_index(dict_table_get_first_index(cfg->m_table)) UNIV_NOTHROW { - m_index = m_cfg->m_indexes; - - m_current_lsn = log_get_lsn(); ut_a(m_current_lsn > 0); - - m_offsets = m_offsets_; rec_offs_init(m_offsets_); - - m_cluster_index = dict_table_get_first_index(m_cfg->m_table); } /** @@ -2104,7 +2102,7 @@ PageConverter::operator() ( we can work on them */ if ((err = update_page(block, page_type)) != DB_SUCCESS) { - return(err); + break; } /* Note: For compressed pages this function will write to the @@ -2141,9 +2139,15 @@ PageConverter::operator() ( "%s: Page %lu at offset " UINT64PF " looks corrupted.", m_filepath, (ulong) (offset / m_page_size), offset); - return(DB_CORRUPTION); + err = DB_CORRUPTION; } + /* If we already had and old page with matching number + in the buffer pool, evict it now, because + we no longer evict the pages on DISCARD TABLESPACE. */ + buf_page_get_gen(get_space_id(), get_zip_size(), block->page.offset, + RW_NO_LATCH, NULL, BUF_EVICT_IF_IN_POOL, + __FILE__, __LINE__, NULL); return(err); } @@ -3717,8 +3721,7 @@ row_import_for_mysql( The only dirty pages generated should be from the pessimistic purge of delete marked records that couldn't be purged in Phase I. */ - buf_LRU_flush_or_remove_pages( - prebuilt->table->space, BUF_REMOVE_FLUSH_WRITE, trx); + buf_LRU_flush_or_remove_pages(prebuilt->table->space, trx); if (trx_is_interrupted(trx)) { ib_logf(IB_LOG_LEVEL_INFO, "Phase III - Flush interrupted"); diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index 67eb1d7de94..3f79c3af6c8 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -2494,10 +2494,7 @@ err_exit: /* We already have .ibd file here. it should be deleted. */ if (table->space - && fil_delete_tablespace( - table->space, - BUF_REMOVE_FLUSH_NO_WRITE) - != DB_SUCCESS) { + && fil_delete_tablespace(table->space) != DB_SUCCESS) { ut_print_timestamp(stderr); fprintf(stderr, @@ -3132,9 +3129,6 @@ row_discard_tablespace( 4) FOREIGN KEY operations: if table->n_foreign_key_checks_running > 0, we do not allow the discard. */ - /* Play safe and remove all insert buffer entries, though we should - have removed them already when DISCARD TABLESPACE was called */ - ibuf_delete_for_discarded_space(table->space); table_id_t new_id; @@ -4516,9 +4510,7 @@ row_drop_table_for_mysql( fil_delete_file(filepath); - } else if (fil_delete_tablespace( - space_id, - BUF_REMOVE_FLUSH_NO_WRITE) + } else if (fil_delete_tablespace(space_id) != DB_SUCCESS) { fprintf(stderr, "InnoDB: We removed now the InnoDB" diff --git a/storage/xtradb/row/row0quiesce.cc b/storage/xtradb/row/row0quiesce.cc index 583fbe60fb3..6c4e6adb96c 100644 --- a/storage/xtradb/row/row0quiesce.cc +++ b/storage/xtradb/row/row0quiesce.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -542,8 +543,7 @@ row_quiesce_table_start( } if (!trx_is_interrupted(trx)) { - buf_LRU_flush_or_remove_pages( - table->space, BUF_REMOVE_FLUSH_WRITE, trx); + buf_LRU_flush_or_remove_pages(table->space, trx); if (trx_is_interrupted(trx)) { |