diff options
| author | Marko Mäkelä <marko.makela@mariadb.com> | 2017-01-19 12:06:13 +0200 |
|---|---|---|
| committer | Marko Mäkelä <marko.makela@mariadb.com> | 2017-01-19 12:06:13 +0200 |
| commit | b05bf8ff0f1acf39afeb71e0e8a148090364d8fc (patch) | |
| tree | d104ee5ed8a27edd28f716319f91992ef8880411 | |
| parent | 833aa97cec771fbfb2ef8dd104de6a3d1e971daa (diff) | |
| parent | 03497129371fe2c16d847b7e83a5eeecab9c34a2 (diff) | |
| download | mariadb-git-b05bf8ff0f1acf39afeb71e0e8a148090364d8fc.tar.gz | |
Merge 10.1 to 10.2.
Most notably, this includes MDEV-11623, which includes a fix and
an upgrade procedure for the InnoDB file format incompatibility
that is present in MariaDB Server 10.1.0 through 10.1.20.
In other words, this merge should address
MDEV-11202 InnoDB 10.1 -> 10.2 migration does not work
81 files changed, 2111 insertions, 2181 deletions
diff --git a/mysql-test/r/cast.result b/mysql-test/r/cast.result index 2edacb6c7e9..1e39c03696f 100644 --- a/mysql-test/r/cast.result +++ b/mysql-test/r/cast.result @@ -583,7 +583,7 @@ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `cast(1 as unsigned)` int(1) unsigned NOT NULL, - `cast(1 as signed)` int(1) NOT NULL, + `cast(1 as signed)` int(2) NOT NULL, `cast(1 as double(5,2))` double(5,2) DEFAULT NULL, `cast(1 as decimal(5,3))` decimal(5,3) NOT NULL, `cast("A" as binary)` varbinary(1) NOT NULL, @@ -822,3 +822,74 @@ utf8_bin select collation(cast("a" as char(10) binary ascii)); collation(cast("a" as char(10) binary ascii)) latin1_bin +# +# MDEV-11030 Assertion `precision > 0' failed in decimal_bin_size +# +SELECT * FROM (SELECT IFNULL(CONVERT(NULL, UNSIGNED), NULL)) sq; +IFNULL(CONVERT(NULL, UNSIGNED), NULL) +NULL +CREATE TABLE t1 AS SELECT IFNULL(CONVERT(NULL, UNSIGNED), NULL); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `IFNULL(CONVERT(NULL, UNSIGNED), NULL)` decimal(1,0) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +DROP TABLE t1; +CREATE TABLE t1 AS SELECT COALESCE(CONVERT(NULL, UNSIGNED), NULL); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `COALESCE(CONVERT(NULL, UNSIGNED), NULL)` decimal(1,0) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +DROP TABLE t1; +CREATE TABLE t1 AS SELECT CASE WHEN TRUE THEN CONVERT(NULL, UNSIGNED) ELSE NULL END; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `CASE WHEN TRUE THEN CONVERT(NULL, UNSIGNED) ELSE NULL END` decimal(1,0) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +DROP TABLE t1; +CREATE TABLE t1 AS SELECT IFNULL(CONVERT(NULL,SIGNED),CONVERT(NULL,UNSIGNED)) AS a; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` decimal(1,0) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +DROP TABLE t1; +CREATE TABLE t1 AS SELECT +-1, +CONVERT(NULL,SIGNED), +CONCAT(CONVERT(NULL,SIGNED)), +1, +CONVERT(NULL,UNSIGNED), +CONCAT(CONVERT(NULL,UNSIGNED)); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `-1` int(2) NOT NULL, + `CONVERT(NULL,SIGNED)` int(2) DEFAULT NULL, + `CONCAT(CONVERT(NULL,SIGNED))` varchar(2) DEFAULT NULL, + `1` int(1) NOT NULL, + `CONVERT(NULL,UNSIGNED)` int(1) unsigned DEFAULT NULL, + `CONCAT(CONVERT(NULL,UNSIGNED))` varchar(1) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +DROP TABLE t1; +CREATE TABLE t1 AS SELECT +CONVERT('',SIGNED), +CONCAT(CONVERT('',SIGNED)), +CONVERT('',UNSIGNED), +CONCAT(CONVERT('',UNSIGNED)); +Warnings: +Warning 1292 Truncated incorrect INTEGER value: '' +Warning 1292 Truncated incorrect INTEGER value: '' +Warning 1292 Truncated incorrect INTEGER value: '' +Warning 1292 Truncated incorrect INTEGER value: '' +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `CONVERT('',SIGNED)` int(2) NOT NULL, + `CONCAT(CONVERT('',SIGNED))` varchar(2) NOT NULL, + `CONVERT('',UNSIGNED)` int(1) unsigned NOT NULL, + `CONCAT(CONVERT('',UNSIGNED))` varchar(1) NOT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +DROP TABLE t1; diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result index 203921ce08f..2b62252a129 100644 --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result @@ -461,6 +461,10 @@ The following options may be given as the first argument: --max-seeks-for-key=# Limit assumed max number of seeks when looking up rows based on a key + --max-session-mem-used=# + Amount of memory a single user session is allowed to + allocate. This limits the value of the session variable + MEM_USED --max-sort-length=# The number of bytes to use when sorting BLOB or TEXT values (only the first max_sort_length bytes of each value are used; the rest are ignored) @@ -1311,6 +1315,7 @@ max-prepared-stmt-count 16382 max-recursive-iterations 18446744073709551615 max-relay-log-size 1073741824 max-seeks-for-key 18446744073709551615 +max-session-mem-used 9223372036854775807 max-sort-length 1024 max-sp-recursion-depth 0 max-statement-time 0 diff --git a/mysql-test/r/trigger_null-8605.result b/mysql-test/r/trigger_null-8605.result index b187fc19554..10315988708 100644 --- a/mysql-test/r/trigger_null-8605.result +++ b/mysql-test/r/trigger_null-8605.result @@ -354,3 +354,13 @@ show columns from t1; Field Type Null Key Default Extra a int(11) NO PRI NULL drop table t1; +create table t1 ( +pk int primary key, +i int, +v1 int as (i) virtual, +v2 int as (i) virtual +); +create trigger tr before update on t1 for each row set @a = 1; +insert into t1 (pk, i) values (null, null); +ERROR 23000: Column 'pk' cannot be null +drop table t1; diff --git a/mysql-test/suite/encryption/disabled.def b/mysql-test/suite/encryption/disabled.def index c2f41b70d03..d92d3495cb8 100644 --- a/mysql-test/suite/encryption/disabled.def +++ b/mysql-test/suite/encryption/disabled.def @@ -10,5 +10,5 @@ # ############################################################################## -innodb_scrub_background : MDEV-8139 background scrubbing does not work reliably -innodb_scrub : MDEV-8139 occasional corruption of delete_3.ibd page 2 +innodb_scrub : MDEV-8139 scrubbing does not work reliably +innodb_scrub_background : MDEV-8139 scrubbing does not work reliably diff --git a/mysql-test/suite/encryption/r/innodb_lotoftables.result b/mysql-test/suite/encryption/r/innodb_lotoftables.result index aea50a1fa67..cf5724b527a 100644 --- a/mysql-test/suite/encryption/r/innodb_lotoftables.result +++ b/mysql-test/suite/encryption/r/innodb_lotoftables.result @@ -86,47 +86,47 @@ Innodb_pages0_read 3 # Restart Success! show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 1 +Innodb_pages0_read 303 show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 1 +Innodb_pages0_read 303 use test; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 1 +Innodb_pages0_read 303 use innodb_encrypted_1; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 1 +Innodb_pages0_read 303 use innodb_encrypted_2; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 1 +Innodb_pages0_read 303 use innodb_encrypted_3; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 1 +Innodb_pages0_read 303 use innodb_encrypted_1; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 1 +Innodb_pages0_read 303 show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 101 +Innodb_pages0_read 303 use innodb_encrypted_2; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 101 +Innodb_pages0_read 303 show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 201 +Innodb_pages0_read 303 use innodb_encrypted_3; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 201 +Innodb_pages0_read 303 show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 301 +Innodb_pages0_read 303 SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; COUNT(*) 100 diff --git a/mysql-test/suite/innodb/include/ibd_convert.pl b/mysql-test/suite/innodb/include/ibd_convert.pl new file mode 100644 index 00000000000..32eef96fd23 --- /dev/null +++ b/mysql-test/suite/innodb/include/ibd_convert.pl @@ -0,0 +1,25 @@ +#!/usr/bin/perl +# Convert tablespace flags to the format understood by MariaDB 10.1.0..10.1.20, +# with the assumption that the flags were correct. + +sub convert_to_mariadb_101 +{ + my ($file, $page_size) = @_; + open(FILE, "+<", $file) or die "Unable to open $file\n"; + sysread(FILE, $_, $page_size)==$page_size||die "Unable to read $file\n"; + sysseek(FILE, 0, 0)||die "Unable to seek $file\n"; + + # FIL_PAGE_DATA + FSP_SPACE_FLAGS = 38 + 16 = 54 bytes from the start + my($flags) = unpack "x[54]N", $_; + my $badflags = ($flags & 0x3f); + my $compression_level=6; + $badflags |= 1<<6|$compression_level<<7 if ($flags & 1 << 16); + $badflags |= ($flags & 15 << 6) << 7; # PAGE_SSIZE + + substr ($_, 54, 4) = pack("N", $badflags); + # Replace the innodb_checksum_algorithm=none checksum + substr ($_, 0, 4) = pack("N", 0xdeadbeef); + substr ($_, $page_size - 8, 4) = pack("N", 0xdeadbeef); + syswrite(FILE, $_, $page_size)==$page_size||die "Unable to write $file\n"; + close(FILE); +} diff --git a/mysql-test/suite/innodb/r/101_compatibility.result b/mysql-test/suite/innodb/r/101_compatibility.result new file mode 100644 index 00000000000..2529eb96190 --- /dev/null +++ b/mysql-test/suite/innodb/r/101_compatibility.result @@ -0,0 +1,48 @@ +# +# MDEV-11623 MariaDB 10.1 fails to start datadir created with +# MariaDB 10.0/MySQL 5.6 using innodb-page-size!=16K +# +call mtr.add_suppression("InnoDB: adjusting FSP_SPACE_FLAGS of tablespace"); +CREATE TABLE tr(a INT)ENGINE=InnoDB ROW_FORMAT=REDUNDANT; +CREATE TABLE tc(a INT)ENGINE=InnoDB ROW_FORMAT=COMPACT; +CREATE TABLE td(a INT)ENGINE=InnoDB ROW_FORMAT=DYNAMIC; +SET INNODB_STRICT_MODE=OFF; +CREATE TABLE tz(a INT)ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1; +SET INNODB_STRICT_MODE=ON; +CREATE TABLE tdd(a INT) ENGINE=InnoDB, DATA DIRECTORY='MYSQL_TMP_DIR'; +CREATE TABLE tp(a INT) ENGINE=InnoDB page_compressed=1; +CREATE TABLE ti(a INT) ENGINE=InnoDB; +FLUSH TABLES ti FOR EXPORT; +backup: ti +UNLOCK TABLES; +ALTER TABLE ti DISCARD TABLESPACE; +restore: ti .ibd and .cfg files +ALTER TABLE ti IMPORT TABLESPACE; +BEGIN; +INSERT INTO tr VALUES(1); +INSERT INTO tc VALUES(1); +INSERT INTO td VALUES(1); +INSERT INTO tz VALUES(1); +INSERT INTO tdd VALUES(1); +INSERT INTO tp VALUES(1); +INSERT INTO ti VALUES(1); +# Kill the server +CHECK TABLE tr,tc,td,tz,tdd,tp,ti; +Table Op Msg_type Msg_text +test.tr check status OK +test.tc check status OK +test.td check status OK +test.tz check status OK +test.tdd check status OK +test.tp check status OK +test.ti check status OK +CHECK TABLE tr,tc,td,tz,tdd,tp,ti; +Table Op Msg_type Msg_text +test.tr check status OK +test.tc check status OK +test.td check status OK +test.tz check status OK +test.tdd check status OK +test.tp check status OK +test.ti check status OK +DROP TABLE tr,tc,td,tz,tdd,tp,ti; diff --git a/mysql-test/suite/innodb/r/doublewrite.result b/mysql-test/suite/innodb/r/doublewrite.result index aa96a5f2d93..6b913f49972 100644 --- a/mysql-test/suite/innodb/r/doublewrite.result +++ b/mysql-test/suite/innodb/r/doublewrite.result @@ -39,6 +39,8 @@ set global innodb_buf_flush_list_now = 1; # Kill the server # Make the first page (page_no=0) of the user tablespace # full of zeroes. +# +# MDEV-11623: Use old FSP_SPACE_FLAGS in the doublewrite buffer. check table t1; Table Op Msg_type Msg_text test.t1 check status OK diff --git a/mysql-test/suite/innodb/r/group_commit_crash.result b/mysql-test/suite/innodb/r/group_commit_crash.result index 80a780ba2c5..f07df897453 100644 --- a/mysql-test/suite/innodb/r/group_commit_crash.result +++ b/mysql-test/suite/innodb/r/group_commit_crash.result @@ -1,4 +1,3 @@ -call mtr.add_suppression("InnoDB: Warning: database page corruption or a failed"); CREATE TABLE t1(a CHAR(255), b CHAR(255), c CHAR(255), diff --git a/mysql-test/suite/innodb/r/group_commit_crash_no_optimize_thread.result b/mysql-test/suite/innodb/r/group_commit_crash_no_optimize_thread.result index 06fdeaef6a7..2cd9f01d7ed 100644 --- a/mysql-test/suite/innodb/r/group_commit_crash_no_optimize_thread.result +++ b/mysql-test/suite/innodb/r/group_commit_crash_no_optimize_thread.result @@ -1,4 +1,3 @@ -call mtr.add_suppression("InnoDB: Warning: database page corruption or a failed"); CREATE TABLE t1(a CHAR(255), b CHAR(255), c CHAR(255), diff --git a/mysql-test/suite/innodb/r/innodb-alter-tempfile.result b/mysql-test/suite/innodb/r/innodb-alter-tempfile.result index 3cc973ca3a3..ce13ad0978b 100644 --- a/mysql-test/suite/innodb/r/innodb-alter-tempfile.result +++ b/mysql-test/suite/innodb/r/innodb-alter-tempfile.result @@ -4,10 +4,6 @@ # Temporary tablename will be unique. This makes sure that future # in-place ALTERs of the same table will not be blocked due to # temporary tablename. -call mtr.add_suppression("InnoDB: Warning: database page corruption or a failed -"); -call mtr.add_suppression("InnoDB: file read of space .* page .*"); -call mtr.add_suppression("InnoDB: Trying to recover it from the doublewrite buffer."); # Crash the server in ha_innobase::commit_inplace_alter_table() CREATE TABLE t1 (f1 INT NOT NULL, f2 INT NOT NULL) ENGINE=innodb; SET debug='d,innodb_alter_commit_crash_before_commit'; diff --git a/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result b/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result index ab6bf0c808e..cb01e5d2465 100644 --- a/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result +++ b/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result @@ -1,10 +1,9 @@ call mtr.add_suppression("InnoDB: Tablespace for table .* is set as discarded."); call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* because the .ibd file is missing. Please refer to .* for how to resolve the issue."); -call mtr.add_suppression("InnoDB: Error: Tablespace flags .* corrupted unused .*"); -call mtr.add_suppression("InnoDB: Tablespace flags: .* corrupted in file: .* "); call mtr.add_suppression("InnoDB: Page 0 at offset 0 looks corrupted in file .*"); -call mtr.add_suppression("InnoDB: Page for tablespace .* "); -flush tables; +call mtr.add_suppression("InnoDB: Page for tablespace "); +call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS=0x"); +FLUSH TABLES; SET SESSION innodb_strict_mode=1; CREATE DATABASE test_wl5522; CREATE TABLE test_wl5522.t1 (c1 INT) ENGINE = Innodb @@ -546,7 +545,7 @@ ERROR HY000: Tablespace has been discarded for table `t1` restore: t1 .ibd and .cfg files SET SESSION debug_dbug="+d,fsp_flags_is_valid_failure"; ALTER TABLE test_wl5522.t1 IMPORT TABLESPACE; -ERROR HY000: Internal error: Cannot reset LSNs in table `test_wl5522`.`t1` : Unsupported +ERROR HY000: Internal error: Cannot reset LSNs in table `test_wl5522`.`t1` : Data structure corruption SET SESSION debug_dbug="-d,fsp_flags_is_valid_failure"; DROP TABLE test_wl5522.t1; unlink: t1.ibd diff --git a/mysql-test/suite/innodb/r/innodb-wl5522-debug.result b/mysql-test/suite/innodb/r/innodb-wl5522-debug.result index 8327ef36909..698950e0d1d 100644 --- a/mysql-test/suite/innodb/r/innodb-wl5522-debug.result +++ b/mysql-test/suite/innodb/r/innodb-wl5522-debug.result @@ -2,13 +2,13 @@ call mtr.add_suppression("InnoDB: Operating system error number .* in a file ope call mtr.add_suppression("InnoDB: The error means the system cannot find the path specified."); call mtr.add_suppression("InnoDB: If you are installing InnoDB, remember that you must create directories yourself, InnoDB does not create them."); call mtr.add_suppression("InnoDB: Cannot open datafile for read-only: .*"); -call mtr.add_suppression("InnoDB: Tablespace flags: .*"); call mtr.add_suppression("InnoDB: Ignoring tablespace .* because it could not be opened."); call mtr.add_suppression("InnoDB: Tablespace for table .* is set as discarded."); call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .*"); -call mtr.add_suppression("InnoDB: Page 0 at offset 0 looks corrupted in file .*"); -call mtr.add_suppression("InnoDB: Page for tablespace .* "); -flush tables; +call mtr.add_suppression("InnoDB: Page 0 at offset 0 looks corrupted in file"); +call mtr.add_suppression("InnoDB: Page for tablespace "); +call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS="); +FLUSH TABLES; SET GLOBAL innodb_file_per_table = 1; SELECT @@innodb_file_per_table; @@innodb_file_per_table @@ -921,7 +921,7 @@ ERROR HY000: Tablespace has been discarded for table `t1` restore: t1 .ibd and .cfg files SET SESSION debug_dbug="+d,fsp_flags_is_valid_failure"; ALTER TABLE test_wl5522.t1 IMPORT TABLESPACE; -ERROR HY000: Internal error: Cannot reset LSNs in table `test_wl5522`.`t1` : Unsupported +ERROR HY000: Internal error: Cannot reset LSNs in table `test_wl5522`.`t1` : Data structure corruption SET SESSION debug_dbug="-d,fsp_flags_is_valid_failure"; DROP TABLE test_wl5522.t1; unlink: t1.ibd diff --git a/mysql-test/suite/innodb/r/xa_recovery.result b/mysql-test/suite/innodb/r/xa_recovery.result index 666a65f8ee1..ae3a06ab9c3 100644 --- a/mysql-test/suite/innodb/r/xa_recovery.result +++ b/mysql-test/suite/innodb/r/xa_recovery.result @@ -1,4 +1,3 @@ -call mtr.add_suppression("InnoDB: Warning: database page corruption or a failed"); CREATE TABLE t1 (a INT) ENGINE=InnoDB; INSERT INTO t1 VALUES (1); connect con1,localhost,root; @@ -8,6 +7,7 @@ XA END 'x'; XA PREPARE 'x'; connection default; call mtr.add_suppression("Found 1 prepared XA transactions"); +# Kill the server disconnect con1; connect con1,localhost,root; SELECT * FROM t1 LOCK IN SHARE MODE; diff --git a/mysql-test/suite/innodb/t/101_compatibility.test b/mysql-test/suite/innodb/t/101_compatibility.test new file mode 100644 index 00000000000..5b94a24852a --- /dev/null +++ b/mysql-test/suite/innodb/t/101_compatibility.test @@ -0,0 +1,102 @@ +--source include/have_innodb.inc +--source include/not_embedded.inc + +-- echo # +-- echo # MDEV-11623 MariaDB 10.1 fails to start datadir created with +-- echo # MariaDB 10.0/MySQL 5.6 using innodb-page-size!=16K +-- echo # + +# This is actually testing the opposite: starting the fixed 10.1 with +# buggy 10.1 files (by manually converting the flags in the files). + +call mtr.add_suppression("InnoDB: adjusting FSP_SPACE_FLAGS of tablespace"); +let INNODB_PAGE_SIZE=`select @@innodb_page_size`; +let MYSQLD_DATADIR=`select @@datadir`; + +CREATE TABLE tr(a INT)ENGINE=InnoDB ROW_FORMAT=REDUNDANT; +CREATE TABLE tc(a INT)ENGINE=InnoDB ROW_FORMAT=COMPACT; +CREATE TABLE td(a INT)ENGINE=InnoDB ROW_FORMAT=DYNAMIC; +--disable_warnings +# ROW_FORMAT=COMPRESSED is not available with innodb_page_size>16k +SET INNODB_STRICT_MODE=OFF; +CREATE TABLE tz(a INT)ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1; +SET INNODB_STRICT_MODE=ON; +--enable_warnings + +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +EVAL CREATE TABLE tdd(a INT) ENGINE=InnoDB, DATA DIRECTORY='$MYSQL_TMP_DIR'; + +CREATE TABLE tp(a INT) ENGINE=InnoDB page_compressed=1; +CREATE TABLE ti(a INT) ENGINE=InnoDB; +FLUSH TABLES ti FOR EXPORT; +perl; +do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl"; +ib_backup_tablespaces("test", "ti"); +EOF +UNLOCK TABLES; +ALTER TABLE ti DISCARD TABLESPACE; +perl; +do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl"; +ib_discard_tablespaces("test", "ti"); +ib_restore_tablespaces("test", "ti"); +do "$ENV{MTR_SUITE_DIR}/include/ibd_convert.pl"; +my $ps = $ENV{INNODB_PAGE_SIZE}; +my $dd = $ENV{MYSQLD_DATADIR}; +convert_to_mariadb_101("$dd/test/ti.ibd", $ps); +EOF + +ALTER TABLE ti IMPORT TABLESPACE; + +BEGIN; +INSERT INTO tr VALUES(1); +INSERT INTO tc VALUES(1); +INSERT INTO td VALUES(1); +INSERT INTO tz VALUES(1); +INSERT INTO tdd VALUES(1); +INSERT INTO tp VALUES(1); +INSERT INTO ti VALUES(1); + +--source include/kill_mysqld.inc + +perl; +do "$ENV{MTR_SUITE_DIR}/include/ibd_convert.pl"; +my $ps = $ENV{INNODB_PAGE_SIZE}; +my $dd = $ENV{MYSQLD_DATADIR}; + +convert_to_mariadb_101("$dd/ibdata1", $ps); +convert_to_mariadb_101("$dd/test/tr.ibd", $ps); +convert_to_mariadb_101("$dd/test/tc.ibd", $ps); +convert_to_mariadb_101("$dd/test/td.ibd", $ps); +convert_to_mariadb_101("$dd/test/tz.ibd", 1024) if $ps<32768; +convert_to_mariadb_101("$dd/test/tp.ibd", $ps); +convert_to_mariadb_101("$dd/test/ti.ibd", $ps); +convert_to_mariadb_101("$ENV{MYSQL_TMP_DIR}/test/tdd.ibd", $ps); +EOF + +--source include/start_mysqld.inc +CHECK TABLE tr,tc,td,tz,tdd,tp,ti; +--source include/shutdown_mysqld.inc + +perl; +do "$ENV{MTR_SUITE_DIR}/include/ibd_convert.pl"; +my $ps = $ENV{INNODB_PAGE_SIZE}; +my $dd = $ENV{MYSQLD_DATADIR}; + +convert_to_mariadb_101("$dd/ibdata1", $ps); +convert_to_mariadb_101("$dd/test/tr.ibd", $ps); +convert_to_mariadb_101("$dd/test/tc.ibd", $ps); +convert_to_mariadb_101("$dd/test/td.ibd", $ps); +convert_to_mariadb_101("$dd/test/tz.ibd", 1024) if $ps<32768; +convert_to_mariadb_101("$dd/test/tp.ibd", $ps); +convert_to_mariadb_101("$dd/test/ti.ibd", $ps); +convert_to_mariadb_101("$ENV{MYSQL_TMP_DIR}/test/tdd.ibd", $ps); +EOF + +--let $restart_parameters=--innodb-read-only +--source include/start_mysqld.inc +CHECK TABLE tr,tc,td,tz,tdd,tp,ti; +--source include/shutdown_mysqld.inc + +--let $restart_parameters= +--source include/start_mysqld.inc +DROP TABLE tr,tc,td,tz,tdd,tp,ti; diff --git a/mysql-test/suite/innodb/t/doublewrite.test b/mysql-test/suite/innodb/t/doublewrite.test index e1984319cb7..bb0733fd1a0 100644 --- a/mysql-test/suite/innodb/t/doublewrite.test +++ b/mysql-test/suite/innodb/t/doublewrite.test @@ -13,11 +13,10 @@ SET GLOBAL innodb_fast_shutdown = 0; --source include/restart_mysqld.inc --disable_query_log -call mtr.add_suppression("InnoDB: Database page [0-9]+:1 contained only zeroes."); call mtr.add_suppression("Header page consists of zero bytes"); call mtr.add_suppression("Checksum mismatch in datafile"); call mtr.add_suppression("but the innodb_page_size start-up parameter is"); -call mtr.add_suppression("Database page corruption"); +call mtr.add_suppression("adjusting FSP_SPACE_FLAGS"); --enable_query_log let INNODB_PAGE_SIZE=`select @@innodb_page_size`; @@ -65,14 +64,48 @@ set global innodb_buf_flush_list_now = 1; --echo # Make the first page (page_no=0) of the user tablespace --echo # full of zeroes. +--echo # +--echo # MDEV-11623: Use old FSP_SPACE_FLAGS in the doublewrite buffer. + perl; use IO::Handle; my $fname= "$ENV{'MYSQLD_DATADIR'}test/t1.ibd"; +my $page_size = $ENV{INNODB_PAGE_SIZE}; +my $page; open(FILE, "+<", $fname) or die; -FILE->autoflush(1); -binmode FILE; -print FILE chr(0) x ($ENV{'INNODB_PAGE_SIZE'}); +sysread(FILE, $page, $page_size)==$page_size||die "Unable to read $name\n"; +sysseek(FILE, 0, 0)||die "Unable to seek $fname\n"; +die unless syswrite(FILE, chr(0) x $page_size, $page_size) == $page_size; close FILE; + +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}ibdata1")||die "cannot open ibdata1\n"; +sysseek(FILE, 6 * $page_size - 190, 0)||die "Unable to seek ibdata1\n"; +sysread(FILE, $_, 12) == 12||die "Unable to read TRX_SYS\n"; +my($magic,$d1,$d2)=unpack "NNN", $_; +die "magic=$magic, $d1, $d2\n" unless $magic == 536853855 && $d2 >= $d1 + 64; +sysseek(FILE, $d1 * $page_size, 0)||die "Unable to seek ibdata1\n"; +# Find the page in the doublewrite buffer +for (my $d = $d1; $d < $d2 + 64; $d++) +{ + sysread(FILE, $_, $page_size)==$page_size||die "Cannot read doublewrite\n"; + next unless $_ eq $page; + sysseek(FILE, $d * $page_size, 0)||die "Unable to seek ibdata1\n"; + # Write buggy MariaDB 10.1.x FSP_SPACE_FLAGS to the doublewrite buffer + my($flags) = unpack "x[54]N", $_; + my $badflags = ($flags & 0x3f); + my $compression_level=6; + $badflags |= 1<<6|$compression_level<<7 if ($flags & 1 << 16); + $badflags |= ($flags & 15 << 6) << 7; # PAGE_SSIZE + + substr ($_, 54, 4) = pack("N", $badflags); + # Replace the innodb_checksum_algorithm=none checksum + substr ($_, 0, 4) = pack("N", 0xdeadbeef); + substr ($_, $page_size - 8, 4) = pack("N", 0xdeadbeef); + syswrite(FILE, $_, $page_size)==$page_size||die; + close(FILE); + exit 0; +} +die "Did not find the page in the doublewrite buffer ($d1,$d2)\n"; EOF --source include/start_mysqld.inc diff --git a/mysql-test/suite/innodb/t/group_commit_crash.test b/mysql-test/suite/innodb/t/group_commit_crash.test index cad349819bd..7ad0d9d1e74 100644 --- a/mysql-test/suite/innodb/t/group_commit_crash.test +++ b/mysql-test/suite/innodb/t/group_commit_crash.test @@ -9,8 +9,6 @@ --source include/have_debug.inc --source include/have_log_bin.inc -call mtr.add_suppression("InnoDB: Warning: database page corruption or a failed"); - let $file_format_max=`SELECT @@innodb_file_format_max`; CREATE TABLE t1(a CHAR(255), b CHAR(255), diff --git a/mysql-test/suite/innodb/t/group_commit_crash_no_optimize_thread.test b/mysql-test/suite/innodb/t/group_commit_crash_no_optimize_thread.test index 8d1f460b64b..9dc2557e687 100644 --- a/mysql-test/suite/innodb/t/group_commit_crash_no_optimize_thread.test +++ b/mysql-test/suite/innodb/t/group_commit_crash_no_optimize_thread.test @@ -9,8 +9,6 @@ --source include/have_debug.inc --source include/have_log_bin.inc -call mtr.add_suppression("InnoDB: Warning: database page corruption or a failed"); - let $file_format_max=`SELECT @@innodb_file_format_max`; CREATE TABLE t1(a CHAR(255), b CHAR(255), diff --git a/mysql-test/suite/innodb/t/innodb-alter-tempfile.test b/mysql-test/suite/innodb/t/innodb-alter-tempfile.test index e1e736fc678..ec1ea35f1cf 100644 --- a/mysql-test/suite/innodb/t/innodb-alter-tempfile.test +++ b/mysql-test/suite/innodb/t/innodb-alter-tempfile.test @@ -20,13 +20,6 @@ --echo # in-place ALTERs of the same table will not be blocked due to --echo # temporary tablename. -# As we intentionally crash below, there could be partially written -# pages that are then recovered from the doublewrite buffer -call mtr.add_suppression("InnoDB: Warning: database page corruption or a failed -"); -call mtr.add_suppression("InnoDB: file read of space .* page .*"); -call mtr.add_suppression("InnoDB: Trying to recover it from the doublewrite buffer."); - let datadir= `select @@datadir`; --let $_server_id= `SELECT @@server_id` diff --git a/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test b/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test index 3b71bd802ef..2858fa7c8e1 100644 --- a/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test +++ b/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test @@ -19,11 +19,10 @@ call mtr.add_suppression("InnoDB: Tablespace for table .* is set as discarded."); call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* because the .ibd file is missing. Please refer to .* for how to resolve the issue."); -call mtr.add_suppression("InnoDB: Error: Tablespace flags .* corrupted unused .*"); -call mtr.add_suppression("InnoDB: Tablespace flags: .* corrupted in file: .* "); call mtr.add_suppression("InnoDB: Page 0 at offset 0 looks corrupted in file .*"); -call mtr.add_suppression("InnoDB: Page for tablespace .* "); -flush tables; +call mtr.add_suppression("InnoDB: Page for tablespace "); +call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS=0x"); +FLUSH TABLES; let MYSQLD_DATADIR =`SELECT @@datadir`; let $pathfix=/: '.*test_wl5522.*t1.ibd'/: 'test_wl5522_t1.ibd'/; diff --git a/mysql-test/suite/innodb/t/innodb-wl5522-debug.test b/mysql-test/suite/innodb/t/innodb-wl5522-debug.test index e537f5c0fc3..6a94c73e622 100644 --- a/mysql-test/suite/innodb/t/innodb-wl5522-debug.test +++ b/mysql-test/suite/innodb/t/innodb-wl5522-debug.test @@ -19,13 +19,13 @@ call mtr.add_suppression("InnoDB: Operating system error number .* in a file ope call mtr.add_suppression("InnoDB: The error means the system cannot find the path specified."); call mtr.add_suppression("InnoDB: If you are installing InnoDB, remember that you must create directories yourself, InnoDB does not create them."); call mtr.add_suppression("InnoDB: Cannot open datafile for read-only: .*"); -call mtr.add_suppression("InnoDB: Tablespace flags: .*"); call mtr.add_suppression("InnoDB: Ignoring tablespace .* because it could not be opened."); call mtr.add_suppression("InnoDB: Tablespace for table .* is set as discarded."); call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .*"); -call mtr.add_suppression("InnoDB: Page 0 at offset 0 looks corrupted in file .*"); -call mtr.add_suppression("InnoDB: Page for tablespace .* "); -flush tables; +call mtr.add_suppression("InnoDB: Page 0 at offset 0 looks corrupted in file"); +call mtr.add_suppression("InnoDB: Page for tablespace "); +call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS="); +FLUSH TABLES; let MYSQLD_DATADIR =`SELECT @@datadir`; let $innodb_file_per_table = `SELECT @@innodb_file_per_table`; diff --git a/mysql-test/suite/innodb/t/xa_recovery.test b/mysql-test/suite/innodb/t/xa_recovery.test index 32373d63d14..9f25e4d1f30 100644 --- a/mysql-test/suite/innodb/t/xa_recovery.test +++ b/mysql-test/suite/innodb/t/xa_recovery.test @@ -12,11 +12,6 @@ if (`select plugin_auth_version <= "5.6.24" from information_schema.plugins wher FLUSH TABLES; --enable_query_log -# -# We kill server belown with timeout 0 that is not fully safe -# -call mtr.add_suppression("InnoDB: Warning: database page corruption or a failed"); - CREATE TABLE t1 (a INT) ENGINE=InnoDB; INSERT INTO t1 VALUES (1); connect (con1,localhost,root); @@ -25,15 +20,8 @@ connection default; call mtr.add_suppression("Found 1 prepared XA transactions"); -# Kill and restart the server. --- exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --- shutdown_server 0 --- source include/wait_until_disconnected.inc - --- exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --- enable_reconnect --- source include/wait_until_connected_again.inc --- disable_reconnect +--source include/kill_mysqld.inc +--source include/start_mysqld.inc disconnect con1; connect (con1,localhost,root); diff --git a/mysql-test/suite/rpl/r/rpl_gtid_crash.result b/mysql-test/suite/rpl/r/rpl_gtid_crash.result index 00cae145673..7b6e95bf718 100644 --- a/mysql-test/suite/rpl/r/rpl_gtid_crash.result +++ b/mysql-test/suite/rpl/r/rpl_gtid_crash.result @@ -4,7 +4,6 @@ connection server_1; call mtr.add_suppression("Checking table:"); call mtr.add_suppression("client is using or hasn't closed the table properly"); call mtr.add_suppression("Table .* is marked as crashed and should be repaired"); -call mtr.add_suppression("InnoDB: Warning: database page corruption or a failed"); flush tables; ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; CREATE TABLE t1 (a INT PRIMARY KEY, b INT) ENGINE=InnoDB; diff --git a/mysql-test/suite/rpl/t/rpl_gtid_crash.test b/mysql-test/suite/rpl/t/rpl_gtid_crash.test index df3ba9a3420..b81cbd38cd3 100644 --- a/mysql-test/suite/rpl/t/rpl_gtid_crash.test +++ b/mysql-test/suite/rpl/t/rpl_gtid_crash.test @@ -12,11 +12,6 @@ call mtr.add_suppression("Checking table:"); call mtr.add_suppression("client is using or hasn't closed the table properly"); call mtr.add_suppression("Table .* is marked as crashed and should be repaired"); -# We have seen this warning a couple of times in Buildbot. Since we crash the -# server deliberately, it seems possible that we could in rare cases crash in -# the middle of a page write. The page is recovered from the doublewrite -# buffer ("[Note] InnoDB: Recovered the page from the doublewrite buffer."). -call mtr.add_suppression("InnoDB: Warning: database page corruption or a failed"); flush tables; ALTER TABLE mysql.gtid_slave_pos ENGINE=InnoDB; diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_embedded,32bit.rdiff b/mysql-test/suite/sys_vars/r/sysvars_server_embedded,32bit.rdiff index 0f34179a16d..66f38d270b0 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_embedded,32bit.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_server_embedded,32bit.rdiff @@ -997,12 +997,12 @@ VARIABLE_SCOPE SESSION -VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_TYPE INT UNSIGNED - VARIABLE_COMMENT Seconds between sending progress reports to the client for time-consuming statements. Set to 0 to disable progress reporting. - NUMERIC_MIN_VALUE 0 - NUMERIC_MAX_VALUE 4294967295 + VARIABLE_COMMENT Seconds between sending progress reports to the client for time-consuming statements. Set to 0 to disable progress reporting. + NUMERIC_MIN_VALUE 0 + NUMERIC_MAX_VALUE 4294967295 @@ -3123,7 +3123,7 @@ - GLOBAL_VALUE_ORIGIN COMPILE-TIME - DEFAULT_VALUE 16384 + GLOBAL_VALUE_ORIGIN COMPILE-TIME + DEFAULT_VALUE 16384 VARIABLE_SCOPE SESSION -VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_TYPE INT UNSIGNED diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result index f37ca909836..968312124dc 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result @@ -2053,6 +2053,20 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED +VARIABLE_NAME MAX_SESSION_MEM_USED +SESSION_VALUE 9223372036854775807 +GLOBAL_VALUE 9223372036854775807 +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE 9223372036854775807 +VARIABLE_SCOPE SESSION +VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_COMMENT Amount of memory a single user session is allowed to allocate. This limits the value of the session variable MEM_USED +NUMERIC_MIN_VALUE 8192 +NUMERIC_MAX_VALUE 18446744073709551615 +NUMERIC_BLOCK_SIZE 1 +ENUM_VALUE_LIST NULL +READ_ONLY NO +COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME MAX_SORT_LENGTH SESSION_VALUE 1024 GLOBAL_VALUE 1024 diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff index 88216992587..e9fa72fb5b7 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded,32bit.rdiff @@ -997,12 +997,12 @@ VARIABLE_SCOPE SESSION -VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_TYPE INT UNSIGNED - VARIABLE_COMMENT Seconds between sending progress reports to the client for time-consuming statements. Set to 0 to disable progress reporting. - NUMERIC_MIN_VALUE 0 - NUMERIC_MAX_VALUE 4294967295 + VARIABLE_COMMENT Seconds between sending progress reports to the client for time-consuming statements. Set to 0 to disable progress reporting. + NUMERIC_MIN_VALUE 0 + NUMERIC_MAX_VALUE 4294967295 @@ -3347,7 +3347,7 @@ - GLOBAL_VALUE_ORIGIN COMPILE-TIME - DEFAULT_VALUE 16384 + GLOBAL_VALUE_ORIGIN COMPILE-TIME + DEFAULT_VALUE 16384 VARIABLE_SCOPE SESSION -VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_TYPE INT UNSIGNED @@ -1204,7 +1204,7 @@ NUMERIC_MAX_VALUE 1048576 @@ -4719,7 +4719,7 @@ GLOBAL_VALUE_ORIGIN AUTO - DEFAULT_VALUE 256 + DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL -VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_TYPE INT UNSIGNED diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result index 15373702057..9f006cbdb9b 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result @@ -2249,6 +2249,20 @@ NUMERIC_BLOCK_SIZE 1 ENUM_VALUE_LIST NULL READ_ONLY NO COMMAND_LINE_ARGUMENT REQUIRED +VARIABLE_NAME MAX_SESSION_MEM_USED +SESSION_VALUE 9223372036854775807 +GLOBAL_VALUE 9223372036854775807 +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE 9223372036854775807 +VARIABLE_SCOPE SESSION +VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_COMMENT Amount of memory a single user session is allowed to allocate. This limits the value of the session variable MEM_USED +NUMERIC_MIN_VALUE 8192 +NUMERIC_MAX_VALUE 18446744073709551615 +NUMERIC_BLOCK_SIZE 1 +ENUM_VALUE_LIST NULL +READ_ONLY NO +COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME MAX_SORT_LENGTH SESSION_VALUE 1024 GLOBAL_VALUE 1024 diff --git a/mysql-test/t/cast.test b/mysql-test/t/cast.test index 58f91571ebb..5316c12c857 100644 --- a/mysql-test/t/cast.test +++ b/mysql-test/t/cast.test @@ -472,3 +472,42 @@ select collation(cast("a" as char(10) ascii binary)); select collation(cast("a" as char(10) binary charset utf8)); select collation(cast("a" as char(10) binary ascii)); +--echo # +--echo # MDEV-11030 Assertion `precision > 0' failed in decimal_bin_size +--echo # + +SELECT * FROM (SELECT IFNULL(CONVERT(NULL, UNSIGNED), NULL)) sq; + +CREATE TABLE t1 AS SELECT IFNULL(CONVERT(NULL, UNSIGNED), NULL); +SHOW CREATE TABLE t1; +DROP TABLE t1; + +CREATE TABLE t1 AS SELECT COALESCE(CONVERT(NULL, UNSIGNED), NULL); +SHOW CREATE TABLE t1; +DROP TABLE t1; + +CREATE TABLE t1 AS SELECT CASE WHEN TRUE THEN CONVERT(NULL, UNSIGNED) ELSE NULL END; +SHOW CREATE TABLE t1; +DROP TABLE t1; + +CREATE TABLE t1 AS SELECT IFNULL(CONVERT(NULL,SIGNED),CONVERT(NULL,UNSIGNED)) AS a; +SHOW CREATE TABLE t1; +DROP TABLE t1; + +CREATE TABLE t1 AS SELECT + -1, + CONVERT(NULL,SIGNED), + CONCAT(CONVERT(NULL,SIGNED)), + 1, + CONVERT(NULL,UNSIGNED), + CONCAT(CONVERT(NULL,UNSIGNED)); +SHOW CREATE TABLE t1; +DROP TABLE t1; + +CREATE TABLE t1 AS SELECT + CONVERT('',SIGNED), + CONCAT(CONVERT('',SIGNED)), + CONVERT('',UNSIGNED), + CONCAT(CONVERT('',UNSIGNED)); +SHOW CREATE TABLE t1; +DROP TABLE t1; diff --git a/mysql-test/t/trigger_null-8605.test b/mysql-test/t/trigger_null-8605.test index 748768a1849..34a57654b48 100644 --- a/mysql-test/t/trigger_null-8605.test +++ b/mysql-test/t/trigger_null-8605.test @@ -373,3 +373,17 @@ show columns from t1; insert into t1 (a) values (3); show columns from t1; drop table t1; + +# +# MDEV-11551 Server crashes in Field::is_real_null +# +create table t1 ( + pk int primary key, + i int, + v1 int as (i) virtual, + v2 int as (i) virtual +); +create trigger tr before update on t1 for each row set @a = 1; +--error ER_BAD_NULL_ERROR +insert into t1 (pk, i) values (null, null); +drop table t1; diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index 7b5c2756f11..42985e1d66d 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -23,66 +23,83 @@ # ############################################################################## -main.alter_table : Modified in 10.1.19 -main.analyze_stmt_slow_query_log : MDEV-7558 - wrong result -main.create : Modified in 10.1.20 +main.alter_table : Modified in 10.1.21 +main.alter_table_trans : MDEV-11805 - timeout +main.analyze_stmt_slow_query_log : Modified in 10.1.21 +main.cast : Modified in 10.1.21 +main.create : Modified in 10.1.21 main.create_delayed : MDEV-10605 - failed with timeout main.create_drop_binlog : Uses binlog_start_pos.inc modified in 10.1.20 -main.create_or_replace : Modified in 10.1.19 +main.ctype_ucs : Modified in 10.1.21 +main.ctype_ucs2_def : Modified in 10.1.21 +main.ctype_ucs2_query_cache : Modified in 10.1.21 +main.ctype_utf16 : Modified in 10.1.21 +main.ctype_utf16_def : Modified in 10.1.21 main.ctype_utf16le : MDEV-10675: timeout or extra warnings +main.ctype_utf32 : Modified in 10.1.21 main.ctype_utf8 : Modified in 10.1.20 main.ctype_utf8mb4 : Modified in 10.1.20 main.default : Modified in 10.1.20 main.derived : Modified in 10.1.20 main.derived_view : Modified in 10.1.20 -main.drop : Modified in 10.1.19 main.events_restart : MDEV-11221: assertion failure +main.events_slowlog : Added in 10.1.21 main.fulltext_charsets : Added in 10.1.20 -main.func_time : Modified in 10.1.20 +main.func_time : Modified in 10.1.21 main.group_by : Modified in 10.1.20 main.group_by_innodb : Modified in 10.1.20 main.host_cache_size_functionality : MDEV-10606 - sporadic failure on shutdown main.index_intersect_innodb : MDEV-10643 - failed with timeout -main.index_merge_innodb : MDEV-7142 - Wrong execution plan -main.information_schema : Modified in 10.1.19 +main.index_merge_innodb : MDEV-7142 - Wrong execution plan, also modified in 10.1.21 +main.information_schema_part : Modified in 10.1.21 main.innodb_mysql_lock : MDEV-7861 - sporadic lock detection failure +main.join_cache : Modified in 10.1.21 main.kill_processlist-6619 : MDEV-10793 - wrong result in processlist -main.loaddata : Modified in 10.1.20 +main.loaddata : Modified in 10.1.21 +main.log_slow : Modified in 10.1.21 main.lowercase_fs_on : Uses search_pattern_in_file.inc modified in 10.1.20 main.mdev-504 : MDEV-10607 - sporadic "can't connect" main.mdev375 : MDEV-10607 - sporadic "can't connect" main.merge : MDEV-10607 - sporadic "can't connect" -main.mysql : Modified in 10.1.19 -main.mysql_not_windows : Modified in 10.1.19 main.mysqlbinlog : Uses binlog_start_pos.inc modified in 10.1.20 main.mysqldump-max : Uses binlog_start_pos.inc modified in 10.1.20 -main.mysqldump-nl : Added in 10.1.19 -main.mysqltest : MDEV-9269 - fails on Alpha; also modified in 10.1.19 +main.mysqlslap : MDEV-11801 - timeout +main.mysqltest : MDEV-9269 - fails on Alpha main.named_pipe : Uses search_pattern_in_file.inc modified in 10.1.20 -main.null : Modified in 10.1.19 +main.order_by : Modified in 10.1.21 main.order_by_optimizer_innodb : MDEV-10683 - wrong execution plan main.parser : Modified in 10.1.20 -main.pool_of_threads : MDEV-10100 - sporadic error on detecting max connections +main.pool_of_threads : Modified in 10.1.21 main.ps : MDEV-11017 - sporadic wrong Prepared_stmt_count main.selectivity : Modified in 10.1.20 -main.selectivity_innodb : Modified in 10.1.19 main.show_explain : MDEV-10674 - sporadic failure -main.sp : Modified in 10.1.20 +main.signal_demo3 : MDEV-11720 - Thread stack overrun on labrador +main.sp : Modified in 10.1.21 +main.sp-prelocking : Modified in 10.1.21 main.sp-security : MDEV-10607 - sporadic "can't connect" main.stat_tables_par_innodb : MDEV-10515 - sporadic wrong results main.statistics : Modified in 10.1.20 main.status : MDEV-8510 - sporadic wrong result -main.subselect : Modified in 10.1.20 +main.subselect : Modified in 10.1.21 +main.subselect2 : Modified in 10.1.21 +main.subselect4 : Modified in 10.1.21 main.subselect_innodb : MDEV-10614 - sporadic wrong results +main.subselect_no_exists_to_in : Uses subselect.test modified in 10.1.21 +main.subselect_no_mat : Uses subselect.test modified in 10.1.21 +main.subselect_no_opts : Uses subselect.test modified in 10.1.21 +main.subselect_no_scache : Uses subselect.test modified in 10.1.21 +main.subselect_no_semijoin : Uses subselect.test modified in 10.1.21 +main.trigger_null-8605 : Modified in 10.1.21 main.type_datetime_hires : MDEV-10687 - timeout main.type_decimal : Modified in 10.1.20 -main.view : Uses search_pattern_in_file.inc modified in 10.1.20 +main.union : Modified in 10.1.21 +main.view : Modified in 10.1.21 main.wait_timeout_not_windows : Uses search_pattern_in_file.inc modified in 10.1.20 #---------------------------------------------------------------- archive.archive-big : MDEV-10615 - table is marked as crashed -archive.discover : MDEV-10510 - table is marked as crashed +archive.discover : MDEV-10510 - table is marked as crashed; modified in 10.1.21 #---------------------------------------------------------------- @@ -101,6 +118,8 @@ binlog_encryption.* : Added in 10.1.20 #---------------------------------------------------------------- +connect.jdbc : Modified in 10.1.21 +connect.jdbc_new : Modified in 10.1.21 connect.tbl : MDEV-9844, MDEV-10179 - sporadic crashes, valgrind warnings, wrong results #---------------------------------------------------------------- @@ -115,27 +134,28 @@ encryption.filekeys_nofile : Uses search_pattern_in_file.inc mo encryption.filekeys_syntax : Uses search_pattern_in_file.inc modified in 10.1.20 encryption.filekeys_tooshort : Uses search_pattern_in_file.inc modified in 10.1.20 encryption.filekeys_unencfile : Uses search_pattern_in_file.inc modified in 10.1.20 -encryption.innodb-bad-key-change : uses keys2.txt modified in 10.1.19 -encryption.innodb-bad-key-change2 : uses keys2.txt modified in 10.1.19 -encryption.innodb-bad-key-change3 : Uses search_pattern_in_file.inc modified in 10.1.20 -encryption.innodb-bad-key-change4 : uses keys2.txt modified in 10.1.19 -encryption.innodb-bad-key-change5 : uses keys2.txt modified in 10.1.19 -encryption.innodb-bad-key-shutdown : MDEV-9105 - valgrind warnings, assertion failures, and uses keys2.txt modified in 10.1.19 -encryption.innodb-discard-import : Uses search_pattern_in_file.inc modified in 10.1.20 -encryption.innodb_encryption_discard_import : MDEV-11218 - wrong result, also uses search_pattern_in_file.inc modified in 10.1.20 +encryption.innodb-bad-key-change3 : Modified in 10.1.21 +encryption.innodb-bad-key-shutdown : MDEV-9105 - valgrind warnings, assertion failures +encryption.innodb-discard-import : Modified in 10.1.21 +encryption.innodb-discard-import-change : Modified in 10.1.21 +encryption.innodb_encryption_discard_import : MDEV-11218 - wrong result, also modified in 10.1.21 encryption.innodb_encryption_filekeys : MDEV-9962 - timeouts +encryption.innodb_encryption-page-compression : MDEV-11420 - Trying to access missing tablespace encryption.innodb_encryption_row_compressed : Uses search_pattern_in_file.inc modified in 10.1.20 encryption.innodb_first_page : MDEV-10689 - crashes -encryption.innodb-log-encrypt : Uses search_pattern_in_file.inc modified in 10.1.20 -encryption.innodb_lotoftables : MDEV-11531 - InnoDB error -encryption.innodb-missing-key : Added in 10.1.19 +encryption.innodb-log-encrypt : Modified in 10.1.21 +encryption.innodb_lotoftables : MDEV-11531 - InnoDB error, also modified in 10.1.21 +encryption.innodb-missing-key : MDEV-9359 - assertion failure encryption.innodb_onlinealter_encryption : MDEV-10099 - wrong results; also uses search_pattern_in_file.inc modified in 10.1.20 encryption.innodb-page_encryption : MDEV-10641 - mutex problem -encryption.innodb_page_encryption_key_change : uses keys2.txt modified in 10.1.19 +encryption.innodb_scrub : MDEV-8139, also was modified in 10.1.21 +encryption.innodb_scrub_background : MDEV_8139, also was modified in 10.1.21 +encryption.innodb_scrub_compressed : MDEV-8139; also was modified and re-enabled in 10.1.21 #---------------------------------------------------------------- -extra/binlog_tests.database : Modified in 10.1.19 (merge) +engines/iuds.* : Not maintained in timely manner +engines/funcs.* : Not maintained in timely manner #---------------------------------------------------------------- @@ -147,51 +167,57 @@ federated.federated_transactions : MDEV-10617, MDEV-10417 - Wrong checksum, time #---------------------------------------------------------------- funcs_1.processlist_val_no_prot : MDEV-11223 - Wrong result + +funcs_2.innodb_charset : Modified in 10.1.21 funcs_2.memory_charset : MDEV-10290 - Timeout funcs_2.myisam_charset : MDEV-11535 - Timeout #---------------------------------------------------------------- -galera.* : Added to default suites in 10.1.19 - -galera.galera_var_dirty_reads : Modified in 10.1.20 +galera.galera_var_cluster_address : Modified in 10.1.21 +galera.galera_var_dirty_reads : Modified in 10.1.21 +galera.MW-284 : Modified in 10.1.21 galera.rpl_row_annotate : Uses binlog_start_pos.inc modified in 10.1.20 +galera_split_brain : Modified in 10.1.21 -galera_3nodes.* : Added to default suites in 10.1.19, MDEV-11490 +galera_3nodes.* : MDEV-11490 - Warnings not suppressed #---------------------------------------------------------------- +innodb.101_compatibility : Added in 10.1.21 innodb.binlog_consistent : MDEV-10618 - Server fails to start; also uses binlog_start_pos.inc modified in 10.1.20 +innodb.doublewrite : Added in 10.1.21 innodb.group_commit_binlog_pos : Uses binlog_start_pos.inc modified in 10.1.20 innodb.group_commit_binlog_pos_no_optimize_thread : Uses binlog_start_pos.inc modified in 10.1.20 +innodb.group_commit_crash : Modified in 10.1.21 +innodb.group_commit_crash_no_optimize_thread : Modified in 10.1.21 innodb.innodb-alter-table : MDEV-10619 - Testcase timeout -innodb.innodb-bug-14068765 : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 -innodb.innodb-bug-14084530 : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 +innodb.innodb-bug-14068765 : MDEV-9105 - valgrind warnings, assertion failures +innodb.innodb-bug-14084530 : MDEV-9105 - valgrind warnings, assertion failures +innodb.innodb_bug14147491 : MDEV-11808, also modified in 10.1.21 +innodb.innodb_bug14676111 : MDEV-11802 - wrong result innodb.innodb_bug30423 : MDEV-7311 - Wrong number of rows in the plan -innodb.innodb-change-buffer-recovery : Uses search_pattern_in_file.inc modified in 10.1.20 +innodb.innodb-change-buffer-recovery : Modified in 10.1.21 innodb.innodb_defragment_fill_factor : Modified in 10.1.20 innodb.innodb-lock-schedule-algorithm : Modified in 10.1.20 innodb.innodb-page_compression_zip : MDEV-10641 - mutex problem innodb.innodb_stats : MDEV-10682 - wrong result innodb.innodb_sys_semaphore_waits : MDEV-10331 - wrong result -innodb.innodb-wl5522 : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 -innodb.innodb-wl5522-1 : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 -innodb.innodb-wl5522-debug : Modified in 10.1.19 -innodb.innodb-wl5522-debug-zip : Modified in 10.1.20 -innodb.innodb-wl5522-zip : MDEV-9105 - valgrind warnings, assertion failures, also uses innodb-util.pl added in 10.1.19 +innodb.innodb-wl5522 : MDEV-9105 - valgrind warnings, assertion failures +innodb.innodb-wl5522-1 : MDEV-9105 - valgrind warnings, assertion failures +innodb.innodb-wl5522-debug-zip : Modified in 10.1.21 +innodb.log_data_file_size : Added in 10.1.21 innodb.table_index_statistics : Added in 10.1.20 innodb.trigger : Modified in 10.1.20 -innodb.xa_recovery : Modified in 10.1.19 +innodb.xa_recovery : Modified in 10.1.21 #---------------------------------------------------------------- innodb_fts.create : Added in 10.1.20 -innodb_fts.innodb_fts_misc : MDEV-11233 - Crash on CREATE FULLTEXT INDEX #---------------------------------------------------------------- maria.collations : Added in 10.1.20 -maria.encrypt-wrong-key : uses keys2.txt modified in 10.1.19 maria.maria-connect : Uses binlog_start_pos.inc modified in 10.1.20 #---------------------------------------------------------------- @@ -225,24 +251,26 @@ perfschema.func_file_io : MDEV-5708 - fails for s390x perfschema.func_mutex : MDEV-5708 - fails for s390x perfschema.setup_actors : MDEV-10679 - rare crash perfschema.socket_summary_by_event_name_func : MDEV-10622 - Socket summary tables do not match +perfschema.stage_mdl_global : MDEV-11803 - wrong result on slow builders perfschema.threads_mysql : MDEV-10677 - sporadic wrong result #---------------------------------------------------------------- +plugins.cracklib_password_check : MDEV-11650 - valgrind warnings plugins.feedback_plugin_send : MDEV-7932 - ssl failed for url plugins.server_audit : MDEV-9562 - crashes on sol10-sparc plugins.thread_pool_server_audit : MDEV-9562 - crashes on sol10-sparc - +plugins.two_password_validations : MDEV-11650 - valgrind warnings #---------------------------------------------------------------- roles.role_case_sensitive-10744 : Added in 10.1.20 roles.create_and_drop_role : Modified in 10.1.20 -roles.create_and_grant_role : MDEV-11533 - Extra grant in output #---------------------------------------------------------------- rpl.last_insert_id : MDEV-10625 - warnings in error log +rpl.rpl_alter_extra_persistent : Added in 10.1.21 rpl.rpl_auto_increment : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_bug45679 : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_update_failure : MDEV-10625 - warnings in error log @@ -252,7 +280,6 @@ rpl.rpl_checksum_cache : MDEV-10626 - Testcase timeout rpl.rpl_circular_for_4_hosts : MDEV-10627 - Testcase timeout rpl.rpl_ddl : MDEV-10417 - Fails on Mips rpl.rpl_domain_id_filter_restart : MDEV-10684 - Wrong result -rpl.rpl_drop_db : Modified in 10.1.19 rpl.rpl_gtid_basic : MDEV-10681 - server startup problem rpl.rpl_gtid_crash : MDEV-9501 - Warning: failed registering on master rpl.rpl_gtid_errorlog : Uses search_pattern_in_file.inc modified in 10.1.20 @@ -260,6 +287,7 @@ rpl.rpl_gtid_master_promote : MDEV-10628 - Timeout in sync_with_master rpl.rpl_gtid_mdev9033 : MDEV-10680 - warnings rpl.rpl_gtid_stop_start : MDEV-10629 - Crash on shutdown rpl.rpl_gtid_until : MDEV-10625 - warnings in error log +rpl.rpl_heartbeat_basic : MDEV-11668 - wrong result rpl.rpl_innodb_bug30888 : MDEV-10417 - Fails on Mips rpl.rpl_insert : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_insert_delayed : MDEV-9329 - Fails on Ubuntu/s390x @@ -278,17 +306,21 @@ rpl.rpl_password_boundaries : MDEV-11534 - Slave IO warnings rpl.rpl_row_drop_create_temp_table : MDEV-10626 - Testcase timeout rpl.rpl_row_flsh_tbls : Uses binlog_start_pos.inc modified in 10.1.20 rpl.rpl_row_log_innodb : MDEV-10688 - Wrong result +rpl.rpl_row_mysqlbinlog : Modified in 10.1.21 rpl.rpl_row_sp001 : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_semi_sync : MDEV-11220 - Wrong result +rpl.rpl_semi_sync_event_after_sync : MDEV-11806 - warnings rpl.rpl_semi_sync_uninstall_plugin : MDEV-7140 - Wrong plugin status +rpl.rpl_semi_sync_wait_point : MDEV-11807 - timeout in wait condition rpl.rpl_show_slave_hosts : MDEV-10681 - server startup problem rpl.rpl_skip_replication : MDEV-9268 - Fails with timeout in sync_slave_with_master on Alpha rpl.rpl_slave_grp_exec : MDEV-10514 - Unexpected deadlock +rpl.rpl_special_charset : Modified in 10.1.21 rpl.rpl_stm_flsh_tbls : Uses binlog_start_pos.inc modified in 10.1.20 rpl.rpl_stop_slave_error : Uses search_pattern_in_file.inc modified in 10.1.20 rpl.rpl_sync : MDEV-10633 - Database page corruption rpl.rpl_temporary_error2 : MDEV-10634 - Wrong number of retries -rpl.sec_behind_master-5114 : MDEV-8518 - Wrong value of Seconds_Behind_Master +rpl.sec_behind_master-5114 : Modified in 10.1.21 #---------------------------------------------------------------- @@ -301,9 +333,11 @@ spider/bg.ha_part : MDEV-9329 - Fails on Ubuntu/s390x spider/bg.spider_fixes : MDEV-7098, MDEV-9329 - Mutex problem, failures on s390x spider/bg.vp_fixes : MDEV-9329 - Fails on Ubuntu/s390x +spider/handler.* : MDEV-10987, MDEV-10990 - Tests have not been maintained + #---------------------------------------------------------------- -sphinx.* : MDEV-10985, MDEV-10986 - Tests have not been maintained +sphinx.* : MDEV-10986 - Tests have not been maintained #---------------------------------------------------------------- @@ -325,6 +359,7 @@ sys_vars.replicate_ignore_db_basic : Modified in 10.1.20 sys_vars.replicate_ignore_table_basic : Modified in 10.1.20 sys_vars.replicate_wild_do_table_basic : Modified in 10.1.20 sys_vars.replicate_wild_ignore_table_basic : Modified in 10.1.20 +sys_vars.rpl_init_slave_func : MDEV-10149 - wrong results sys_vars.sysvars_innodb : MDEV-6958 - error-prone rdiffs sys_vars.sysvars_server_embedded : MDEV-6958 - error-prone rdiffs sys_vars.table_open_cache_instances_basic : Modified in 10.1.20 @@ -339,6 +374,7 @@ tokudb.table_index_statistics : Added in 10.1.20 tokudb_bugs.checkpoint_lock : MDEV-10637 - Wrong processlist output tokudb_bugs.checkpoint_lock_3 : MDEV-10637 - Wrong processlist output +tokudb_bugs.xa : MDEV-11804 - Lock wait timeout tokudb_rpl.rpl_parallel_optimistic : Added in 10.1.20 tokudb_rpl.rpl_tokudb_rfr_partition_table : Added in 10.1.20 @@ -355,9 +391,7 @@ vcol.vcol_keys_innodb : MDEV-10639 - Testcase timeout #---------------------------------------------------------------- wsrep.binlog_format : MDEV-11532 - WSREP has not yet prepared node -wsrep.mdev_10186 : Modified in 10.1.19 #---------------------------------------------------------------- -wsrep_info.* : suite.pm modified in 10.1.19 wsrep_info.plugin : MDEV-11530 - Warnings; also modified in 10.1.20 diff --git a/mysys/my_malloc.c b/mysys/my_malloc.c index e533230106e..dc02d3896bd 100644 --- a/mysys/my_malloc.c +++ b/mysys/my_malloc.c @@ -48,7 +48,6 @@ static inline size_t malloc_size_and_flag(void *p, my_bool *is_thread_specific) #define MALLOC_FIX_POINTER_FOR_FREE(p) (((char*) (p)) - MALLOC_PREFIX_SIZE) #endif /* SAFEMALLOC */ -static MALLOC_SIZE_CB malloc_size_cb_func= NULL; /** Inform application that memory usage has changed @@ -59,17 +58,19 @@ static MALLOC_SIZE_CB malloc_size_cb_func= NULL; The type os size is long long, to be able to handle negative numbers to decrement the memory usage + + @return 0 - ok + 1 - failure, abort the allocation */ +static void dummy(long long size __attribute__((unused)), + my_bool is_thread_specific __attribute__((unused))) +{} -static void update_malloc_size(long long size, my_bool is_thread_specific) -{ - if (malloc_size_cb_func) - malloc_size_cb_func(size, is_thread_specific); -} +static MALLOC_SIZE_CB update_malloc_size= dummy; void set_malloc_size_cb(MALLOC_SIZE_CB func) { - malloc_size_cb_func= func; + update_malloc_size= func ? func : dummy; } diff --git a/sql/item.cc b/sql/item.cc index 682f88317f1..8698b0f7fa4 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -544,7 +544,14 @@ uint Item::decimal_precision() const unsigned_flag); return MY_MIN(prec, DECIMAL_MAX_PRECISION); } - return MY_MIN(max_char_length(), DECIMAL_MAX_PRECISION); + uint res= max_char_length(); + /* + Return at least one decimal digit, even if Item::max_char_length() + returned 0. This is important to avoid attempts to create fields of types + INT(0) or DECIMAL(0,0) when converting NULL or empty strings to INT/DECIMAL: + CREATE TABLE t1 AS SELECT CONVERT(NULL,SIGNED) AS a; + */ + return res ? MY_MIN(res, DECIMAL_MAX_PRECISION) : 1; } diff --git a/sql/item_func.h b/sql/item_func.h index d4314f7bd6a..63b97a1e0c5 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -636,8 +636,15 @@ public: } void fix_length_and_dec() { - fix_char_length(MY_MIN(args[0]->max_char_length(), - MY_INT64_NUM_DECIMAL_DIGITS)); + uint32 char_length= MY_MIN(args[0]->max_char_length(), + MY_INT64_NUM_DECIMAL_DIGITS); + /* + args[0]->max_char_length() can return 0. + Reserve max_length to fit at least one character for one digit, + plus one character for the sign (if signed). + */ + set_if_bigger(char_length, 1 + (unsigned_flag ? 0 : 1)); + fix_char_length(char_length); } virtual void print(String *str, enum_query_type query_type); uint decimal_precision() const { return args[0]->decimal_precision(); } diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 4b79ea43c7c..8930bf930d1 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2015, Oracle and/or its affiliates. - Copyright (c) 2008, 2016, MariaDB + Copyright (c) 2008, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -4087,8 +4087,8 @@ extern "C" { static void my_malloc_size_cb_func(long long size, my_bool is_thread_specific) { THD *thd= current_thd; - - if (likely(is_thread_specific)) /* If thread specific memory */ + + if (is_thread_specific) /* If thread specific memory */ { /* When thread specfic is set, both mysqld_server_initialized and thd @@ -4100,14 +4100,22 @@ static void my_malloc_size_cb_func(long long size, my_bool is_thread_specific) (longlong) thd->status_var.local_memory_used, size)); thd->status_var.local_memory_used+= size; + if (thd->status_var.local_memory_used > (int64)thd->variables.max_mem_used && + !thd->killed) + { + char buf[1024]; + thd->killed= KILL_QUERY; + my_snprintf(buf, sizeof(buf), "--max-thread-mem-used=%llu", + thd->variables.max_mem_used); + my_error(ER_OPTION_PREVENTS_STATEMENT, MYF(0), buf); + } DBUG_ASSERT((longlong) thd->status_var.local_memory_used >= 0 || !debug_assert_on_not_freed_memory); } else if (likely(thd)) { DBUG_PRINT("info", ("global thd memory_used: %lld size: %lld", - (longlong) thd->status_var.global_memory_used, - size)); + (longlong) thd->status_var.global_memory_used, size)); thd->status_var.global_memory_used+= size; } else diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 5c7cfdce657..f78382ded7d 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2000, 2015, Oracle and/or its affiliates. - Copyright (c) 2008, 2016, MariaDB + Copyright (c) 2008, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -742,8 +742,8 @@ THD::THD(my_thread_id id, bool is_wsrep_applier) #endif { ulong tmp; + bzero(&variables, sizeof(variables)); - mdl_context.init(this); /* We set THR_THD to temporally point to this THD to register all the variables that allocates memory for this THD @@ -753,8 +753,11 @@ THD::THD(my_thread_id id, bool is_wsrep_applier) status_var.local_memory_used= sizeof(THD); status_var.global_memory_used= 0; variables.pseudo_thread_id= thread_id; + variables.max_mem_used= global_system_variables.max_mem_used; main_da.init(); + mdl_context.init(this); + /* Pass nominal parameters to init_alloc_root only to ensure that the destructor works OK in case of an error. The main_mem_root @@ -800,7 +803,6 @@ THD::THD(my_thread_id id, bool is_wsrep_applier) connection_name.str= 0; connection_name.length= 0; - bzero(&variables, sizeof(variables)); file_id = 0; query_id= 0; query_name_consts= 0; diff --git a/sql/sql_class.h b/sql/sql_class.h index 3a7454e0a89..b6dafd939bc 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -546,6 +546,7 @@ typedef struct system_variables ulonglong sortbuff_size; ulonglong group_concat_max_len; ulonglong default_regex_flags; + ulonglong max_mem_used; /** Place holders to store Multi-source variables in sys_var.cc during diff --git a/sql/sql_trigger.cc b/sql/sql_trigger.cc index bd02289b135..00540399abd 100644 --- a/sql/sql_trigger.cc +++ b/sql/sql_trigger.cc @@ -1206,7 +1206,7 @@ bool Table_triggers_list::prepare_record_accessors(TABLE *table) (table->s->stored_fields != table->s->null_fields)) { - int null_bytes= (table->s->stored_fields - table->s->null_fields + 7)/8; + int null_bytes= (table->s->fields - table->s->null_fields + 7)/8; if (!(extra_null_bitmap= (uchar*)alloc_root(&table->mem_root, null_bytes))) return 1; if (!(record0_field= (Field **)alloc_root(&table->mem_root, diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index bdfedc30d46..56913092354 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2002, 2015, Oracle and/or its affiliates. - Copyright (c) 2012, 2016, MariaDB Corporation + Copyright (c) 2012, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -5468,6 +5468,13 @@ static Sys_var_ulong Sys_log_tc_size( BLOCK_SIZE(my_getpagesize())); #endif +static Sys_var_ulonglong Sys_max_thread_mem( + "max_session_mem_used", "Amount of memory a single user session " + "is allowed to allocate. This limits the value of the " + "session variable MEM_USED", SESSION_VAR(max_mem_used), + CMD_LINE(REQUIRED_ARG), VALID_RANGE(8192, ULONGLONG_MAX), + DEFAULT(LONGLONG_MAX), BLOCK_SIZE(1)); + #ifndef EMBEDDED_LIBRARY static Sys_var_sesvartrack Sys_track_session_sys_vars( diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index f555875559d..4deee54d97f 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -509,8 +509,9 @@ buf_dblwr_init_or_load_pages( return(err); } - } else { - + } else if (memcmp(field_ref_zero, page + FIL_PAGE_LSN, 8)) { + /* Each valid page header must contain + a nonzero FIL_PAGE_LSN field. */ recv_dblwr.add(page); } @@ -544,11 +545,9 @@ buf_dblwr_process(void) for (recv_dblwr_t::list::iterator i = recv_dblwr.pages.begin(); i != recv_dblwr.pages.end(); ++i, ++page_no_dblwr) { - bool is_compressed = false; - - const byte* page = *i; - ulint page_no = page_get_page_no(page); - ulint space_id = page_get_space_id(page); + byte* page = *i; + ulint page_no = page_get_page_no(page); + ulint space_id = page_get_space_id(page); fil_space_t* space = fil_space_get(space_id); @@ -560,158 +559,130 @@ buf_dblwr_process(void) fil_space_open_if_needed(space); + const page_id_t page_id(space_id, page_no); + if (page_no >= space->size) { - /* Do not report the warning if the tablespace is - schedule for truncate or was truncated and we have live - MLOG_TRUNCATE record in redo. */ - bool skip_warning = - srv_is_tablespace_truncated(space_id) - || srv_was_tablespace_truncated(space); - - if (!skip_warning) { - ib::warn() << "Page " << page_no_dblwr - << " in the doublewrite buffer is" - " not within space bounds: page " - << page_id_t(space_id, page_no); + /* Do not report the warning if the tablespace + is scheduled for truncation or was truncated + and we have parsed an MLOG_TRUNCATE record. */ + if (!srv_is_tablespace_truncated(space_id) + && !srv_was_tablespace_truncated(space)) { + ib::warn() << "A copy of page " << page_id + << " in the doublewrite buffer slot " + << page_no_dblwr + << " is not within space bounds"; } - } else { - const page_size_t page_size(space->flags); - const page_id_t page_id(space_id, page_no); + continue; + } - /* We want to ensure that for partial reads the - unread portion of the page is NUL. */ - memset(read_buf, 0x0, page_size.physical()); + const page_size_t page_size(space->flags); + ut_ad(!buf_page_is_zeroes(page, page_size)); - IORequest request; + /* We want to ensure that for partial reads the + unread portion of the page is NUL. */ + memset(read_buf, 0x0, page_size.physical()); - request.dblwr_recover(); + IORequest request; - /* Read in the actual page from the file */ - dberr_t err = fil_io( - request, true, - page_id, page_size, - 0, page_size.physical(), read_buf, NULL, NULL); + request.dblwr_recover(); - if (err != DB_SUCCESS) { + /* Read in the actual page from the file */ + dberr_t err = fil_io( + request, true, + page_id, page_size, + 0, page_size.physical(), read_buf, NULL, NULL); - ib::warn() - << "Double write buffer recovery: " - << page_id << " read failed with " - << "error: " << ut_strerr(err); - } + if (err != DB_SUCCESS) { + ib::warn() + << "Double write buffer recovery: " + << page_id << " read failed with " + << "error: " << ut_strerr(err); + } - /* Is page compressed ? */ - is_compressed = fil_page_is_compressed_encrypted(read_buf) | - fil_page_is_compressed(read_buf); + const bool is_all_zero = buf_page_is_zeroes( + read_buf, page_size); - /* If page was compressed, decompress it before we - check checksum. */ - if (is_compressed) { - fil_decompress_page(NULL, read_buf, UNIV_PAGE_SIZE, NULL, true); + if (is_all_zero) { + /* We will check if the copy in the + doublewrite buffer is valid. If not, we will + ignore this page (there should be redo log + records to initialize it). */ + } else { + if (fil_page_is_compressed_encrypted(read_buf) || + fil_page_is_compressed(read_buf)) { + /* Decompress the page before + validating the checksum. */ + fil_decompress_page( + NULL, read_buf, UNIV_PAGE_SIZE, + NULL, true); } - if (err != DB_SUCCESS) { - ib::warn() - << "Double write buffer recovery: " - << page_id << " read failed with " - << "error: " << ut_strerr(err); + if (fil_space_verify_crypt_checksum( + read_buf, page_size) + || !buf_page_is_corrupted( + true, read_buf, page_size, false)) { + /* The page is good; there is no need + to consult the doublewrite buffer. */ + continue; } - if (fil_space_verify_crypt_checksum(read_buf, page_size)) { - - /* page is encrypted and checksum is OK */ - } else if (buf_page_is_corrupted( - true, read_buf, page_size, - fsp_is_checksum_disabled(space_id))) { - - ib::warn() << "Database page corruption or" - << " a failed file read of page " - << page_id - << ". Trying to recover it from the" - << " doublewrite buffer."; - - /* Is page compressed ? */ - is_compressed = fil_page_is_compressed_encrypted(page) | - fil_page_is_compressed(page); - - /* If page was compressed, decompress it before we - check checksum. */ - if (is_compressed) { - fil_decompress_page(NULL, (byte*)page, UNIV_PAGE_SIZE, NULL, true); - } - - if (fil_space_verify_crypt_checksum(page, page_size)) { - /* the doublewrite buffer page is encrypted and OK */ - } else if (buf_page_is_corrupted( - true, page, page_size, - fsp_is_checksum_disabled(space_id))) { - - ib::error() << "Dump of the page:"; - - buf_page_print( - read_buf, page_size, - BUF_PAGE_PRINT_NO_CRASH); - ib::error() << "Dump of corresponding" - " page in doublewrite buffer:"; - - buf_page_print( - page, page_size, - BUF_PAGE_PRINT_NO_CRASH); - - ib::fatal() << "The page in the" - " doublewrite buffer is" - " corrupt. Cannot continue" - " operation. You can try to" - " recover the database with" - " innodb_force_recovery=6"; - } - } else if (buf_page_is_zeroes(read_buf, page_size) - && !buf_page_is_zeroes(page, page_size) - && !buf_page_is_corrupted( - true, page, page_size, - fsp_is_checksum_disabled(space_id))) { - - /* Database page contained only zeroes, while - a valid copy is available in dblwr buffer. */ - - } else { - - bool t1 = buf_page_is_zeroes( - read_buf, page_size); - - bool t2 = buf_page_is_zeroes(page, page_size); - - bool t3 = buf_page_is_corrupted( - true, page, page_size, - fsp_is_checksum_disabled(space_id)); + /* We intentionally skip this message for + is_all_zero pages. */ + ib::info() + << "Trying to recover page " << page_id + << " from the doublewrite buffer."; + } - if (t1 && !(t2 || t3)) { + /* Next, validate the doublewrite page. */ + if (fil_page_is_compressed_encrypted(page) || + fil_page_is_compressed(page)) { + /* Decompress the page before + validating the checksum. */ + fil_decompress_page( + NULL, page, UNIV_PAGE_SIZE, NULL, true); + } - /* Database page contained only - zeroes, while a valid copy is - available in dblwr buffer. */ + if (!fil_space_verify_crypt_checksum(page, page_size) + && buf_page_is_corrupted(true, page, page_size, false)) { + if (!is_all_zero) { + ib::warn() << "A doublewrite copy of page " + << page_id << " is corrupted."; + } + /* Theoretically we could have another good + copy for this page in the doublewrite + buffer. If not, we will report a fatal error + for a corrupted page somewhere else if that + page was truly needed. */ + continue; + } - } else { - continue; - } + if (page_no == 0) { + /* Check the FSP_SPACE_FLAGS. */ + ulint flags = fsp_header_get_flags(page); + if (!fsp_flags_is_valid(flags) + && fsp_flags_convert_from_101(flags) + == ULINT_UNDEFINED) { + ib::warn() << "Ignoring a doublewrite copy" + " of page " << page_id + << " due to invalid flags " + << ib::hex(flags); + continue; } + /* The flags on the page should be converted later. */ + } - IORequest write_request(IORequest::WRITE); + /* Write the good page from the doublewrite buffer to + the intended position. */ - /* Write the good page from the doublewrite - buffer to the intended position. */ + IORequest write_request(IORequest::WRITE); - fil_io(write_request, true, - page_id, page_size, - 0, page_size.physical(), - const_cast<byte*>(page), NULL, NULL); + fil_io(write_request, true, page_id, page_size, + 0, page_size.physical(), + const_cast<byte*>(page), NULL, NULL); - ib::info() - << "Recovered page " - << page_id - << " from the doublewrite buffer."; - } + ib::info() << "Recovered page " << page_id + << " from the doublewrite buffer."; } recv_dblwr.pages.clear(); diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 3cf236e643b..73c7bf027ca 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -7236,56 +7236,6 @@ dict_index_zip_pad_optimal_page_size( return(ut_max(sz, min_sz)); } -/** Convert a 32 bit integer table flags to the 32 bit FSP Flags. -Fsp Flags are written into the tablespace header at the offset -FSP_SPACE_FLAGS and are also stored in the fil_space_t::flags field. -The following chart shows the translation of the low order bit. -Other bits are the same. - Low order bit - | REDUNDANT | COMPACT | COMPRESSED | DYNAMIC -dict_table_t::flags | 0 | 1 | 1 | 1 -fil_space_t::flags | 0 | 0 | 1 | 1 -@param[in] table_flags dict_table_t::flags -@return tablespace flags (fil_space_t::flags) */ -ulint -dict_tf_to_fsp_flags(ulint table_flags) -{ - DBUG_EXECUTE_IF("dict_tf_to_fsp_flags_failure", - return(ULINT_UNDEFINED);); - - bool has_atomic_blobs = - DICT_TF_HAS_ATOMIC_BLOBS(table_flags); - page_size_t page_size = dict_tf_get_page_size(table_flags); - bool has_data_dir = DICT_TF_HAS_DATA_DIR(table_flags); - bool page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); - ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); - - ut_ad(!page_size.is_compressed() || has_atomic_blobs); - - ulint fsp_flags = fsp_flags_init(page_size, - has_atomic_blobs, - has_data_dir, - 0, - 0, - 0); - - /* In addition, tablespace flags also contain if the page - compression is used for this table. */ - if (page_compression) { - fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION(fsp_flags, page_compression); - } - - /* In addition, tablespace flags also contain page compression level - if page compression is used for this table. */ - if (page_compression && page_compression_level) { - fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(fsp_flags, page_compression_level); - } - - ut_ad(fsp_flags_is_valid(fsp_flags)); - - return(fsp_flags); -} - /*************************************************************//** Convert table flag to row format string. @return row format name. */ diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index 756e1010d66..db14983cd8f 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -1215,7 +1215,7 @@ dict_check_sys_tables( look to see if it is already in the tablespace cache. */ if (fil_space_for_table_exists_in_mem( space_id, table_name.m_name, - false, true, NULL, 0, NULL)) { + false, true, NULL, 0, NULL, flags)) { /* Recovery can open a datafile that does not match SYS_DATAFILES. If they don't match, update SYS_DATAFILES. */ @@ -1239,15 +1239,13 @@ dict_check_sys_tables( char* filepath = dict_get_first_path(space_id); /* Check that the .ibd file exists. */ - ulint fsp_flags = dict_tf_to_fsp_flags(flags); validate = true; /* Encryption */ dberr_t err = fil_ibd_open( validate, !srv_read_only_mode && srv_log_file_size != 0, FIL_TYPE_TABLESPACE, - space_id, - fsp_flags, + space_id, dict_tf_to_fsp_flags(flags), table_name.m_name, filepath, NULL); @@ -2628,7 +2626,7 @@ dict_load_tablespace( /* The tablespace may already be open. */ if (fil_space_for_table_exists_in_mem( table->space, space_name, false, - true, heap, table->id, table)) { + true, heap, table->id, table, table->flags)) { return; } @@ -2657,10 +2655,10 @@ dict_load_tablespace( /* Try to open the tablespace. We set the 2nd param (fix_dict) to false because we do not have an x-lock on dict_operation_lock */ - ulint fsp_flags = dict_tf_to_fsp_flags(table->flags); dberr_t err = fil_ibd_open( true, false, FIL_TYPE_TABLESPACE, table->space, - fsp_flags, space_name, filepath, table); + dict_tf_to_fsp_flags(table->flags), + space_name, filepath, table); if (err != DB_SUCCESS) { /* We failed to find a sensible tablespace file */ diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 3070badd6db..a0442808eaa 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -585,11 +585,6 @@ fil_node_open_file( { os_offset_t size_bytes; bool success; - byte* buf2; - byte* page; - ulint flags; - ulint min_size; - ulint space_id; bool read_only_mode; fil_space_t* space = node->space; @@ -635,95 +630,80 @@ retry: ut_a(size_bytes != (os_offset_t) -1); ut_a(space->purpose != FIL_TYPE_LOG); + const page_size_t page_size(space->flags); + const ulint psize = page_size.physical(); + const ulint min_size = FIL_IBD_FILE_INITIAL_SIZE + * psize; + + if (size_bytes < min_size) { + ib::error() << "The size of the file " << node->name + << " is only " << size_bytes + << " bytes, should be at least " << min_size; + os_file_close(node->handle); + return(false); + } /* Read the first page of the tablespace */ - buf2 = static_cast<byte*>(ut_malloc_nokey(2 * UNIV_PAGE_SIZE)); + byte* buf2 = static_cast<byte*>(ut_malloc_nokey(2 * psize)); /* Align the memory for file i/o if we might have O_DIRECT set */ - page = static_cast<byte*>(ut_align(buf2, UNIV_PAGE_SIZE)); - ut_ad(page == page_align(page)); + byte* page = static_cast<byte*>(ut_align(buf2, psize)); IORequest request(IORequest::READ); success = os_file_read( request, - node->handle, page, 0, UNIV_PAGE_SIZE); + node->handle, page, 0, psize); srv_stats.page0_read.add(1); - space_id = fsp_header_get_space_id(page); - flags = fsp_header_get_flags(page); - - /* Close the file now that we have read the space id from it */ - + const ulint space_id + = fsp_header_get_space_id(page); + ulint flags = fsp_header_get_flags(page); + const ulint size = fsp_header_get_field( + page, FSP_SIZE); + const ulint free_limit = fsp_header_get_field( + page, FSP_FREE_LIMIT); + const ulint free_len = flst_get_len( + FSP_HEADER_OFFSET + FSP_FREE + page); + ut_free(buf2); os_file_close(node->handle); - const page_size_t page_size(space->flags); - - min_size = FIL_IBD_FILE_INITIAL_SIZE * page_size.physical(); - - if (size_bytes < min_size) { - - ib::error() << "The size of tablespace " << space_id << " file " - << node->name << " is only " << size_bytes - << ", should be at least " << min_size << "!"; - - ut_error; - } - - if (space->flags != flags) { - ulint sflags = (space->flags & ~FSP_FLAGS_MASK_DATA_DIR); - ulint fflags = (flags & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - - /* DATA_DIR option is on different place on MariaDB - compared to MySQL. If this is the difference. Fix - it. */ - - if (sflags == fflags) { - ib::warn() - << "Tablespace " << space_id - << " flags " << space->flags - << " in the data dictionary but in file " << node->name - << " are " << flags - << ". Temporally corrected because DATA_DIR option to " - << space->flags; - - flags = space->flags; - } else { - ib::fatal() - << "Table flags are " - << ib::hex(space->flags) << " in the data" - " dictionary but the flags in file " - << node->name << " are " << ib::hex(flags) - << "!"; + if (!fsp_flags_is_valid(flags)) { + ulint cflags = fsp_flags_convert_from_101(flags); + if (cflags == ULINT_UNDEFINED) { + ib::error() + << "Expected tablespace flags " + << ib::hex(flags) + << " but found " << ib::hex(flags) + << " in the file " << node->name; + return(false); } - } + flags = cflags; + } - { - ulint size = fsp_header_get_field( - page, FSP_SIZE); - ulint free_limit = fsp_header_get_field( - page, FSP_FREE_LIMIT); - ulint free_len = flst_get_len( - FSP_HEADER_OFFSET + FSP_FREE + page); - - ut_ad(space->free_limit == 0 - || space->free_limit == free_limit); - ut_ad(space->free_len == 0 - || space->free_len == free_len); - space->size_in_header = size; - space->free_limit = free_limit; - space->free_len = free_len; + if (UNIV_UNLIKELY(space_id != space->id)) { + ib::error() + << "Expected tablespace id " << space->id + << " but found " << space_id + << "in the file" << node->name; + return(false); } - ut_free(buf2); + ut_ad(space->free_limit == 0 + || space->free_limit == free_limit); + ut_ad(space->free_len == 0 + || space->free_len == free_len); + space->size_in_header = size; + space->free_limit = free_limit; + space->free_len = free_len; if (node->size == 0) { ulint extent_size; - extent_size = page_size.physical() * FSP_EXTENT_SIZE; + extent_size = psize * FSP_EXTENT_SIZE; /* After apply-incremental, tablespaces are not extended to a whole megabyte. Do not cut off valid data. */ @@ -734,9 +714,7 @@ retry: extent_size); } - node->size = (ulint) - (size_bytes / page_size.physical()); - + node->size = size_bytes / psize; space->size += node->size; } } @@ -1575,8 +1553,9 @@ fil_space_create( fil_space_t* space; ut_ad(fil_system); - ut_ad(fsp_flags_is_valid(flags)); - ut_ad(srv_page_size == UNIV_PAGE_SIZE_ORIG || flags != 0); + ut_ad(fsp_flags_is_valid(flags & ~FSP_FLAGS_MEM_MASK)); + ut_ad(purpose == FIL_TYPE_LOG + || srv_page_size == UNIV_PAGE_SIZE_ORIG || flags != 0); DBUG_EXECUTE_IF("fil_space_create_failure", return(NULL);); @@ -2344,6 +2323,7 @@ fil_op_write_log( ulint len; ut_ad(first_page_no == 0); + ut_ad(fsp_flags_is_valid(flags)); /* fil_name_parse() requires that there be at least one path separator and that the file path end with ".ibd". */ @@ -2564,7 +2544,7 @@ fil_recreate_tablespace( /* Align the memory for file i/o */ page = static_cast<byte*>(ut_align(buf, UNIV_PAGE_SIZE)); - flags = fsp_flags_set_page_size(flags, univ_page_size); + flags |= FSP_FLAGS_PAGE_SSIZE(); fsp_header_init_fields(page, space_id, flags); @@ -2640,13 +2620,11 @@ fil_recreate_tablespace( byte* page = buf_block_get_frame(block); - if (!fsp_flags_is_compressed(flags)) { - + if (!FSP_FLAGS_GET_ZIP_SSIZE(flags)) { ut_ad(!page_size.is_compressed()); buf_flush_init_for_writing( - block, page, NULL, recv_lsn, - fsp_is_checksum_disabled(space_id)); + block, page, NULL, recv_lsn, false); err = fil_write(cur_page_id, page_size, 0, page_size.physical(), page); @@ -3699,7 +3677,7 @@ fil_ibd_create( byte* buf2; byte* page; bool success; - bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags); + bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags) != 0; fil_space_t* space = NULL; fil_space_crypt_t *crypt_data = NULL; @@ -3707,7 +3685,7 @@ fil_ibd_create( ut_ad(!srv_read_only_mode); ut_a(space_id < SRV_LOG_SPACE_FIRST_ID); ut_a(size >= FIL_IBD_FILE_INITIAL_SIZE); - ut_a(fsp_flags_is_valid(flags)); + ut_a(fsp_flags_is_valid(flags & ~FSP_FLAGS_MEM_MASK)); /* Create the subdirectories in the path, if they are not there already. */ @@ -3809,10 +3787,7 @@ fil_ibd_create( memset(page, '\0', UNIV_PAGE_SIZE); - /* Add the UNIV_PAGE_SIZE to the table flags and write them to the - tablespace header. */ - flags = fsp_flags_set_page_size(flags, univ_page_size); - + flags |= FSP_FLAGS_PAGE_SSIZE(); fsp_header_init_fields(page, space_id, flags); mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id); @@ -3889,7 +3864,7 @@ fil_ibd_create( } space = fil_space_create(name, space_id, flags, FIL_TYPE_TABLESPACE, - crypt_data, true); + crypt_data, true); if (!fil_node_create_low(path, size, space, false, true)) { if (crypt_data) { @@ -3901,20 +3876,23 @@ fil_ibd_create( mtr_t mtr; const fil_node_t* file = UT_LIST_GET_FIRST(space->chain); - mtr_start(&mtr); + mtr.start(); fil_op_write_log( MLOG_FILE_CREATE2, space_id, 0, file->name, - NULL, space->flags, &mtr); + NULL, space->flags & ~FSP_FLAGS_MEM_MASK, &mtr); fil_name_write(space, 0, file, &mtr); - mtr_commit(&mtr); + mtr.commit(); + err = DB_SUCCESS; } os_file_close(file); + if (err != DB_SUCCESS) { if (has_data_dir) { RemoteDatafile::delete_link_file(name); } + os_file_delete(innodb_data_file_key, path); } @@ -3945,7 +3923,7 @@ statement to update the dictionary tables if they are incorrect. @param[in] fix_dict true if the dictionary is available to be fixed @param[in] purpose FIL_TYPE_TABLESPACE or FIL_TYPE_TEMPORARY @param[in] id tablespace ID -@param[in] flags tablespace flags +@param[in] flags expected FSP_SPACE_FLAGS @param[in] space_name tablespace name of the datafile If file-per-table, it is the table name in the databasename/tablename format @param[in] path_in expected filepath, usually read from dictionary @@ -3970,7 +3948,6 @@ fil_ibd_open( RemoteDatafile df_remote; /* remote location */ ulint tablespaces_found = 0; ulint valid_tablespaces_found = 0; - bool for_import = (purpose == FIL_TYPE_IMPORT); ut_ad(!fix_dict || rw_lock_own(dict_operation_lock, RW_LOCK_X)); @@ -3979,10 +3956,13 @@ fil_ibd_open( ut_ad(!fix_dict || srv_log_file_size != 0); ut_ad(fil_type_is_data(purpose)); - if (!fsp_flags_is_valid(flags)) { + /* Table flags can be ULINT_UNDEFINED if + dict_tf_to_fsp_flags_failure is set. */ + if (flags == ULINT_UNDEFINED) { return(DB_CORRUPTION); } + ut_ad(fsp_flags_is_valid(flags & ~FSP_FLAGS_MEM_MASK)); df_default.init(space_name, flags); df_dict.init(space_name, flags); df_remote.init(space_name, flags); @@ -4075,16 +4055,13 @@ fil_ibd_open( /* Read and validate the first page of these three tablespace locations, if found. */ valid_tablespaces_found += - (df_remote.validate_to_dd(id, flags, for_import) - == DB_SUCCESS) ? 1 : 0; + (df_remote.validate_to_dd(id, flags) == DB_SUCCESS); valid_tablespaces_found += - (df_default.validate_to_dd(id, flags, for_import) - == DB_SUCCESS) ? 1 : 0; + (df_default.validate_to_dd(id, flags) == DB_SUCCESS); valid_tablespaces_found += - (df_dict.validate_to_dd(id, flags, for_import) - == DB_SUCCESS) ? 1 : 0; + (df_dict.validate_to_dd(id, flags) == DB_SUCCESS); /* Make sense of these three possible locations. First, bail out if no tablespace files were found. */ @@ -4095,7 +4072,7 @@ fil_ibd_open( return(DB_CORRUPTION); } if (!validate) { - return(DB_SUCCESS); + goto skip_validate; } /* Do not open any tablespaces if more than one tablespace with @@ -4256,6 +4233,10 @@ skip_validate: true) == NULL) { err = DB_ERROR; } + + if (purpose != FIL_TYPE_IMPORT && !srv_read_only_mode) { + fsp_flags_try_adjust(id, flags & ~FSP_FLAGS_MEM_MASK); + } } return(err); @@ -4547,8 +4528,16 @@ fil_ibd_load( ut_ad(space == NULL); + /* Adjust the memory-based flags that would normally be set by + dict_tf_to_fsp_flags(). In recovery, we have no data dictionary. */ + ulint flags = file.flags(); + if (FSP_FLAGS_HAS_PAGE_COMPRESSION(flags)) { + flags |= page_zip_level + << FSP_FLAGS_MEM_COMPRESSION_LEVEL; + } + space = fil_space_create( - file.name(), space_id, file.flags(), FIL_TYPE_TABLESPACE, + file.name(), space_id, flags, FIL_TYPE_TABLESPACE, file.get_crypt_info(), false); if (space == NULL) { @@ -4620,7 +4609,41 @@ fil_report_missing_tablespace( " you deleted or moved .ibd files?"; } -/** Returns true if a matching tablespace exists in the InnoDB tablespace +/** Try to adjust FSP_SPACE_FLAGS if they differ from the expectations. +(Typically when upgrading from MariaDB 10.1.0..10.1.20.) +@param[in] space_id tablespace ID +@param[in] flags desired tablespace flags */ +UNIV_INTERN +void +fsp_flags_try_adjust(ulint space_id, ulint flags) +{ + ut_ad(!srv_read_only_mode); + ut_ad(fsp_flags_is_valid(flags)); + + mtr_t mtr; + mtr_start(&mtr); + if (buf_block_t* b = buf_page_get( + page_id_t(space_id, 0), page_size_t(flags), + RW_X_LATCH, &mtr)) { + ulint f = fsp_header_get_flags(b->frame); + /* Suppress the message if only the DATA_DIR flag to differs. */ + if ((f ^ flags) & ~(1U << FSP_FLAGS_POS_RESERVED)) { + ib::warn() + << "adjusting FSP_SPACE_FLAGS of tablespace " + << space_id + << " from " << ib::hex(f) + << " to " << ib::hex(flags); + } + if (f != flags) { + mlog_write_ulint(FSP_HEADER_OFFSET + + FSP_SPACE_FLAGS + b->frame, + flags, MLOG_4BYTES, &mtr); + } + } + mtr_commit(&mtr); +} + +/** Determine if a matching tablespace exists in the InnoDB tablespace memory cache. Note that if we have not done a crash recovery at the database startup, there may be many tablespaces which are not yet in the memory cache. @param[in] id Tablespace ID @@ -4632,6 +4655,7 @@ error log if a matching tablespace is not found from memory. @param[in] heap Heap memory @param[in] table_id table id @param[in] table table +@param[in] table_flags table flags @return true if a matching tablespace exists in the memory cache */ bool fil_space_for_table_exists_in_mem( @@ -4641,12 +4665,13 @@ fil_space_for_table_exists_in_mem( bool adjust_space, mem_heap_t* heap, table_id_t table_id, - dict_table_t* table) + dict_table_t* table, + ulint table_flags) { - fil_space_t* fnamespace = NULL; + fil_space_t* fnamespace; fil_space_t* space; - ut_ad(fil_system); + const ulint expected_flags = dict_tf_to_fsp_flags(table_flags); mutex_enter(&fil_system->mutex); @@ -4654,50 +4679,41 @@ fil_space_for_table_exists_in_mem( space = fil_space_get_by_id(id); - /* If tablespace contains encryption information - copy it also to table. */ - if (space && space->crypt_data && - table && !table->crypt_data) { - table->crypt_data = space->crypt_data; - } + /* Look if there is a space with the same name; the name is the + directory path from the datadir to the file */ - if (space != NULL) { - /* If this space has the expected name, use it. */ - fnamespace = fil_space_get_by_name(name); - if (space == fnamespace) { - /* Found */ + fnamespace = fil_space_get_by_name(name); + bool valid = space && !((space->flags ^ expected_flags) + & ~FSP_FLAGS_MEM_MASK); - mutex_exit(&fil_system->mutex); - return(true); - } + if (valid && table && !table->crypt_data) { + table->crypt_data = space->crypt_data; } - /* Info from "fnamespace" comes from the ibd file itself, it can - be different from data obtained from System tables since file - operations are not transactional. If adjust_space is set, and the - mismatching space are between a user table and its temp table, we - shall adjust the ibd file name according to system table info */ - if (adjust_space - && space != NULL - && row_is_mysql_tmp_table_name(space->name) - && !row_is_mysql_tmp_table_name(name)) { - + if (!space) { + } else if (!valid || space == fnamespace) { + /* Found with the same file name, or got a flag mismatch. */ + goto func_exit; + } else if (adjust_space + && row_is_mysql_tmp_table_name(space->name) + && !row_is_mysql_tmp_table_name(name)) { + /* Info from fnamespace comes from the ibd file + itself, it can be different from data obtained from + System tables since renaming files is not + transactional. We shall adjust the ibd file name + according to system table info. */ mutex_exit(&fil_system->mutex); DBUG_EXECUTE_IF("ib_crash_before_adjust_fil_space", DBUG_SUICIDE();); - if (fnamespace) { - const char* tmp_name; - - tmp_name = dict_mem_create_temporary_tablename( - heap, name, table_id); + const char* tmp_name = dict_mem_create_temporary_tablename( + heap, name, table_id); - fil_rename_tablespace( - fnamespace->id, - UT_LIST_GET_FIRST(fnamespace->chain)->name, - tmp_name, NULL); - } + fil_rename_tablespace( + fnamespace->id, + UT_LIST_GET_FIRST(fnamespace->chain)->name, + tmp_name, NULL); DBUG_EXECUTE_IF("ib_crash_after_adjust_one_fil_space", DBUG_SUICIDE();); @@ -4712,16 +4728,12 @@ fil_space_for_table_exists_in_mem( mutex_enter(&fil_system->mutex); fnamespace = fil_space_get_by_name(name); ut_ad(space == fnamespace); - mutex_exit(&fil_system->mutex); - - return(true); + goto func_exit; } if (!print_error_if_does_not_exist) { - - mutex_exit(&fil_system->mutex); - - return(false); + valid = false; + goto func_exit; } if (space == NULL) { @@ -4740,10 +4752,8 @@ fil_space_for_table_exists_in_mem( } error_exit: ib::warn() << TROUBLESHOOT_DATADICT_MSG; - - mutex_exit(&fil_system->mutex); - - return(false); + valid = false; + goto func_exit; } if (0 != strcmp(space->name, name)) { @@ -4762,9 +4772,19 @@ error_exit: goto error_exit; } +func_exit: + if (valid) { + /* Adjust the flags that are in FSP_FLAGS_MEM_MASK. + FSP_SPACE_FLAGS will not be written back here. */ + space->flags = expected_flags; + } mutex_exit(&fil_system->mutex); - return(false); + if (valid && !srv_read_only_mode) { + fsp_flags_try_adjust(id, expected_flags & ~FSP_FLAGS_MEM_MASK); + } + + return(valid); } /** Return the space ID based on the tablespace name. @@ -6425,15 +6445,12 @@ truncate_t::truncate( { dberr_t err = DB_SUCCESS; char* path; - bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags); ut_a(!is_system_tablespace(space_id)); - if (has_data_dir) { + if (FSP_FLAGS_HAS_DATA_DIR(flags)) { ut_ad(dir_path != NULL); - path = fil_make_filepath(dir_path, tablename, IBD, true); - } else { path = fil_make_filepath(NULL, tablename, IBD, false); } diff --git a/storage/innobase/fsp/fsp0file.cc b/storage/innobase/fsp/fsp0file.cc index cdcd9924d41..22cd4862162 100644 --- a/storage/innobase/fsp/fsp0file.cc +++ b/storage/innobase/fsp/fsp0file.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2013, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -343,17 +344,37 @@ Datafile::read_first_page(bool read_only_mode) } } - if (err == DB_SUCCESS && m_order == 0) { + if (err != DB_SUCCESS) { + return(err); + } + if (m_order == 0) { + m_space_id = fsp_header_get_space_id(m_first_page); m_flags = fsp_header_get_flags(m_first_page); + if (!fsp_flags_is_valid(m_flags)) { + ulint cflags = fsp_flags_convert_from_101(m_flags); + if (cflags == ULINT_UNDEFINED) { + ib::error() + << "Invalid flags " << ib::hex(m_flags) + << " in " << m_filepath; + return(DB_CORRUPTION); + } else { + m_flags = cflags; + } + } + } - m_space_id = fsp_header_get_space_id(m_first_page); + const page_size_t ps(m_flags); + if (ps.physical() > page_size) { + ib::error() << "File " << m_filepath + << " should be longer than " + << page_size << " bytes"; + return(DB_CORRUPTION); } m_crypt_info = fil_space_read_crypt_data( m_space_id, m_first_page, - FSP_HEADER_OFFSET + fsp_header_get_encryption_offset( - page_size_t(m_flags))); + FSP_HEADER_OFFSET + fsp_header_get_encryption_offset(ps)); return(err); } @@ -374,14 +395,10 @@ space ID and flags. The file should exist and be successfully opened in order for this function to validate it. @param[in] space_id The expected tablespace ID. @param[in] flags The expected tablespace flags. -@param[in] for_import if it is for importing @retval DB_SUCCESS if tablespace is valid, DB_ERROR if not. m_is_valid is also set true on success, else false. */ dberr_t -Datafile::validate_to_dd( - ulint space_id, - ulint flags, - bool for_import) +Datafile::validate_to_dd(ulint space_id, ulint flags) { dberr_t err; @@ -392,17 +409,17 @@ Datafile::validate_to_dd( /* Validate this single-table-tablespace with the data dictionary, but do not compare the DATA_DIR flag, in case the tablespace was remotely located. */ - err = validate_first_page(0, for_import); + err = validate_first_page(0); if (err != DB_SUCCESS) { return(err); } + flags &= ~FSP_FLAGS_MEM_MASK; + /* Make sure the datafile we found matched the space ID. If the datafile is a file-per-table tablespace then also match the row format and zip page size. */ - if (m_space_id == space_id - && ((m_flags & ~FSP_FLAGS_MASK_DATA_DIR) - == (flags & ~FSP_FLAGS_MASK_DATA_DIR))) { + if (m_space_id == space_id && m_flags == flags) { /* Datafile matches the tablespace expected. */ return(DB_SUCCESS); } @@ -411,9 +428,10 @@ Datafile::validate_to_dd( m_is_valid = false; ib::error() << "In file '" << m_filepath << "', tablespace id and" - " flags are " << m_space_id << " and " << m_flags << ", but in" - " the InnoDB data dictionary they are " << space_id << " and " - << flags << ". Have you moved InnoDB .ibd files around without" + " flags are " << m_space_id << " and " << ib::hex(m_flags) + << ", but in the InnoDB data dictionary they are " + << space_id << " and " << ib::hex(flags) + << ". Have you moved InnoDB .ibd files around without" " using the commands DISCARD TABLESPACE and IMPORT TABLESPACE?" " " << TROUBLESHOOT_DATADICT_MSG; @@ -435,7 +453,7 @@ Datafile::validate_for_recovery() ut_ad(is_open()); ut_ad(!srv_read_only_mode); - err = validate_first_page(0, false); + err = validate_first_page(0); switch (err) { case DB_SUCCESS: @@ -463,14 +481,13 @@ Datafile::validate_for_recovery() return(err); } - err = restore_from_doublewrite(0); - if (err != DB_SUCCESS) { - return(err); + if (restore_from_doublewrite()) { + return(DB_CORRUPTION); } /* Free the previously read first page and then re-validate. */ free_first_page(); - err = validate_first_page(0, false); + err = validate_first_page(0); } if (err == DB_SUCCESS) { @@ -485,14 +502,11 @@ tablespace is opened. This occurs before the fil_space_t is created so the Space ID found here must not already be open. m_is_valid is set true on success, else false. @param[out] flush_lsn contents of FIL_PAGE_FILE_FLUSH_LSN -@param[in] for_import if it is for importing -(only valid for the first file of the system tablespace) @retval DB_SUCCESS on if the datafile is valid @retval DB_CORRUPTION if the datafile is not readable @retval DB_TABLESPACE_EXISTS if there is a duplicate space_id */ dberr_t -Datafile::validate_first_page(lsn_t* flush_lsn, - bool for_import) +Datafile::validate_first_page(lsn_t* flush_lsn) { char* prev_name; char* prev_filepath; @@ -550,8 +564,7 @@ Datafile::validate_first_page(lsn_t* flush_lsn, free_first_page(); return(DB_ERROR); - } else if (!fsp_flags_is_valid(m_flags) - || FSP_FLAGS_GET_TEMPORARY(m_flags)) { + } else if (!fsp_flags_is_valid(m_flags)) { /* Tablespace flags must be valid. */ error_txt = "Tablespace flags are invalid"; } else if (page_get_page_no(m_first_page) != 0) { @@ -768,17 +781,15 @@ Datafile::find_space_id() } -/** Finds a given page of the given space id from the double write buffer -and copies it to the corresponding .ibd file. -@param[in] page_no Page number to restore -@return DB_SUCCESS if page was restored from doublewrite, else DB_ERROR */ -dberr_t -Datafile::restore_from_doublewrite( - ulint restore_page_no) +/** Restore the first page of the tablespace from +the double write buffer. +@return whether the operation failed */ +bool +Datafile::restore_from_doublewrite() { /* Find if double write buffer contains page_no of given space id. */ - const byte* page = recv_sys->dblwr.find_page( - m_space_id, restore_page_no); + const byte* page = recv_sys->dblwr.find_page(m_space_id, 0); + const page_id_t page_id(m_space_id, 0); if (page == NULL) { /* If the first page of the given user tablespace is not there @@ -786,23 +797,34 @@ Datafile::restore_from_doublewrite( now. Hence this is treated as an error. */ ib::error() - << "Corrupted page " - << page_id_t(m_space_id, restore_page_no) + << "Corrupted page " << page_id << " of datafile '" << m_filepath << "' could not be found in the doublewrite buffer."; - return(DB_CORRUPTION); + return(true); } - const ulint flags = mach_read_from_4( + ulint flags = mach_read_from_4( FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + if (!fsp_flags_is_valid(flags)) { + ulint cflags = fsp_flags_convert_from_101(flags); + if (cflags == ULINT_UNDEFINED) { + ib::warn() + << "Ignoring a doublewrite copy of page " + << page_id + << " due to invalid flags " << ib::hex(flags); + return(true); + } + flags = cflags; + /* The flags on the page should be converted later. */ + } + const page_size_t page_size(flags); - ut_a(page_get_page_no(page) == restore_page_no); + ut_a(page_get_page_no(page) == page_id.page_no()); - ib::info() << "Restoring page " - << page_id_t(m_space_id, restore_page_no) + ib::info() << "Restoring page " << page_id << " of datafile '" << m_filepath << "' from the doublewrite buffer. Writing " << page_size.physical() << " bytes into file '" @@ -812,7 +834,8 @@ Datafile::restore_from_doublewrite( return(os_file_write( request, - m_filepath, m_handle, page, 0, page_size.physical())); + m_filepath, m_handle, page, 0, page_size.physical()) + != DB_SUCCESS); } /** Create a link filename based on the contents of m_name, diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 70d73ead0b7..27990e8243e 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -181,146 +181,6 @@ fsp_get_space_header( return(header); } -/** Convert a 32 bit integer tablespace flags to the 32 bit table flags. -This can only be done for a tablespace that was built as a file-per-table -tablespace. Note that the fsp_flags cannot show the difference between a -Compact and Redundant table, so an extra Compact boolean must be supplied. - Low order bit - | REDUNDANT | COMPACT | COMPRESSED | DYNAMIC -fil_space_t::flags | 0 | 0 | 1 | 1 -dict_table_t::flags | 0 | 1 | 1 | 1 -@param[in] fsp_flags fil_space_t::flags -@param[in] compact true if not Redundant row format -@return tablespace flags (fil_space_t::flags) */ -ulint -fsp_flags_to_dict_tf( - ulint fsp_flags, - bool compact) -{ - /* If the table in this file-per-table tablespace is Compact - row format, the low order bit will not indicate Compact. */ - bool post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(fsp_flags); - ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(fsp_flags); - bool atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(fsp_flags); - bool data_dir = FSP_FLAGS_HAS_DATA_DIR(fsp_flags); - bool page_compressed = FSP_FLAGS_GET_PAGE_COMPRESSION(fsp_flags); - ulint comp_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(fsp_flags); - - /* FSP_FLAGS_GET_TEMPORARY(fsp_flags) does not have an equivalent - flag position in the table flags. But it would go into flags2 if - any code is created where that is needed. */ - - ulint flags = dict_tf_init(post_antelope | compact, zip_ssize, - atomic_blobs, data_dir, - page_compressed, comp_level, 0); - - return(flags); -} - -/** Validate the tablespace flags. -These flags are stored in the tablespace header at offset FSP_SPACE_FLAGS. -They should be 0 for ROW_FORMAT=COMPACT and ROW_FORMAT=REDUNDANT. -The newer row formats, COMPRESSED and DYNAMIC, use a file format > Antelope -so they should have a file format number plus the DICT_TF_COMPACT bit set. -@param[in] flags Tablespace flags -@return true if valid, false if not */ -bool -fsp_flags_is_valid( - ulint flags) -{ - bool post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(flags); - ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); - bool atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(flags); - ulint page_ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); - bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags); - bool is_temp = FSP_FLAGS_GET_TEMPORARY(flags); - ulint unused = FSP_FLAGS_GET_UNUSED(flags); - bool page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(flags); - ulint page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags); - - const char *file; - ulint line; - -#define GOTO_ERROR file = __FILE__; line = __LINE__; goto err_exit; - - DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", return(false);); - - /* The Antelope row formats REDUNDANT and COMPACT did - not use tablespace flags, so the entire 4-byte field - is zero for Antelope row formats. */ - if (flags == 0) { - return(true); - } - - /* Barracuda row formats COMPRESSED and DYNAMIC use a feature called - ATOMIC_BLOBS which builds on the page structure introduced for the - COMPACT row format by allowing long fields to be broken into prefix - and externally stored parts. So if it is Post_antelope, it uses - Atomic BLOBs. */ - if (post_antelope != atomic_blobs) { - GOTO_ERROR; - return(false); - } - - /* Make sure there are no bits that we do not know about. */ - if (unused != 0) { - GOTO_ERROR; - } - - /* The zip ssize can be zero if it is other than compressed row format, - or it could be from 1 to the max. */ - if (zip_ssize > PAGE_ZIP_SSIZE_MAX) { - GOTO_ERROR; - } - - /* The actual page size must be within 4k and 16K (3 =< ssize =< 5). */ - if (page_ssize != 0 - && (page_ssize < UNIV_PAGE_SSIZE_MIN - || page_ssize > UNIV_PAGE_SSIZE_MAX)) { - GOTO_ERROR; - } - - /* Only single-table tablespaces use the DATA DIRECTORY clause. - It is not compatible with the TABLESPACE clause. Nor is it - compatible with the TEMPORARY clause. */ - if (has_data_dir && is_temp) { - GOTO_ERROR; - return(false); - } - - /* Page compression level requires page compression and atomic blobs - to be set */ - if (page_compression_level || page_compression) { - if (!page_compression || !atomic_blobs) { - GOTO_ERROR; - } - } - -#if UNIV_FORMAT_MAX != UNIV_FORMAT_B -# error UNIV_FORMAT_MAX != UNIV_FORMAT_B, Add more validations. -#endif -#if FSP_FLAGS_POS_UNUSED != 13 - //# error You have added a new FSP_FLAG without adding a validation check. -#endif - - return(true); - -err_exit: - ib::error() << "Tablespace flags: " << flags << " corrupted " - << " in file: " << file << " line: " << line - << " post_antelope: " << post_antelope - << " atomic_blobs: " << atomic_blobs - << " unused: " << unused - << " zip_ssize: " << zip_ssize << " max: " << PAGE_ZIP_SSIZE_MAX - << " page_ssize: " << page_ssize - << " " << UNIV_PAGE_SSIZE_MIN << ":" << UNIV_PAGE_SSIZE_MAX - << " has_data_dir: " << has_data_dir - << " is_temp: " << is_temp - << " page_compressed: " << page_compression - << " page_compression_level: " << page_compression_level; - return (false); -} - /** Check if checksum is disabled for the given space. @param[in] space_id tablespace ID @return true if checksum is disabled for given space. */ @@ -835,6 +695,7 @@ fsp_header_init_fields( ulint space_id, /*!< in: space id */ ulint flags) /*!< in: tablespace flags (FSP_SPACE_FLAGS) */ { + flags &= ~FSP_FLAGS_MEM_MASK; ut_a(fsp_flags_is_valid(flags)); mach_write_to_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page, @@ -889,7 +750,8 @@ fsp_header_init( mlog_write_ulint(header + FSP_SIZE, size, MLOG_4BYTES, mtr); mlog_write_ulint(header + FSP_FREE_LIMIT, 0, MLOG_4BYTES, mtr); - mlog_write_ulint(header + FSP_SPACE_FLAGS, space->flags, + mlog_write_ulint(header + FSP_SPACE_FLAGS, + space->flags & ~FSP_FLAGS_MEM_MASK, MLOG_4BYTES, mtr); mlog_write_ulint(header + FSP_FRAG_N_USED, 0, MLOG_4BYTES, mtr); diff --git a/storage/innobase/fsp/fsp0space.cc b/storage/innobase/fsp/fsp0space.cc index ed557fb8334..76269a749f9 100644 --- a/storage/innobase/fsp/fsp0space.cc +++ b/storage/innobase/fsp/fsp0space.cc @@ -126,15 +126,13 @@ Tablespace::open_or_create(bool is_temp) if (it == begin) { /* First data file. */ - ulint flags; - - flags = fsp_flags_set_page_size(0, univ_page_size); - /* Create the tablespace entry for the multi-file tablespace in the tablespace manager. */ space = fil_space_create( - m_name, m_space_id, flags, is_temp - ? FIL_TYPE_TEMPORARY : FIL_TYPE_TABLESPACE, it->m_crypt_info, + m_name, m_space_id, FSP_FLAGS_PAGE_SSIZE(), + is_temp + ? FIL_TYPE_TEMPORARY : FIL_TYPE_TABLESPACE, + it->m_crypt_info, false); } diff --git a/storage/innobase/fsp/fsp0sysspace.cc b/storage/innobase/fsp/fsp0sysspace.cc index 9a9e7051403..6f7d09b6faa 100644 --- a/storage/innobase/fsp/fsp0sysspace.cc +++ b/storage/innobase/fsp/fsp0sysspace.cc @@ -577,11 +577,11 @@ SysTablespace::read_lsn_and_check_flags(lsn_t* flushed_lsn) first datafile. */ for (int retry = 0; retry < 2; ++retry) { - err = it->validate_first_page(flushed_lsn, false); + err = it->validate_first_page(flushed_lsn); if (err != DB_SUCCESS && (retry == 1 - || it->restore_from_doublewrite(0) != DB_SUCCESS)) { + || it->restore_from_doublewrite())) { it->close(); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 93f5c936915..a5fd7788af2 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -3865,7 +3865,6 @@ innobase_init( uint format_id; ulong num_pll_degree; ulint srv_buf_pool_size_org = 0; - ulint fsp_flags =0; DBUG_ENTER("innobase_init"); handlerton* innobase_hton= (handlerton*) p; @@ -4095,12 +4094,7 @@ innobase_init( page_size_t(srv_page_size, srv_page_size, false)); srv_sys_space.set_space_id(TRX_SYS_SPACE); - - /* Create the filespace flags. */ - fsp_flags = fsp_flags_init( - univ_page_size, false, false, false, 0, 0); - srv_sys_space.set_flags(fsp_flags); - + srv_sys_space.set_flags(FSP_FLAGS_PAGE_SSIZE()); srv_sys_space.set_name("innodb_system"); srv_sys_space.set_path(srv_data_home); @@ -4121,11 +4115,7 @@ innobase_init( Set the name and path. */ srv_tmp_space.set_name("innodb_temporary"); srv_tmp_space.set_path(srv_data_home); - - /* Create the filespace flags with the temp flag set. */ - fsp_flags = fsp_flags_init( - univ_page_size, false, false, false, 0, 0); - srv_tmp_space.set_flags(fsp_flags); + srv_tmp_space.set_flags(FSP_FLAGS_PAGE_SSIZE()); if (!srv_tmp_space.parse_params(innobase_temp_data_file_path, false)) { DBUG_RETURN(innobase_init_abort()); diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index 349b105be6a..6e86cd3eb4a 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -1037,6 +1037,7 @@ fil_space_t::flags | 0 | 0 | 1 | 1 ================================================================== @param[in] table_flags dict_table_t::flags @return tablespace flags (fil_space_t::flags) */ +UNIV_INLINE ulint dict_tf_to_fsp_flags(ulint table_flags) MY_ATTRIBUTE((const)); diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index 3bb00294bfa..a75f0b245af 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -968,54 +968,56 @@ dict_tf_set( } } -/** Initialize a dict_table_t::flags pointer. -@param[in] compact, Table uses Compact or greater -@param[in] zip_ssize Zip Shift Size (log 2 minus 9) -@param[in] atomic_blobs Table uses Compressed or Dynamic -@param[in] data_dir Table uses DATA DIRECTORY -@param[in] page_compression Table uses page compression -@param[in] page_compression_level used compression level -@param[in] not_used For future */ +/** Convert a 32 bit integer table flags to the 32 bit FSP Flags. +Fsp Flags are written into the tablespace header at the offset +FSP_SPACE_FLAGS and are also stored in the fil_space_t::flags field. +The following chart shows the translation of the low order bit. +Other bits are the same. +========================= Low order bit ========================== + | REDUNDANT | COMPACT | COMPRESSED | DYNAMIC +dict_table_t::flags | 0 | 1 | 1 | 1 +fil_space_t::flags | 0 | 0 | 1 | 1 +================================================================== +@param[in] table_flags dict_table_t::flags +@return tablespace flags (fil_space_t::flags) */ UNIV_INLINE ulint -dict_tf_init( - bool compact, - ulint zip_ssize, - bool atomic_blobs, - bool data_dir, - bool page_compressed, - ulint page_compression_level, - ulint not_used) +dict_tf_to_fsp_flags(ulint table_flags) { - ulint flags = 0; + ulint fsp_flags; + ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL( + table_flags); + ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); - if (compact) { - flags |= DICT_TF_COMPACT; - } + ut_ad((DICT_TF_GET_PAGE_COMPRESSION(table_flags) == 0) + == (page_compression_level == 0)); - if (zip_ssize) { - flags |= (zip_ssize << DICT_TF_POS_ZIP_SSIZE); - } + DBUG_EXECUTE_IF("dict_tf_to_fsp_flags_failure", + return(ULINT_UNDEFINED);); - if (atomic_blobs) { - flags |= (1 << DICT_TF_POS_ATOMIC_BLOBS); - } + /* Adjust bit zero. */ + fsp_flags = DICT_TF_HAS_ATOMIC_BLOBS(table_flags) ? 1 : 0; + + /* ZIP_SSIZE and ATOMIC_BLOBS are at the same position. */ + fsp_flags |= table_flags + & (DICT_TF_MASK_ZIP_SSIZE | DICT_TF_MASK_ATOMIC_BLOBS); + + fsp_flags |= FSP_FLAGS_PAGE_SSIZE(); - if (data_dir) { - flags |= (1 << DICT_TF_POS_DATA_DIR); + if (page_compression_level) { + fsp_flags |= FSP_FLAGS_MASK_PAGE_COMPRESSION; } - if (page_compressed) { - flags |= (1 << DICT_TF_POS_ATOMIC_BLOBS) - | (1 << DICT_TF_POS_PAGE_COMPRESSION) - | (page_compression_level << DICT_TF_POS_PAGE_COMPRESSION_LEVEL); + ut_a(fsp_flags_is_valid(fsp_flags)); - ut_ad(zip_ssize == 0); - ut_ad(dict_tf_get_page_compression(flags) == TRUE); - ut_ad(dict_tf_get_page_compression_level(flags) == page_compression_level); + if (DICT_TF_HAS_DATA_DIR(table_flags)) { + fsp_flags |= 1U << FSP_FLAGS_MEM_DATA_DIR; } - return(flags); + fsp_flags |= atomic_writes << FSP_FLAGS_MEM_ATOMIC_WRITES; + fsp_flags |= page_compression_level << FSP_FLAGS_MEM_COMPRESSION_LEVEL; + + return(fsp_flags); } /********************************************************************//** @@ -1863,4 +1865,3 @@ dict_table_have_virtual_index( return(false); } - diff --git a/storage/innobase/include/dict0pagecompress.h b/storage/innobase/include/dict0pagecompress.h index f8873aec965..6641f6ba85f 100644 --- a/storage/innobase/include/dict0pagecompress.h +++ b/storage/innobase/include/dict0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013 SkySQL Ab. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -56,17 +56,6 @@ dict_table_page_compression_level( const dict_table_t* table) /*!< in: table */ __attribute__((const)); -/********************************************************************//** -Verify that dictionary flags match tablespace flags -@return true if flags match, false if not */ -UNIV_INLINE -ibool -dict_tf_verify_flags( -/*=================*/ - ulint table_flags, /*!< in: dict_table_t::flags */ - ulint fsp_flags) /*!< in: fil_space_t::flags */ - __attribute__((const)); - #ifndef UNIV_NONINL #include "dict0pagecompress.ic" #endif diff --git a/storage/innobase/include/dict0pagecompress.ic b/storage/innobase/include/dict0pagecompress.ic index 05a26f00711..67f4a26723f 100644 --- a/storage/innobase/include/dict0pagecompress.ic +++ b/storage/innobase/include/dict0pagecompress.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013 SkySQL Ab. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -25,80 +25,6 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com ***********************************************************************/ /********************************************************************//** -Verify that dictionary flags match tablespace flags -@return true if flags match, false if not */ -UNIV_INLINE -ibool -dict_tf_verify_flags( -/*=================*/ - ulint table_flags, /*!< in: dict_table_t::flags */ - ulint fsp_flags) /*!< in: fil_space_t::flags */ -{ - ulint table_unused = DICT_TF_GET_UNUSED(table_flags); - ulint compact = DICT_TF_GET_COMPACT(table_flags); - ulint ssize = DICT_TF_GET_ZIP_SSIZE(table_flags); - ulint atomic_blobs = DICT_TF_HAS_ATOMIC_BLOBS(table_flags); - ulint data_dir = DICT_TF_HAS_DATA_DIR(table_flags); - ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); - ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); - ulint post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(fsp_flags); - ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(fsp_flags); - ulint fsp_atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(fsp_flags); - ulint page_ssize = FSP_FLAGS_GET_PAGE_SSIZE(fsp_flags); - ulint fsp_unused = FSP_FLAGS_GET_UNUSED(fsp_flags); - ulint fsp_page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(fsp_flags); - ulint fsp_page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(fsp_flags); - - DBUG_EXECUTE_IF("dict_tf_verify_flags_failure", - return(ULINT_UNDEFINED);); - - ut_a(!table_unused); - ut_a(!fsp_unused); - ut_a(page_ssize == 0 || page_ssize != 0); /* silence compiler */ - ut_a(compact == 0 || compact == 1); /* silence compiler */ - ut_a(data_dir == 0 || data_dir == 1); /* silence compiler */ - ut_a(post_antelope == 0 || post_antelope == 1); /* silence compiler */ - - if (ssize != zip_ssize) { - fprintf(stderr, - "InnoDB: Error: table flags has zip_ssize %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has zip_ssize %ld\n", - ssize, zip_ssize); - return (FALSE); - } - if (atomic_blobs != fsp_atomic_blobs) { - fprintf(stderr, - "InnoDB: Error: table flags has atomic_blobs %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has atomic_blobs %ld\n", - atomic_blobs, fsp_atomic_blobs); - - return (FALSE); - } - if (page_compression != fsp_page_compression) { - fprintf(stderr, - "InnoDB: Error: table flags has page_compression %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file ahas page_compression %ld\n", - page_compression, fsp_page_compression); - - return (FALSE); - } - if (page_compression_level != fsp_page_compression_level) { - fprintf(stderr, - "InnoDB: Error: table flags has page_compression_level %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has page_compression_level %ld\n", - page_compression_level, fsp_page_compression_level); - - return (FALSE); - } - - return(TRUE); -} - -/********************************************************************//** Extract the page compression level from dict_table_t::flags. These flags are in memory, so assert that they are valid. @return page compression level, or 0 if not compressed */ diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 5cf75395d8c..7428ff2c936 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -42,7 +42,6 @@ Created 10/25/1995 Heikki Tuuri struct trx_t; class page_id_t; class truncate_t; -struct btr_create_t; /* structure containing encryption specification */ typedef struct fil_space_crypt_struct fil_space_crypt_t; @@ -131,7 +130,8 @@ struct fil_space_t { /*!< recovered tablespace size in pages; 0 if no size change was read from the redo log, or if the size change was implemented */ - ulint flags; /*!< tablespace flags; see + ulint flags; /*!< FSP_SPACE_FLAGS and FSP_FLAGS_MEM_ flags; + see fsp0types.h, fsp_flags_is_valid(), page_size_t(ulint) (constructor) */ ulint n_reserved_extents; @@ -970,6 +970,14 @@ fil_ibd_create( ulint key_id) MY_ATTRIBUTE((warn_unused_result)); +/** Try to adjust FSP_SPACE_FLAGS if they differ from the expectations. +(Typically when upgrading from MariaDB 10.1.0..10.1.20.) +@param[in] space_id tablespace ID +@param[in] flags desired tablespace flags */ +UNIV_INTERN +void +fsp_flags_try_adjust(ulint space_id, ulint flags); + /********************************************************************//** Tries to open a single-table tablespace and optionally checks the space id is right in it. If does not succeed, prints an error message to the .err log. This @@ -994,7 +1002,7 @@ statement to update the dictionary tables if they are incorrect. @param[in] fix_dict true if the dictionary is available to be fixed @param[in] purpose FIL_TYPE_TABLESPACE or FIL_TYPE_TEMPORARY @param[in] id tablespace ID -@param[in] flags tablespace flags +@param[in] flags expected FSP_SPACE_FLAGS @param[in] space_name tablespace name of the datafile If file-per-table, it is the table name in the databasename/tablename format @param[in] path_in expected filepath, usually read from dictionary @@ -1070,7 +1078,8 @@ fil_space_for_table_exists_in_mem( when find table space mismatch */ mem_heap_t* heap, /*!< in: heap memory */ table_id_t table_id, /*!< in: table id */ - dict_table_t* table); /*!< in: table or NULL */ + dict_table_t* table, /*!< in: table or NULL */ + ulint table_flags); /*!< in: table flags */ /** Try to extend a tablespace if it is smaller than the specified size. @param[in,out] space tablespace diff --git a/storage/innobase/include/fil0pagecompress.h b/storage/innobase/include/fil0pagecompress.h index e65d3491155..0bf6aa75f19 100644 --- a/storage/innobase/include/fil0pagecompress.h +++ b/storage/innobase/include/fil0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2016 MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017 MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -49,15 +49,6 @@ fil_space_is_page_compressed( /*=========================*/ ulint id); /*!< in: space id */ /*******************************************************************//** -Returns the page compression flag of the space, or false if the space -is not compressed. The tablespace must be cached in the memory cache. -@return true if page compressed, false if not or space not found */ -UNIV_INTERN -bool -fil_space_get_page_compressed( -/*=========================*/ - fil_space_t* space); /*!< in: space id */ -/*******************************************************************//** Returns the atomic writes flag of the space, or false if the space is not using atomic writes. The tablespace must be cached in the memory cache. @return atomic write table option value */ diff --git a/storage/innobase/include/fsp0file.h b/storage/innobase/include/fsp0file.h index 5129c63108a..1c4ac8152c9 100644 --- a/storage/innobase/include/fsp0file.h +++ b/storage/innobase/include/fsp0file.h @@ -231,13 +231,9 @@ public: successfully opened in order for this function to validate it. @param[in] space_id The expected tablespace ID. @param[in] flags The expected tablespace flags. - @param[in] for_import is it for importing @retval DB_SUCCESS if tablespace is valid, DB_ERROR if not. m_is_valid is also set true on success, else false. */ - dberr_t validate_to_dd( - ulint space_id, - ulint flags, - bool for_import) + dberr_t validate_to_dd(ulint space_id, ulint flags) MY_ATTRIBUTE((warn_unused_result)); /** Validates this datafile for the purpose of recovery. @@ -256,13 +252,10 @@ public: so the Space ID found here must not already be open. m_is_valid is set true on success, else false. @param[out] flush_lsn contents of FIL_PAGE_FILE_FLUSH_LSN - @param[in] for_import if it is for importing - (only valid for the first file of the system tablespace) @retval DB_SUCCESS on if the datafile is valid @retval DB_CORRUPTION if the datafile is not readable @retval DB_TABLESPACE_EXISTS if there is a duplicate space_id */ - dberr_t validate_first_page(lsn_t* flush_lsn, - bool for_import) + dberr_t validate_first_page(lsn_t* flush_lsn) MY_ATTRIBUTE((warn_unused_result)); /** Get Datafile::m_name. @@ -414,12 +407,10 @@ private: else DB_ERROR. */ dberr_t find_space_id(); - /** Finds a given page of the given space id from the double write - buffer and copies it to the corresponding .ibd file. - @param[in] page_no Page number to restore - @return DB_SUCCESS if page was restored, else DB_ERROR */ - dberr_t restore_from_doublewrite( - ulint restore_page_no); + /** Restore the first page of the tablespace from + the double write buffer. + @return whether the operation failed */ + bool restore_from_doublewrite(); /** Points into m_filepath to the file name with extension */ char* m_filename; diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index 2551677ecbe..fa8d5b76960 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -43,17 +43,55 @@ Created 12/18/1995 Heikki Tuuri #endif /* !UNIV_INNOCHECKSUM */ #include "fsp0types.h" -#define FSP_FLAGS_POS_DATA_DIR_ORACLE (FSP_FLAGS_POS_ATOMIC_BLOBS \ - + FSP_FLAGS_WIDTH_ATOMIC_BLOBS \ - + FSP_FLAGS_WIDTH_PAGE_SSIZE) -/** Bit mask of the DATA_DIR field */ -#define FSP_FLAGS_MASK_DATA_DIR_ORACLE \ - ((~(~0U << FSP_FLAGS_WIDTH_DATA_DIR)) \ - << FSP_FLAGS_POS_DATA_DIR_ORACLE) - -#define FSP_FLAGS_HAS_DATA_DIR_ORACLE(flags) \ - ((flags & FSP_FLAGS_MASK_DATA_DIR_ORACLE) \ - >> FSP_FLAGS_POS_DATA_DIR_ORACLE) +/** @return the PAGE_SSIZE flags for the current innodb_page_size */ +#define FSP_FLAGS_PAGE_SSIZE() \ + ((UNIV_PAGE_SIZE == UNIV_PAGE_SIZE_ORIG) ? \ + 0 : (UNIV_PAGE_SIZE_SHIFT - UNIV_ZIP_SIZE_SHIFT_MIN + 1) \ + << FSP_FLAGS_POS_PAGE_SSIZE) + +/* @defgroup Compatibility macros for MariaDB 10.1.0 through 10.1.20; +see the table in fsp0types.h @{ */ +/** Zero relative shift position of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101 \ + (FSP_FLAGS_POS_ATOMIC_BLOBS \ + + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) +/** Zero relative shift position of the PAGE_COMPRESSION_LEVEL field */ +#define FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101 \ + (FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101 + 1) +/** Zero relative shift position of the ATOMIC_WRITES field */ +#define FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101 \ + (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101 + 4) +/** Zero relative shift position of the PAGE_SSIZE field */ +#define FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101 \ + (FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101 + 2) + +/** Bit mask of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION_MARIADB101 \ + (1U << FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101) +/** Bit mask of the PAGE_COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL_MARIADB101 \ + (15U << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101) +/** Bit mask of the ATOMIC_WRITES field */ +#define FSP_FLAGS_MASK_ATOMIC_WRITES_MARIADB101 \ + (3U << FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101) +/** Bit mask of the PAGE_SSIZE field */ +#define FSP_FLAGS_MASK_PAGE_SSIZE_MARIADB101 \ + (15U << FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101) + +/** Return the value of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101) +/** Return the value of the PAGE_COMPRESSION_LEVEL field */ +#define FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101) +/** Return the value of the PAGE_SSIZE field */ +#define FSP_FLAGS_GET_PAGE_SSIZE_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_SSIZE_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101) + +/* @} */ /* @defgroup Tablespace Header Constants (moved from fsp0fsp.c) @{ */ @@ -629,47 +667,135 @@ fseg_print( mtr_t* mtr); /*!< in/out: mini-transaction */ #endif /* UNIV_BTR_PRINT */ -/** Determine if the tablespace is compressed from tablespace flags. -@param[in] flags Tablespace flags -@return true if compressed, false if not compressed */ -UNIV_INLINE -bool -fsp_flags_is_compressed( - ulint flags); - -/** Initialize an FSP flags integer. -@param[in] page_size page sizes in bytes and compression flag. -@param[in] atomic_blobs Used by Dynammic and Compressed. -@param[in] has_data_dir This tablespace is in a remote location. -@param[in] page_compressed Table uses page compression -@param[in] page_compression_level Page compression level -@param[in] not_used For future -@return tablespace flags after initialization */ +/** Convert FSP_SPACE_FLAGS from the buggy MariaDB 10.1.0..10.1.20 format. +@param[in] flags the contents of FSP_SPACE_FLAGS +@return the flags corrected from the buggy MariaDB 10.1 format +@retval ULINT_UNDEFINED if the flags are not in the buggy 10.1 format */ +MY_ATTRIBUTE((warn_unused_result, const)) UNIV_INLINE ulint -fsp_flags_init( - const page_size_t& page_size, - bool atomic_blobs, - bool has_data_dir, - bool page_compression, - ulint page_compression_level, - ulint not_used); - -/** Convert a 32 bit integer tablespace flags to the 32 bit table flags. -This can only be done for a tablespace that was built as a file-per-table -tablespace. Note that the fsp_flags cannot show the difference between a -Compact and Redundant table, so an extra Compact boolean must be supplied. - Low order bit - | REDUNDANT | COMPACT | COMPRESSED | DYNAMIC -fil_space_t::flags | 0 | 0 | 1 | 1 -dict_table_t::flags | 0 | 1 | 1 | 1 -@param[in] fsp_flags fil_space_t::flags -@param[in] compact true if not Redundant row format -@return tablespace flags (fil_space_t::flags) */ -ulint -fsp_flags_to_dict_tf( - ulint fsp_flags, - bool compact); +fsp_flags_convert_from_101(ulint flags) +{ + DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", + return(ULINT_UNDEFINED);); + if (flags == 0) { + return(flags); + } + + if (flags >> 18) { + /* The most significant FSP_SPACE_FLAGS bit that was ever set + by MariaDB 10.1.0 to 10.1.20 was bit 17 (misplaced DATA_DIR flag). + The flags must be less than 1<<18 in order to be valid. */ + return(ULINT_UNDEFINED); + } + + if ((flags & (FSP_FLAGS_MASK_POST_ANTELOPE | FSP_FLAGS_MASK_ATOMIC_BLOBS)) + == FSP_FLAGS_MASK_ATOMIC_BLOBS) { + /* If the "atomic blobs" flag (indicating + ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED) flag + is set, then the "post Antelope" (ROW_FORMAT!=REDUNDANT) flag + must also be set. */ + return(ULINT_UNDEFINED); + } + + /* Bits 6..10 denote compression in MariaDB 10.1.0 to 10.1.20. + They must be either 0b00000 or 0b00011 through 0b10011. + In correct versions, these bits would be + 0bd0sss where d is the DATA_DIR flag (garbage bit) and + sss is the PAGE_SSIZE (3, 4, 6, or 7). + + NOTE: MariaDB 10.1.0 to 10.1.20 can misinterpret + uncompressed data files with innodb_page_size=4k or 64k as + compressed innodb_page_size=16k files. Below is an exhaustive + state space analysis. + + -0by1zzz: impossible (the bit 4 must be clean; see above) + -0b101xx: DATA_DIR, innodb_page_size>4k: invalid (COMPRESSION_LEVEL>9) + +0bx0011: innodb_page_size=4k: + !!! Misinterpreted as COMPRESSION_LEVEL=9 or 1, COMPRESSION=1. + -0bx0010: impossible, because sss must be 0b011 or 0b1xx + -0bx0001: impossible, because sss must be 0b011 or 0b1xx + -0b10000: DATA_DIR, innodb_page_size=16: + invalid (COMPRESSION_LEVEL=8 but COMPRESSION=0) + +0b00111: no DATA_DIR, innodb_page_size=64k: + !!! Misinterpreted as COMPRESSION_LEVEL=3, COMPRESSION=1. + -0b00101: impossible, because sss must be 0 for 16k, not 0b101 + -0b001x0: no DATA_DIR, innodb_page_size=32k or 8k: + invalid (COMPRESSION_LEVEL=3 but COMPRESSION=0) + +0b00000: innodb_page_size=16k (looks like COMPRESSION=0) + ??? Could actually be compressed; see PAGE_SSIZE below */ + const ulint level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL_MARIADB101( + flags); + if (FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) != (level != 0) + || level > 9) { + /* The compression flags are not in the buggy MariaDB + 10.1 format. */ + return(ULINT_UNDEFINED); + } + if (!(~flags & FSP_FLAGS_MASK_ATOMIC_WRITES_MARIADB101)) { + /* The ATOMIC_WRITES flags cannot be 0b11. + (The bits 11..12 should actually never be 0b11, + because in MySQL they would be SHARED|TEMPORARY.) */ + return(ULINT_UNDEFINED); + } + + /* Bits 13..16 are the wrong position for PAGE_SSIZE, and they + should contain one of the values 3,4,6,7, that is, be of the form + 0b0011 or 0b01xx (except 0b0110). + In correct versions, these bits should be 0bc0se + where c is the MariaDB COMPRESSED flag + and e is the MySQL 5.7 ENCRYPTION flag + and s is the MySQL 8.0 SDI flag. MariaDB can only support s=0, e=0. + + Compressed innodb_page_size=16k tables with correct FSP_SPACE_FLAGS + will be properly rejected by older MariaDB 10.1.x because they + would read as PAGE_SSIZE>=8 which is not valid. */ + + const ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE_MARIADB101(flags); + if (ssize == 1 || ssize == 2 || ssize == 5 || ssize & 8) { + /* the page_size is not between 4k and 64k; + 16k should be encoded as 0, not 5 */ + return(ULINT_UNDEFINED); + } + const ulint zssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); + if (zssize == 0) { + /* not ROW_FORMAT=COMPRESSED */ + } else if (zssize > (ssize ? ssize : 5)) { + /* invalid KEY_BLOCK_SIZE */ + return(ULINT_UNDEFINED); + } else if (~flags & (FSP_FLAGS_MASK_POST_ANTELOPE + | FSP_FLAGS_MASK_ATOMIC_BLOBS)) { + /* both these flags should be set for + ROW_FORMAT=COMPRESSED */ + return(ULINT_UNDEFINED); + } + + flags = ((flags & 0x3f) | ssize << FSP_FLAGS_POS_PAGE_SSIZE + | FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) + << FSP_FLAGS_POS_PAGE_COMPRESSION); + ut_ad(fsp_flags_is_valid(flags)); + return(flags); +} + +/** Compare tablespace flags. +@param[in] expected expected flags from dict_tf_to_fsp_flags() +@param[in] actual flags read from FSP_SPACE_FLAGS +@return whether the flags match */ +MY_ATTRIBUTE((warn_unused_result)) +UNIV_INLINE +bool +fsp_flags_match(ulint expected, ulint actual) +{ + expected &= ~FSP_FLAGS_MEM_MASK; + ut_ad(fsp_flags_is_valid(expected)); + + if (actual == expected) { + return(true); + } + + actual = fsp_flags_convert_from_101(actual); + return(actual == expected); +} /** Calculates the descriptor index within a descriptor page. @param[in] page_size page size diff --git a/storage/innobase/include/fsp0fsp.ic b/storage/innobase/include/fsp0fsp.ic index 6d0d6f05a9f..6317a67a089 100644 --- a/storage/innobase/include/fsp0fsp.ic +++ b/storage/innobase/include/fsp0fsp.ic @@ -40,162 +40,6 @@ fsp_descr_page( == FSP_XDES_OFFSET); } -/** Determine if the tablespace is compressed from tablespace flags. -@param[in] flags Tablespace flags -@return true if compressed, false if not compressed */ -UNIV_INLINE -bool -fsp_flags_is_compressed( - ulint flags) -{ - return(FSP_FLAGS_GET_ZIP_SSIZE(flags) != 0); -} - -#define ACTUAL_SSIZE(ssize) (0 == ssize ? UNIV_PAGE_SSIZE_ORIG : ssize) - -/** Convert a page size, which is a power of 2, to an ssize, which is -the number of bit shifts from 512 to make that page size. -@param[in] page_size compressed page size in bytes -@return an ssize created from the page size provided. */ -UNIV_INLINE -ulint -page_size_to_ssize( - ulint page_size) -{ - ulint ssize; - - for (ssize = UNIV_ZIP_SIZE_SHIFT_MIN; - ((ulint) 1 << ssize) < page_size; - ssize++) {}; - - return(ssize - UNIV_ZIP_SIZE_SHIFT_MIN + 1); -} - -/** Add the compressed page size to the tablespace flags. -@param[in] flags Tablespace flags -@param[in] page_size page sizes in bytes and compression flag. -@return tablespace flags after zip size is added */ -UNIV_INLINE -ulint -fsp_flags_set_zip_size( - ulint flags, - const page_size_t& page_size) -{ - if (!page_size.is_compressed()) { - return(flags); - } - - /* Zip size should be a power of 2 between UNIV_ZIP_SIZE_MIN - and UNIV_ZIP_SIZE_MAX */ - ut_ad(page_size.physical() >= UNIV_ZIP_SIZE_MIN); - ut_ad(page_size.physical() <= UNIV_ZIP_SIZE_MAX); - ut_ad(ut_is_2pow(page_size.physical())); - - ulint ssize = page_size_to_ssize(page_size.physical()); - - ut_ad(ssize > 0); - ut_ad(ssize <= UNIV_PAGE_SSIZE_MAX); - - flags |= (ssize << FSP_FLAGS_POS_ZIP_SSIZE); - - ut_ad(fsp_flags_is_valid(flags)); - - return(flags); -} - -/** Add the page size to the tablespace flags. -@param[in] flags Tablespace flags -@param[in] page_size page sizes in bytes and compression flag. -@return tablespace flags after page size is added */ -UNIV_INLINE -ulint -fsp_flags_set_page_size( - ulint flags, - const page_size_t& page_size) -{ - /* Page size should be a power of two between UNIV_PAGE_SIZE_MIN - and UNIV_PAGE_SIZE */ - ut_ad(page_size.logical() >= UNIV_PAGE_SIZE_MIN); - ut_ad(page_size.logical() <= UNIV_PAGE_SIZE_MAX); - ut_ad(ut_is_2pow(page_size.logical())); - - /* Remove this assert once we add support for different - page size per tablespace. Currently all tablespaces must - have a page size that is equal to innodb-page-size */ - ut_ad(page_size.logical() == UNIV_PAGE_SIZE); - - if (page_size.logical() == UNIV_PAGE_SIZE_ORIG) { - ut_ad(0 == FSP_FLAGS_GET_PAGE_SSIZE(flags)); - - } else { - ulint ssize = page_size_to_ssize(page_size.logical()); - - ut_ad(ssize); - ut_ad(ssize <= UNIV_PAGE_SSIZE_MAX); - - flags |= (ssize << FSP_FLAGS_POS_PAGE_SSIZE); - } - - ut_ad(fsp_flags_is_valid(flags)); - - return(flags); -} - -/** Initialize an FSP flags integer. -@param[in] page_size page sizes in bytes and compression flag. -@param[in] atomic_blobs Used by Dynammic and Compressed. -@param[in] has_data_dir This tablespace is in a remote location. -@param[in] page_compressed Table uses page compression -@param[in] page_compression_level Page compression level -@param[in] not_used For future -@@return tablespace flags after initialization */ -UNIV_INLINE -ulint -fsp_flags_init( - const page_size_t& page_size, - bool atomic_blobs, - bool has_data_dir, - bool page_compression, - ulint page_compression_level, - ulint not_used) -{ - ut_ad(page_size.physical() <= page_size.logical()); - ut_ad(!page_size.is_compressed() || atomic_blobs); - - /* Page size should be a power of two between UNIV_PAGE_SIZE_MIN - and UNIV_PAGE_SIZE, but zip_size may be 0 if not compressed. */ - ulint flags = fsp_flags_set_page_size(0, page_size); - - if (atomic_blobs) { - flags |= FSP_FLAGS_MASK_POST_ANTELOPE - | FSP_FLAGS_MASK_ATOMIC_BLOBS; - } - - /* If the zip_size is explicit and different from the default, - compressed row format is implied. */ - flags = fsp_flags_set_zip_size(flags, page_size); - - if (has_data_dir) { - flags |= FSP_FLAGS_MASK_DATA_DIR; - } - - /* In addition, tablespace flags also contain if the page - compression is used for this table. */ - if (page_compression) { - flags |= FSP_FLAGS_SET_PAGE_COMPRESSION(flags, page_compression); - } - - /* In addition, tablespace flags also contain page compression level - if page compression is used for this table. */ - if (page_compression && page_compression_level) { - flags |= FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(flags, page_compression_level); - } - - ut_ad(fsp_flags_is_valid(flags)); - - return(flags); -} - /** Calculates the descriptor index within a descriptor page. @param[in] page_size page size @param[in] offset page offset diff --git a/storage/innobase/include/fsp0pagecompress.h b/storage/innobase/include/fsp0pagecompress.h index 9038aa0fdef..b45a4b87890 100644 --- a/storage/innobase/include/fsp0pagecompress.h +++ b/storage/innobase/include/fsp0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -48,15 +48,6 @@ fsp_header_get_compression_level( const page_t* page); /*!< in: first page of a tablespace */ /********************************************************************//** -Determine if the tablespace is page compressed from dict_table_t::flags. -@return TRUE if page compressed, FALSE if not compressed */ -UNIV_INLINE -bool -fsp_flags_is_page_compressed( -/*=========================*/ - ulint flags); /*!< in: tablespace flags */ - -/********************************************************************//** Extract the page compression level from tablespace flags. A tablespace has only one physical page compression level whether that page is compressed or not. diff --git a/storage/innobase/include/fsp0pagecompress.ic b/storage/innobase/include/fsp0pagecompress.ic index 0915fae4b92..bb11371d712 100644 --- a/storage/innobase/include/fsp0pagecompress.ic +++ b/storage/innobase/include/fsp0pagecompress.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -26,18 +26,6 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@mariadb.com ***********************************************************************/ /********************************************************************//** -Determine if the tablespace is page compressed from dict_table_t::flags. -@return TRUE if page compressed, FALSE if not page compressed */ -UNIV_INLINE -bool -fsp_flags_is_page_compressed( -/*=========================*/ - ulint flags) /*!< in: tablespace flags */ -{ - return(FSP_FLAGS_GET_PAGE_COMPRESSION(flags)); -} - -/********************************************************************//** Determine the tablespace is page compression level from dict_table_t::flags. @return page compression level or 0 if not compressed*/ UNIV_INLINE @@ -119,16 +107,10 @@ fil_space_is_page_compressed( /*=========================*/ ulint id) /*!< in: space id */ { - ulint flags; - - flags = fil_space_get_flags(id); - - if (flags && flags != ULINT_UNDEFINED) { - - return(fsp_flags_is_page_compressed(flags)); - } + ulint flags = fil_space_get_flags(id); - return(0); + return(flags != ULINT_UNDEFINED + && FSP_FLAGS_HAS_PAGE_COMPRESSION(flags)); } #endif /* UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/include/fsp0types.h b/storage/innobase/include/fsp0types.h index 1c535677e83..429af4210b7 100644 --- a/storage/innobase/include/fsp0types.h +++ b/storage/innobase/include/fsp0types.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2016, MariaDB Corporation. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -187,18 +187,6 @@ every XDES_DESCRIBED_PER_PAGE pages in every tablespace. */ /*--------------------------------------*/ /* @} */ -/** Validate the tablespace flags. -These flags are stored in the tablespace header at offset FSP_SPACE_FLAGS. -They should be 0 for ROW_FORMAT=COMPACT and ROW_FORMAT=REDUNDANT. -The newer row formats, COMPRESSED and DYNAMIC, use a file format > Antelope -so they should have a file format number plus the DICT_TF_COMPACT bit set. -@param[in] flags Tablespace flags -@return true if valid, false if not */ -bool -fsp_flags_is_valid( - ulint flags) - MY_ATTRIBUTE((warn_unused_result, const)); - /** Check if tablespace is system temporary. @param[in] space_id verify is checksum is enabled for given space. @return true if tablespace is system temporary. */ @@ -240,43 +228,68 @@ to the two Barracuda row formats COMPRESSED and DYNAMIC. */ #define FSP_FLAGS_WIDTH_ATOMIC_BLOBS 1 /** Number of flag bits used to indicate the tablespace page size */ #define FSP_FLAGS_WIDTH_PAGE_SSIZE 4 -/** Width of the DATA_DIR flag. This flag indicates that the tablespace -is found in a remote location, not the default data directory. */ -#define FSP_FLAGS_WIDTH_DATA_DIR 1 -/** Width of the SHARED flag. This flag indicates that the tablespace -was created with CREATE TABLESPACE and can be shared by multiple tables. */ -#define FSP_FLAGS_WIDTH_SHARED 1 -/** Width of the TEMPORARY flag. This flag indicates that the tablespace -is a temporary tablespace and everything in it is temporary, meaning that -it is for a single client and should be deleted upon startup if it exists. */ -#define FSP_FLAGS_WIDTH_TEMPORARY 1 -/** Width of the encryption flag. This flag indicates that the tablespace -is a tablespace with encryption. */ -#define FSP_FLAGS_WIDTH_ENCRYPTION 1 - -/** Number of flag bits used to indicate the page compression and compression level */ -#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION 1 -#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL 4 - -/** Number of flag bits used to indicate atomic writes for this tablespace */ -#define FSP_FLAGS_WIDTH_ATOMIC_WRITES 2 - -/** Width of all the currently known tablespace flags */ +/** Number of reserved bits */ +#define FSP_FLAGS_WIDTH_RESERVED 6 +/** Number of flag bits used to indicate the page compression */ +#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION 1 + +/** Width of all the currently known persistent tablespace flags */ #define FSP_FLAGS_WIDTH (FSP_FLAGS_WIDTH_POST_ANTELOPE \ + FSP_FLAGS_WIDTH_ZIP_SSIZE \ + FSP_FLAGS_WIDTH_ATOMIC_BLOBS \ + FSP_FLAGS_WIDTH_PAGE_SSIZE \ - + FSP_FLAGS_WIDTH_DATA_DIR \ - + FSP_FLAGS_WIDTH_SHARED \ - + FSP_FLAGS_WIDTH_TEMPORARY \ - + FSP_FLAGS_WIDTH_ENCRYPTION \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES ) - -/** A mask of all the known/used bits in tablespace flags */ + + FSP_FLAGS_WIDTH_RESERVED \ + + FSP_FLAGS_WIDTH_PAGE_COMPRESSION) + +/** A mask of all the known/used bits in FSP_SPACE_FLAGS */ #define FSP_FLAGS_MASK (~(~0U << FSP_FLAGS_WIDTH)) +/* FSP_SPACE_FLAGS position and name in MySQL 5.6/MariaDB 10.0 or older +and MariaDB 10.1.20 or older MariaDB 10.1 and in MariaDB 10.1.21 +or newer. +MySQL 5.6 MariaDB 10.1.x MariaDB 10.1.21 +==================================================================== +Below flags in same offset +==================================================================== +0: POST_ANTELOPE 0:POST_ANTELOPE 0: POST_ANTELOPE +1..4: ZIP_SSIZE(0..5) 1..4:ZIP_SSIZE(0..5) 1..4: ZIP_SSIZE(0..5) +(NOTE: bit 4 is always 0) +5: ATOMIC_BLOBS 5:ATOMIC_BLOBS 5: ATOMIC_BLOBS +===================================================================== +Below note the order difference: +===================================================================== +6..9: PAGE_SSIZE(3..7) 6: COMPRESSION 6..9: PAGE_SSIZE(3..7) +10: DATA_DIR 7..10: COMP_LEVEL(0..9) 10: RESERVED (5.6 DATA_DIR) +===================================================================== +The flags below were in incorrect position in MariaDB 10.1, +or have been introduced in MySQL 5.7 or 8.0: +===================================================================== +11: UNUSED 11..12:ATOMIC_WRITES 11: RESERVED (5.7 SHARED) + 12: RESERVED (5.7 TEMPORARY) + 13..15:PAGE_SSIZE(3..7) 13: RESERVED (5.7 ENCRYPTION) + 14: RESERVED (8.0 SDI) + 15: RESERVED + 16: PAGE_SSIZE_msb(0) 16: COMPRESSION + 17: DATA_DIR 17: UNUSED + 18: UNUSED +===================================================================== +The flags below only exist in fil_space_t::flags, not in FSP_SPACE_FLAGS: +===================================================================== + 25: DATA_DIR + 26..27: ATOMIC_WRITES + 28..31: COMPRESSION_LEVEL +*/ + +/** A mask of the memory-only flags in fil_space_t::flags */ +#define FSP_FLAGS_MEM_MASK (~0U << FSP_FLAGS_MEM_DATA_DIR) + +/** Zero relative shift position of the DATA_DIR flag */ +#define FSP_FLAGS_MEM_DATA_DIR 25 +/** Zero relative shift position of the ATOMIC_WRITES field */ +#define FSP_FLAGS_MEM_ATOMIC_WRITES 26 +/** Zero relative shift position of the COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MEM_COMPRESSION_LEVEL 28 + /** Zero relative shift position of the POST_ANTELOPE field */ #define FSP_FLAGS_POS_POST_ANTELOPE 0 /** Zero relative shift position of the ZIP_SSIZE field */ @@ -285,34 +298,16 @@ is a tablespace with encryption. */ /** Zero relative shift position of the ATOMIC_BLOBS field */ #define FSP_FLAGS_POS_ATOMIC_BLOBS (FSP_FLAGS_POS_ZIP_SSIZE \ + FSP_FLAGS_WIDTH_ZIP_SSIZE) -/** Zero relative shift position of the PAGE_SSIZE field */ +/** Zero relative shift position of the start of the PAGE_SSIZE bits */ #define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_ATOMIC_BLOBS \ - + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) -/** Zero relative shift position of the start of the DATA_DIR bit */ -#define FSP_FLAGS_POS_DATA_DIR (FSP_FLAGS_POS_PAGE_SSIZE \ + + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) +/** Zero relative shift position of the start of the RESERVED bits +these are only used in MySQL 5.7 and used for compatibility. */ +#define FSP_FLAGS_POS_RESERVED (FSP_FLAGS_POS_PAGE_SSIZE \ + FSP_FLAGS_WIDTH_PAGE_SSIZE) -/** Zero relative shift position of the start of the SHARED bit */ -#define FSP_FLAGS_POS_SHARED (FSP_FLAGS_POS_DATA_DIR \ - + FSP_FLAGS_WIDTH_DATA_DIR) -/** Zero relative shift position of the start of the TEMPORARY bit */ -#define FSP_FLAGS_POS_TEMPORARY (FSP_FLAGS_POS_SHARED \ - + FSP_FLAGS_WIDTH_SHARED) -/** Zero relative shift position of the start of the ENCRYPTION bit */ -#define FSP_FLAGS_POS_ENCRYPTION (FSP_FLAGS_POS_TEMPORARY \ - + FSP_FLAGS_WIDTH_TEMPORARY) /** Zero relative shift position of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_POS_PAGE_COMPRESSION (FSP_FLAGS_POS_ENCRYPTION \ - + FSP_FLAGS_WIDTH_ENCRYPTION) -/** Zero relative shift position of the PAGE_COMPRESSION_LEVEL field */ -#define FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL (FSP_FLAGS_POS_PAGE_COMPRESSION \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION) -/** Zero relative shift position of the ATOMIC_WRITES field */ -#define FSP_FLAGS_POS_ATOMIC_WRITES (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL) -/** Zero relative shift position of the start of the UNUSED bits */ -#define FSP_FLAGS_POS_UNUSED (FSP_FLAGS_POS_ATOMIC_WRITES \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES) - +#define FSP_FLAGS_POS_PAGE_COMPRESSION (FSP_FLAGS_POS_RESERVED \ + + FSP_FLAGS_WIDTH_RESERVED) /** Bit mask of the POST_ANTELOPE field */ #define FSP_FLAGS_MASK_POST_ANTELOPE \ @@ -330,26 +325,22 @@ is a tablespace with encryption. */ #define FSP_FLAGS_MASK_PAGE_SSIZE \ ((~(~0U << FSP_FLAGS_WIDTH_PAGE_SSIZE)) \ << FSP_FLAGS_POS_PAGE_SSIZE) -/** Bit mask of the DATA_DIR field */ -#define FSP_FLAGS_MASK_DATA_DIR \ - ((~(~0U << FSP_FLAGS_WIDTH_DATA_DIR)) \ - << FSP_FLAGS_POS_DATA_DIR) -/** Bit mask of the TEMPORARY field */ -#define FSP_FLAGS_MASK_TEMPORARY \ - ((~(~0U << FSP_FLAGS_WIDTH_TEMPORARY)) \ - << FSP_FLAGS_POS_TEMPORARY) +/** Bit mask of the RESERVED1 field */ +#define FSP_FLAGS_MASK_RESERVED \ + ((~(~0U << FSP_FLAGS_WIDTH_RESERVED)) \ + << FSP_FLAGS_POS_RESERVED) /** Bit mask of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_MASK_PAGE_COMPRESSION \ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION \ ((~(~0U << FSP_FLAGS_WIDTH_PAGE_COMPRESSION)) \ << FSP_FLAGS_POS_PAGE_COMPRESSION) -/** Bit mask of the PAGE_COMPRESSION_LEVEL field */ -#define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL \ - ((~(~0U << FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL)) \ - << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) -/** Bit mask of the ATOMIC_WRITES field */ -#define FSP_FLAGS_MASK_ATOMIC_WRITES \ - ((~(~0U << FSP_FLAGS_WIDTH_ATOMIC_WRITES)) \ - << FSP_FLAGS_POS_ATOMIC_WRITES) + +/** Bit mask of the in-memory ATOMIC_WRITES field */ +#define FSP_FLAGS_MASK_MEM_ATOMIC_WRITES \ + (3U << FSP_FLAGS_MEM_ATOMIC_WRITES) + +/** Bit mask of the in-memory COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MASK_MEM_COMPRESSION_LEVEL \ + (15U << FSP_FLAGS_MEM_COMPRESSION_LEVEL) /** Return the value of the POST_ANTELOPE field */ #define FSP_FLAGS_GET_POST_ANTELOPE(flags) \ @@ -367,43 +358,88 @@ is a tablespace with encryption. */ #define FSP_FLAGS_GET_PAGE_SSIZE(flags) \ ((flags & FSP_FLAGS_MASK_PAGE_SSIZE) \ >> FSP_FLAGS_POS_PAGE_SSIZE) -/** Return the value of the DATA_DIR field */ -#define FSP_FLAGS_HAS_DATA_DIR(flags) \ - ((flags & FSP_FLAGS_MASK_DATA_DIR) \ - >> FSP_FLAGS_POS_DATA_DIR) -/** Return the contents of the TEMPORARY field */ -#define FSP_FLAGS_GET_TEMPORARY(flags) \ - ((flags & FSP_FLAGS_MASK_TEMPORARY) \ - >> FSP_FLAGS_POS_TEMPORARY) +/** @return the RESERVED flags */ +#define FSP_FLAGS_GET_RESERVED(flags) \ + ((flags & FSP_FLAGS_MASK_RESERVED) \ + >> FSP_FLAGS_POS_RESERVED) +/** @return the PAGE_COMPRESSION flag */ +#define FSP_FLAGS_HAS_PAGE_COMPRESSION(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION) + /** Return the contents of the UNUSED bits */ #define FSP_FLAGS_GET_UNUSED(flags) \ (flags >> FSP_FLAGS_POS_UNUSED) -/** Return the value of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_GET_PAGE_COMPRESSION(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION) \ - >> FSP_FLAGS_POS_PAGE_COMPRESSION) -/** Return the value of the PAGE_COMPRESSION_LEVEL field */ + +/** @return the value of the DATA_DIR field */ +#define FSP_FLAGS_HAS_DATA_DIR(flags) \ + (flags & 1U << FSP_FLAGS_MEM_DATA_DIR) +/** @return the COMPRESSION_LEVEL field */ #define FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL) \ - >> FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) -/** Return the value of the ATOMIC_WRITES field */ + ((flags & FSP_FLAGS_MASK_MEM_COMPRESSION_LEVEL) \ + >> FSP_FLAGS_MEM_COMPRESSION_LEVEL) +/** @return the ATOMIC_WRITES field */ #define FSP_FLAGS_GET_ATOMIC_WRITES(flags) \ - ((flags & FSP_FLAGS_MASK_ATOMIC_WRITES) \ - >> FSP_FLAGS_POS_ATOMIC_WRITES) -/* @} */ + ((flags & FSP_FLAGS_MASK_MEM_ATOMIC_WRITES) \ + >> FSP_FLAGS_MEM_ATOMIC_WRITES) -/** Set a PAGE_COMPRESSION into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_COMPRESSION(flags, compression) \ - (flags | (compression << FSP_FLAGS_POS_PAGE_COMPRESSION)) +/* @} */ -/** Set a PAGE_COMPRESSION_LEVEL into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(flags, level) \ - (flags | (level << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL)) +/** Validate the tablespace flags, which are stored in the +tablespace header at offset FSP_SPACE_FLAGS. +@param[in] flags the contents of FSP_SPACE_FLAGS +@return whether the flags are correct (not in the buggy 10.1) format */ +MY_ATTRIBUTE((warn_unused_result, const)) +UNIV_INLINE +bool +fsp_flags_is_valid(ulint flags) +{ + DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", + return(false);); + if (flags == 0) { + return(true); + } + if (flags & ~FSP_FLAGS_MASK) { + return(false); + } + if ((flags & (FSP_FLAGS_MASK_POST_ANTELOPE | FSP_FLAGS_MASK_ATOMIC_BLOBS)) + == FSP_FLAGS_MASK_ATOMIC_BLOBS) { + /* If the "atomic blobs" flag (indicating + ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED) flag + is set, then the "post Antelope" (ROW_FORMAT!=REDUNDANT) flag + must also be set. */ + return(false); + } + /* Bits 10..14 should be 0b0000d where d is the DATA_DIR flag + of MySQL 5.6 and MariaDB 10.0, which we ignore. + In the buggy FSP_SPACE_FLAGS written by MariaDB 10.1.0 to 10.1.20, + bits 10..14 would be nonzero 0bsssaa where sss is + nonzero PAGE_SSIZE (3, 4, 6, or 7) + and aa is ATOMIC_WRITES (not 0b11). */ + if (FSP_FLAGS_GET_RESERVED(flags) & ~1) { + return(false); + } + + const ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); + if (ssize == 1 || ssize == 2 || ssize == 5 || ssize & 8) { + /* the page_size is not between 4k and 64k; + 16k should be encoded as 0, not 5 */ + return(false); + } + const ulint zssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); + if (zssize == 0) { + /* not ROW_FORMAT=COMPRESSED */ + } else if (zssize > (ssize ? ssize : 5)) { + /* invalid KEY_BLOCK_SIZE */ + return(false); + } else if (~flags & (FSP_FLAGS_MASK_POST_ANTELOPE + | FSP_FLAGS_MASK_ATOMIC_BLOBS)) { + /* both these flags should be set for + ROW_FORMAT=COMPRESSED */ + return(false); + } + + return(true); +} -/** Set a ATOMIC_WRITES into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_ATOMIC_WRITES(flags, atomics) \ - (flags | (atomics << FSP_FLAGS_POS_ATOMIC_WRITES)) #endif /* fsp0types_h */ diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h index f54c935027b..acffa1b562b 100644 --- a/storage/innobase/include/log0recv.h +++ b/storage/innobase/include/log0recv.h @@ -173,7 +173,7 @@ struct recv_addr_t{ struct recv_dblwr_t { /** Add a page frame to the doublewrite recovery buffer. */ - void add(const byte* page) { + void add(byte* page) { pages.push_back(page); } @@ -184,7 +184,7 @@ struct recv_dblwr_t { @retval NULL if no page was found */ const byte* find_page(ulint space_id, ulint page_no); - typedef std::list<const byte*, ut_allocator<const byte*> > list; + typedef std::list<byte*, ut_allocator<byte*> > list; /** Recovered doublewrite buffer page frames */ list pages; diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index 7fe925ae3ee..a6f4b221302 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -361,8 +361,7 @@ public: m_space(ULINT_UNDEFINED), m_xdes(), m_xdes_page_no(ULINT_UNDEFINED), - m_space_flags(ULINT_UNDEFINED), - m_table_flags(ULINT_UNDEFINED) UNIV_NOTHROW { } + m_space_flags(ULINT_UNDEFINED) UNIV_NOTHROW { } /** Free any extent descriptor instance */ virtual ~AbstractCallback() @@ -529,10 +528,6 @@ protected: /** Flags value read from the header page */ ulint m_space_flags; - - /** Derived from m_space_flags and row format type, the row format - type is determined from the page header. */ - ulint m_table_flags; }; /** Determine the page size to use for traversing the tablespace @@ -547,6 +542,18 @@ AbstractCallback::init( const page_t* page = block->frame; m_space_flags = fsp_header_get_flags(page); + if (!fsp_flags_is_valid(m_space_flags)) { + ulint cflags = fsp_flags_convert_from_101(m_space_flags); + if (cflags == ULINT_UNDEFINED) { + ib::error() << "Invalid FSP_SPACE_FLAGS=" + << ib::hex(m_space_flags); + return(DB_CORRUPTION); + } + m_space_flags = cflags; + } + + /* Clear the DATA_DIR flag, which is basically garbage. */ + m_space_flags &= ~(1U << FSP_FLAGS_POS_RESERVED); m_page_size.copy_from(page_size_t(m_space_flags)); if (!is_compressed_table() && !m_page_size.equals_to(univ_page_size)) { @@ -614,45 +621,6 @@ struct FetchIndexRootPages : public AbstractCallback { return(m_space); } - /** Check if the .ibd file row format is the same as the table's. - @param ibd_table_flags determined from space and page. - @return DB_SUCCESS or error code. */ - dberr_t check_row_format(ulint ibd_table_flags) UNIV_NOTHROW - { - dberr_t err; - rec_format_t ibd_rec_format; - rec_format_t table_rec_format; - - if (!dict_tf_is_valid(ibd_table_flags)) { - - ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - ".ibd file has invalid table flags: %lx", - ibd_table_flags); - - return(DB_CORRUPTION); - } - - ibd_rec_format = dict_tf_get_rec_format(ibd_table_flags); - table_rec_format = dict_tf_get_rec_format(m_table->flags); - - if (table_rec_format != ibd_rec_format) { - - ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Table has %s row format, .ibd" - " file has %s row format.", - dict_tf_to_row_format_string(m_table->flags), - dict_tf_to_row_format_string(ibd_table_flags)); - - err = DB_CORRUPTION; - } else { - err = DB_SUCCESS; - } - - return(err); - } - /** Called for each block as it is read from the file. @param offset physical offset in the file @param block block to convert, it is not from the buffer pool. @@ -713,12 +681,17 @@ FetchIndexRootPages::operator() ( m_indexes.push_back(Index(id, block->page.id.page_no())); if (m_indexes.size() == 1) { - - m_table_flags = fsp_flags_to_dict_tf( - m_space_flags, - page_is_comp(page) ? true : false); - - err = check_row_format(m_table_flags); + /* Check that the tablespace flags match the table flags. */ + ulint expected = dict_tf_to_fsp_flags(m_table->flags); + if (!fsp_flags_match(expected, m_space_flags)) { + ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, + ER_TABLE_SCHEMA_MISMATCH, + "Expected FSP_SPACE_FLAGS=0x%x, .ibd " + "file contains 0x%x.", + unsigned(expected), + unsigned(m_space_flags)); + return(DB_CORRUPTION); + } } } @@ -1896,19 +1869,14 @@ PageConverter::update_header( ib::warn() << "Space id check in the header failed: ignored"; } - ulint space_flags = fsp_header_get_flags(get_frame(block)); - - if (!fsp_flags_is_valid(space_flags)) { - - ib::error() << "Unsupported tablespace format " - << space_flags; - - return(DB_UNSUPPORTED); - } - mach_write_to_8( get_frame(block) + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, m_current_lsn); + + /* Write back the adjusted flags. */ + mach_write_to_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + + get_frame(block), m_space_flags); + /* Write space_id to the tablespace header, page 0. */ mach_write_to_4( get_frame(block) + FSP_HEADER_OFFSET + FSP_SPACE_ID, diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 187dd06acf1..60b375dd4c5 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -3546,19 +3546,22 @@ This deletes the fil_space_t if found and the file on disk. @param[in] space_id Tablespace ID @param[in] tablename Table name, same as the tablespace name @param[in] filepath File path of tablespace to delete +@param[in] table_flags table flags @return error code or DB_SUCCESS */ UNIV_INLINE dberr_t row_drop_single_table_tablespace( ulint space_id, const char* tablename, - const char* filepath) + const char* filepath, + ulint table_flags) { dberr_t err = DB_SUCCESS; /* If the tablespace is not in the cache, just delete the file. */ if (!fil_space_for_table_exists_in_mem( - space_id, tablename, true, false, NULL, 0, NULL)) { + space_id, tablename, true, false, NULL, 0, NULL, + table_flags)) { /* Force a delete of any discarded or temporary files. */ fil_delete_file(filepath); @@ -4039,11 +4042,13 @@ row_drop_table_for_mysql( ulint space_id; bool ibd_file_missing; bool is_discarded; + ulint table_flags; case DB_SUCCESS: space_id = table->space; ibd_file_missing = table->ibd_file_missing; is_discarded = dict_table_is_discarded(table); + table_flags = table->flags; ut_ad(!dict_table_is_temporary(table)); err = row_drop_ancillary_fts_tables(table, trx); @@ -4079,7 +4084,7 @@ row_drop_table_for_mysql( /* We can now drop the single-table tablespace. */ err = row_drop_single_table_tablespace( - space_id, tablename, filepath); + space_id, tablename, filepath, table_flags); break; case DB_OUT_OF_FILE_SPACE: diff --git a/storage/innobase/row/row0trunc.cc b/storage/innobase/row/row0trunc.cc index 094c0b45e6d..1dea23bc657 100644 --- a/storage/innobase/row/row0trunc.cc +++ b/storage/innobase/row/row0trunc.cc @@ -904,7 +904,7 @@ TruncateLogger::operator()(mtr_t* mtr, btr_pcur_t* pcur) /* For compressed tables we need to store extra meta-data required during btr_create(). */ - if (fsp_flags_is_compressed(m_flags)) { + if (FSP_FLAGS_GET_ZIP_SSIZE(m_flags)) { const dict_index_t* dict_index = find(index.m_id); @@ -2581,7 +2581,7 @@ truncate_t::parse( ut_ad(!m_indexes.empty()); - if (fsp_flags_is_compressed(m_tablespace_flags)) { + if (FSP_FLAGS_GET_ZIP_SSIZE(m_tablespace_flags)) { /* Parse the number of index fields from TRUNCATE log record */ for (ulint i = 0; i < m_indexes.size(); ++i) { @@ -2880,12 +2880,12 @@ truncate_t::create_indexes( ++it) { btr_create_t btr_redo_create_info( - fsp_flags_is_compressed(flags) + FSP_FLAGS_GET_ZIP_SSIZE(flags) ? &it->m_fields[0] : NULL); btr_redo_create_info.format_flags = format_flags; - if (fsp_flags_is_compressed(flags)) { + if (FSP_FLAGS_GET_ZIP_SSIZE(flags)) { btr_redo_create_info.n_fields = it->m_n_fields; /* Skip the NUL appended field */ @@ -3020,7 +3020,7 @@ truncate_t::write( } /* If tablespace compressed then field info of each index. */ - if (fsp_flags_is_compressed(flags)) { + if (FSP_FLAGS_GET_ZIP_SSIZE(flags)) { for (ulint i = 0; i < m_indexes.size(); ++i) { diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index d0b7ddc0449..2b6946d70c1 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -443,12 +443,8 @@ create_log_files( has been completed and renamed. */ sprintf(logfilename + dirnamelen, "ib_logfile%u", INIT_LOG_FILE0); - /* Disable the doublewrite buffer for log files, not required */ - fil_space_t* log_space = fil_space_create( - "innodb_redo_log", SRV_LOG_SPACE_FIRST_ID, - fsp_flags_set_page_size(0, univ_page_size), - FIL_TYPE_LOG, + "innodb_redo_log", SRV_LOG_SPACE_FIRST_ID, 0, FIL_TYPE_LOG, NULL, /* No encryption yet */ true /* this is create */); ut_a(fil_validate()); @@ -645,7 +641,6 @@ srv_undo_tablespace_open( { os_file_t fh; bool ret; - ulint flags; dberr_t err = DB_ERROR; char undo_name[sizeof "innodb_undo000"]; @@ -690,11 +685,9 @@ srv_undo_tablespace_open( fil_set_max_space_id_if_bigger(space_id); - /* Set the compressed page size to 0 (non-compressed) */ - flags = fsp_flags_init( - univ_page_size, false, false, false, 0, 0); space = fil_space_create( - undo_name, space_id, flags, FIL_TYPE_TABLESPACE, NULL, true); + undo_name, space_id, FSP_FLAGS_PAGE_SSIZE(), + FIL_TYPE_TABLESPACE, NULL, true); ut_a(fil_validate()); ut_a(space); @@ -2035,8 +2028,7 @@ innobase_start_or_create_for_mysql(void) /* Disable the doublewrite buffer for log files. */ fil_space_t* log_space = fil_space_create( "innodb_redo_log", - SRV_LOG_SPACE_FIRST_ID, - fsp_flags_set_page_size(0, univ_page_size), + SRV_LOG_SPACE_FIRST_ID, 0, FIL_TYPE_LOG, NULL /* no encryption yet */, true /* create */); @@ -2425,6 +2417,13 @@ files_checked: srv_startup_is_before_trx_rollback_phase = false; if (!srv_read_only_mode) { + const ulint flags = FSP_FLAGS_PAGE_SSIZE(); + for (ulint id = 0; id <= srv_undo_tablespaces; id++) { + if (fil_space_get(id)) { + fsp_flags_try_adjust(id, flags); + } + } + /* Create the thread which watches the timeouts for lock waits */ thread_handles[2 + SRV_MAX_N_IO_THREADS] = os_thread_create( diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index 7f6b6caee9d..68bb83e4903 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -455,8 +455,11 @@ buf_dblwr_init_or_load_pages( os_file_write(path, file, page, source_page_no * UNIV_PAGE_SIZE, UNIV_PAGE_SIZE); - } else if (load_corrupt_pages) { - + } else if (load_corrupt_pages + && !buf_page_is_zeroes(page, FIL_PAGE_DATA)) { + /* Each valid page header must contain some + nonzero bytes, such as FIL_PAGE_OFFSET + or FIL_PAGE_LSN. */ recv_dblwr.add(page); } @@ -492,8 +495,6 @@ buf_dblwr_process() for (std::list<byte*>::iterator i = recv_dblwr.pages.begin(); i != recv_dblwr.pages.end(); ++i, ++page_no_dblwr ) { - bool is_compressed = false; - page = *i; page_no = mach_read_from_4(page + FIL_PAGE_OFFSET); space_id = mach_read_from_4(page + FIL_PAGE_SPACE_ID); @@ -501,138 +502,119 @@ buf_dblwr_process() if (!fil_tablespace_exists_in_mem(space_id)) { /* Maybe we have dropped the single-table tablespace and this page once belonged to it: do nothing */ + continue; + } - } else if (!fil_check_adress_in_tablespace(space_id, - page_no)) { + if (!fil_check_adress_in_tablespace(space_id, page_no)) { ib_logf(IB_LOG_LEVEL_WARN, - "A page in the doublewrite buffer is not " - "within space bounds; space id %lu " - "page number %lu, page %lu in " - "doublewrite buf.", - (ulong) space_id, (ulong) page_no, - page_no_dblwr); + "A copy of page " ULINTPF ":" ULINTPF + " in the doublewrite buffer slot " ULINTPF + " is not within space bounds", + space_id, page_no, page_no_dblwr); + continue; + } + + ulint zip_size = fil_space_get_zip_size(space_id); + ut_ad(!buf_page_is_zeroes(page, zip_size)); + + /* Read in the actual page from the file */ + fil_io(OS_FILE_READ, + true, + space_id, + zip_size, + page_no, + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + read_buf, + NULL, + 0); + + const bool is_all_zero = buf_page_is_zeroes( + read_buf, zip_size); + + if (is_all_zero) { + /* We will check if the copy in the + doublewrite buffer is valid. If not, we will + ignore this page (there should be redo log + records to initialize it). */ } else { - ulint zip_size = fil_space_get_zip_size(space_id); - - /* Read in the actual page from the file */ - fil_io(OS_FILE_READ, - true, - space_id, - zip_size, - page_no, - 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - read_buf, - NULL, - 0); - - /* Is page compressed ? */ - is_compressed = fil_page_is_compressed_encrypted(read_buf) | - fil_page_is_compressed(read_buf); - - /* If page was compressed, decompress it before we - check checksum. */ - if (is_compressed) { - fil_decompress_page(NULL, read_buf, UNIV_PAGE_SIZE, NULL, true); + if (fil_page_is_compressed_encrypted(read_buf) || + fil_page_is_compressed(read_buf)) { + /* Decompress the page before + validating the checksum. */ + fil_decompress_page( + NULL, read_buf, UNIV_PAGE_SIZE, + NULL, true); } - if (fil_space_verify_crypt_checksum(read_buf, zip_size)) { - /* page is encrypted and checksum is OK */ - } else if (buf_page_is_corrupted(true, read_buf, zip_size)) { - - fprintf(stderr, - "InnoDB: Database page" - " corruption or a failed\n" - "InnoDB: file read of" - " space %lu page %lu.\n" - "InnoDB: Trying to recover it from" - " the doublewrite buffer.\n", - (ulong) space_id, (ulong) page_no); - - /* Is page compressed ? */ - is_compressed = fil_page_is_compressed_encrypted(page) | - fil_page_is_compressed(page); - - /* If page was compressed, decompress it before we - check checksum. */ - if (is_compressed) { - fil_decompress_page(NULL, page, UNIV_PAGE_SIZE, NULL, true); - } + if (fil_space_verify_crypt_checksum( + read_buf, zip_size) + || !buf_page_is_corrupted( + true, read_buf, zip_size)) { + /* The page is good; there is no need + to consult the doublewrite buffer. */ + continue; + } - if (fil_space_verify_crypt_checksum(page, zip_size)) { - /* the doublewrite buffer page is encrypted and OK */ - } else if (buf_page_is_corrupted(true, - page, - zip_size)) { - fprintf(stderr, - "InnoDB: Dump of the page:\n"); - buf_page_print( - read_buf, zip_size, - BUF_PAGE_PRINT_NO_CRASH); - fprintf(stderr, - "InnoDB: Dump of" - " corresponding page" - " in doublewrite buffer:\n"); - buf_page_print( - page, zip_size, - BUF_PAGE_PRINT_NO_CRASH); - - fprintf(stderr, - "InnoDB: Also the page in the" - " doublewrite buffer" - " is corrupt.\n" - "InnoDB: Cannot continue" - " operation.\n" - "InnoDB: You can try to" - " recover the database" - " with the my.cnf\n" - "InnoDB: option:\n" - "InnoDB:" - " innodb_force_recovery=6\n"); - ut_error; - } + /* We intentionally skip this message for + is_all_zero pages. */ + ib_logf(IB_LOG_LEVEL_INFO, + "Trying to recover page " ULINTPF ":" ULINTPF + " from the doublewrite buffer.", + space_id, page_no); + } - /* Write the good page from the - doublewrite buffer to the intended - position */ - - fil_io(OS_FILE_WRITE, - true, - space_id, - zip_size, - page_no, - 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - page, - NULL, - 0); - - ib_logf(IB_LOG_LEVEL_INFO, - "Recovered the page from" - " the doublewrite buffer."); - - } else if (buf_page_is_zeroes(read_buf, zip_size)) { - - if (!buf_page_is_zeroes(page, zip_size) - && !buf_page_is_corrupted(true, page, - zip_size)) { - - /* Database page contained only - zeroes, while a valid copy is - available in dblwr buffer. */ - - fil_io(OS_FILE_WRITE, - true, - space_id, - zip_size, - page_no, 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - page, - NULL, - 0); - } + /* Next, validate the doublewrite page. */ + if (fil_page_is_compressed_encrypted(page) || + fil_page_is_compressed(page)) { + /* Decompress the page before + validating the checksum. */ + fil_decompress_page( + NULL, page, UNIV_PAGE_SIZE, NULL, true); + } + + if (!fil_space_verify_crypt_checksum(page, zip_size) + && buf_page_is_corrupted(true, page, zip_size)) { + if (!is_all_zero) { + ib_logf(IB_LOG_LEVEL_WARN, + "A doublewrite copy of page " + ULINTPF ":" ULINTPF " is corrupted.", + space_id, page_no); } + /* Theoretically we could have another good + copy for this page in the doublewrite + buffer. If not, we will report a fatal error + for a corrupted page somewhere else if that + page was truly needed. */ + continue; } + + if (page_no == 0) { + /* Check the FSP_SPACE_FLAGS. */ + ulint flags = fsp_header_get_flags(page); + if (!fsp_flags_is_valid(flags) + && fsp_flags_convert_from_101(flags) + == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_WARN, + "Ignoring a doublewrite copy of page " + ULINTPF ":0 due to invalid flags 0x%x", + space_id, int(flags)); + continue; + } + /* The flags on the page should be converted later. */ + } + + /* Write the good page from the doublewrite buffer to + the intended position. */ + + fil_io(OS_FILE_WRITE, true, space_id, zip_size, page_no, 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + page, NULL, 0); + + ib_logf(IB_LOG_LEVEL_INFO, + "Recovered page " ULINTPF ":" ULINTPF " from" + " the doublewrite buffer.", + space_id, page_no); } ut_free(unaligned_read_buf); diff --git a/storage/xtradb/dict/dict0load.cc b/storage/xtradb/dict/dict0load.cc index 13c4ac467e3..b843891f16c 100644 --- a/storage/xtradb/dict/dict0load.cc +++ b/storage/xtradb/dict/dict0load.cc @@ -1052,8 +1052,6 @@ loop: btr_pcur_store_position(&pcur, &mtr); - mtr_commit(&mtr); - /* For tables created with old versions of InnoDB, SYS_TABLES.MIX_LEN may contain garbage. Such tables would always be in ROW_FORMAT=REDUNDANT. Pretend that @@ -1087,16 +1085,19 @@ loop: if (space_id == 0) { /* The system tablespace always exists. */ ut_ad(!discarded); - goto next_tablespace; + mem_free(name); + goto loop; } + mtr_commit(&mtr); + switch (dict_check) { case DICT_CHECK_ALL_LOADED: /* All tablespaces should have been found in fil_load_single_table_tablespaces(). */ if (fil_space_for_table_exists_in_mem( - space_id, name, TRUE, !(is_temp || discarded), - false, NULL, 0) + space_id, name, !(is_temp || discarded), + false, NULL, 0, flags) && !(is_temp || discarded)) { /* If user changes the path of .ibd files in *.isl files before doing crash recovery , @@ -1128,8 +1129,8 @@ loop: /* Some tablespaces may have been opened in trx_resurrect_table_locks(). */ if (fil_space_for_table_exists_in_mem( - space_id, name, FALSE, FALSE, - false, NULL, 0)) { + space_id, name, false, + false, NULL, 0, flags)) { break; } /* fall through */ @@ -1191,7 +1192,6 @@ loop: max_space_id = space_id; } -next_tablespace: mem_free(name); mtr_start(&mtr); @@ -2383,8 +2383,8 @@ err_exit: table->ibd_file_missing = TRUE; } else if (!fil_space_for_table_exists_in_mem( - table->space, name, FALSE, FALSE, true, heap, - table->id)) { + table->space, name, false, true, heap, + table->id, table->flags)) { if (DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY)) { /* Do not bother to retry opening temporary tables. */ diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 133960ae8b4..f4301d47028 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -601,10 +601,7 @@ fil_node_open_file( ibool success; byte* buf2; byte* page; - ulint space_id; - ulint flags=0; ulint page_size; - ulint atomic_writes=0; ut_ad(mutex_own(&(system->mutex))); ut_a(node->n_pending == 0); @@ -626,8 +623,6 @@ fil_node_open_file( /* The following call prints an error message */ os_file_get_last_error(true); - ut_print_timestamp(stderr); - ib_logf(IB_LOG_LEVEL_WARN, "InnoDB: Error: cannot " "open %s\n. InnoDB: Have you deleted .ibd " "files under a running mysqld server?\n", @@ -653,17 +648,13 @@ fil_node_open_file( ut_a(fil_is_user_tablespace_id(space->id)); if (size_bytes < FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE) { - fprintf(stderr, - "InnoDB: Error: the size of single-table" - " tablespace file %s\n" - "InnoDB: is only " UINT64PF "," - " should be at least %lu!\n", - node->name, - size_bytes, - (ulong) (FIL_IBD_FILE_INITIAL_SIZE - * UNIV_PAGE_SIZE)); - - ut_a(0); + ib_logf(IB_LOG_LEVEL_ERROR, + "The size of the file %s is only " UINT64PF + " bytes, should be at least " ULINTPF, + node->name, size_bytes, + FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE); + os_file_close(node->handle); + return(false); } /* Read the first page of the tablespace */ @@ -676,77 +667,34 @@ fil_node_open_file( success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE); srv_stats.page0_read.add(1); - space_id = fsp_header_get_space_id(page); - flags = fsp_header_get_flags(page); - - page_size = fsp_flags_get_page_size(flags); - atomic_writes = fsp_flags_get_atomic_writes(flags); + const ulint space_id = fsp_header_get_space_id(page); + ulint flags = fsp_header_get_flags(page); ut_free(buf2); - - /* Close the file now that we have read the space id from it */ - os_file_close(node->handle); - if (UNIV_UNLIKELY(space_id != space->id)) { - fprintf(stderr, - "InnoDB: Error: tablespace id is %lu" - " in the data dictionary\n" - "InnoDB: but in file %s it is %lu!\n", - space->id, node->name, space_id); - - ut_error; - } - - if (UNIV_UNLIKELY(space_id == ULINT_UNDEFINED - || space_id == 0)) { - fprintf(stderr, - "InnoDB: Error: tablespace id %lu" - " in file %s is not sensible\n", - (ulong) space_id, node->name); - - ut_error; - } - - if (UNIV_UNLIKELY(fsp_flags_get_page_size(space->flags) - != page_size)) { - fprintf(stderr, - "InnoDB: Error: tablespace file %s" - " has page size 0x%lx\n" - "InnoDB: but the data dictionary" - " expects page size 0x%lx!\n", - node->name, flags, - fsp_flags_get_page_size(space->flags)); + if (!fsp_flags_is_valid(flags)) { + ulint cflags = fsp_flags_convert_from_101(flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Expected tablespace flags 0x%x" + " but found 0x%x in the file %s", + int(space->flags), int(flags), + node->name); + return(false); + } - ut_error; + flags = cflags; } - if (UNIV_UNLIKELY(space->flags != flags)) { - ulint sflags = (space->flags & ~FSP_FLAGS_MASK_DATA_DIR); - ulint fflags = (flags & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - - /* DATA_DIR option is on different place on MariaDB - compared to MySQL. If this is the difference. Fix - it. */ - - if (sflags == fflags) { - fprintf(stderr, - "InnoDB: Warning: Table flags 0x%lx" - " in the data dictionary but in file %s are 0x%lx!\n" - " Temporally corrected because DATA_DIR option to 0x%lx.\n", - space->flags, node->name, flags, space->flags); - - flags = space->flags; - } + page_size = fsp_flags_get_page_size(flags); - if (!dict_tf_verify_flags(space->flags, flags)) { - fprintf(stderr, - "InnoDB: Error: table flags are 0x%lx" - " in the data dictionary\n" - "InnoDB: but the flags in file %s are 0x%lx!\n", - space->flags, node->name, flags); - ut_error; - } + if (UNIV_UNLIKELY(space_id != space->id)) { + ib_logf(IB_LOG_LEVEL_ERROR, + "tablespace id is " ULINTPF " in the data dictionary" + " but in file %s it is " ULINTPF "!\n", + space->id, node->name, space_id); + return(false); } if (size_bytes >= (1024*1024)) { @@ -768,7 +716,7 @@ add_size: space->size += node->size; } - atomic_writes = fsp_flags_get_atomic_writes(space->flags); + ulint atomic_writes = fsp_flags_get_atomic_writes(space->flags); /* printf("Opening file %s\n", node->name); */ @@ -1572,7 +1520,6 @@ fil_space_create( fil_system->tablespace_version++; space->tablespace_version = fil_system->tablespace_version; - space->mark = FALSE; if (purpose == FIL_TABLESPACE && !recv_recovery_on && id > fil_system->max_assigned_id) { @@ -2324,27 +2271,21 @@ fil_write_flushed_lsn_to_data_files( return(DB_SUCCESS); } -/*******************************************************************//** -Checks the consistency of the first data page of a tablespace +/** Check the consistency of the first data page of a tablespace at database startup. +@param[in] page page frame +@param[in] space_id tablespace identifier +@param[in] flags tablespace flags @retval NULL on success, or if innodb_force_recovery is set @return pointer to an error message string */ static MY_ATTRIBUTE((warn_unused_result)) const char* -fil_check_first_page( -/*=================*/ - const page_t* page) /*!< in: data page */ +fil_check_first_page(const page_t* page, ulint space_id, ulint flags) { - ulint space_id; - ulint flags; - if (srv_force_recovery >= SRV_FORCE_IGNORE_CORRUPT) { return(NULL); } - space_id = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page); - flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); - if (UNIV_PAGE_SIZE != fsp_flags_get_page_size(flags)) { fprintf(stderr, "InnoDB: Error: Current page size %lu != " @@ -2393,7 +2334,7 @@ fil_read_first_page( ibool one_read_already, /*!< in: TRUE if min and max parameters below already contain sensible data */ - ulint* flags, /*!< out: tablespace flags */ + ulint* flags, /*!< out: FSP_SPACE_FLAGS */ ulint* space_id, /*!< out: tablespace ID */ lsn_t* min_flushed_lsn, /*!< out: min of flushed lsn values in data files */ @@ -2423,12 +2364,22 @@ fil_read_first_page( *flags and *space_id as they were read from the first file and do not validate the first page. */ if (!one_read_already) { - *flags = fsp_header_get_flags(page); *space_id = fsp_header_get_space_id(page); - } + *flags = fsp_header_get_flags(page); - if (!one_read_already) { - check_msg = fil_check_first_page(page); + if (!fsp_flags_is_valid(*flags)) { + ulint cflags = fsp_flags_convert_from_101(*flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Invalid flags 0x%x in tablespace %u", + unsigned(*flags), unsigned(*space_id)); + return "invalid tablespace flags"; + } else { + *flags = cflags; + } + } + + check_msg = fil_check_first_page(page, *space_id, *flags); } flushed_lsn = mach_read_from_8(page + @@ -2612,6 +2563,7 @@ fil_op_write_log( ulint len; log_ptr = mlog_open(mtr, 11 + 2 + 1); + ut_ad(fsp_flags_is_valid(flags)); if (!log_ptr) { /* Logging in mtr is switched off during crash recovery: @@ -3823,7 +3775,7 @@ fil_create_new_single_table_tablespace( ibool success; /* TRUE if a table is created with CREATE TEMPORARY TABLE */ bool is_temp = !!(flags2 & DICT_TF2_TEMPORARY); - bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags); + bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags) != 0; ulint atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(flags); fil_space_crypt_t *crypt_data = NULL; @@ -3831,7 +3783,7 @@ fil_create_new_single_table_tablespace( ut_ad(!srv_read_only_mode); ut_a(space_id < SRV_LOG_SPACE_FIRST_ID); ut_a(size >= FIL_IBD_FILE_INITIAL_SIZE); - ut_a(fsp_flags_is_valid(flags)); + ut_a(fsp_flags_is_valid(flags & ~FSP_FLAGS_MEM_MASK)); if (is_temp) { /* Temporary table filepath */ @@ -3924,12 +3876,9 @@ fil_create_new_single_table_tablespace( memset(page, '\0', UNIV_PAGE_SIZE); - /* Add the UNIV_PAGE_SIZE to the table flags and write them to the - tablespace header. */ - flags = fsp_flags_set_page_size(flags, UNIV_PAGE_SIZE); + flags |= FSP_FLAGS_PAGE_SSIZE(); fsp_header_init_fields(page, space_id, flags); mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id); - ut_ad(fsp_flags_is_valid(flags)); if (!(fsp_flags_is_compressed(flags))) { buf_flush_init_for_writing(page, NULL, 0); @@ -4008,7 +3957,8 @@ fil_create_new_single_table_tablespace( fil_op_write_log(flags ? MLOG_FILE_CREATE2 : MLOG_FILE_CREATE, - space_id, mlog_file_flag, flags, + space_id, mlog_file_flag, + flags & ~FSP_FLAGS_MEM_MASK, tablename, NULL, &mtr); mtr_commit(&mtr); @@ -4072,6 +4022,39 @@ fil_report_bad_tablespace( (ulong) expected_id, (ulong) expected_flags); } +/** Try to adjust FSP_SPACE_FLAGS if they differ from the expectations. +(Typically when upgrading from MariaDB 10.1.0..10.1.20.) +@param[in] space_id tablespace ID +@param[in] flags desired tablespace flags */ +UNIV_INTERN +void +fsp_flags_try_adjust(ulint space_id, ulint flags) +{ + ut_ad(!srv_read_only_mode); + ut_ad(fsp_flags_is_valid(flags)); + + mtr_t mtr; + mtr_start(&mtr); + if (buf_block_t* b = buf_page_get( + space_id, fsp_flags_get_zip_size(flags), 0, RW_X_LATCH, + &mtr)) { + ulint f = fsp_header_get_flags(b->frame); + /* Suppress the message if only the DATA_DIR flag to differs. */ + if ((f ^ flags) & ~(1U << FSP_FLAGS_POS_RESERVED)) { + ib_logf(IB_LOG_LEVEL_WARN, + "adjusting FSP_SPACE_FLAGS of tablespace " + ULINTPF " from 0x%x to 0x%x", + space_id, int(f), int(flags)); + } + if (f != flags) { + mlog_write_ulint(FSP_HEADER_OFFSET + + FSP_SPACE_FLAGS + b->frame, + flags, MLOG_4BYTES, &mtr); + } + } + mtr_commit(&mtr); +} + /********************************************************************//** Tries to open a single-table tablespace and optionally checks that the space id in it is correct. If this does not succeed, print an error message @@ -4101,7 +4084,7 @@ fil_open_single_table_tablespace( bool validate, /*!< in: Do we validate tablespace? */ bool fix_dict, /*!< in: Can we fix the dictionary? */ ulint id, /*!< in: space id */ - ulint flags, /*!< in: tablespace flags */ + ulint flags, /*!< in: expected FSP_SPACE_FLAGS */ const char* tablename, /*!< in: table name in the databasename/tablename format */ const char* path_in, /*!< in: tablespace filepath */ @@ -4126,20 +4109,13 @@ fil_open_single_table_tablespace( /* Table flags can be ULINT_UNDEFINED if dict_tf_to_fsp_flags_failure is set. */ - if (flags != ULINT_UNDEFINED) { - if (!fsp_flags_is_valid(flags)) { - return(DB_CORRUPTION); - } - } else { + if (flags == ULINT_UNDEFINED) { return(DB_CORRUPTION); } + ut_ad(fsp_flags_is_valid(flags & ~FSP_FLAGS_MEM_MASK)); atomic_writes = fsp_flags_get_atomic_writes(flags); - /* If the tablespace was relocated, we do not - compare the DATA_DIR flag */ - ulint mod_flags = flags & ~FSP_FLAGS_MASK_DATA_DIR; - memset(&def, 0, sizeof(def)); memset(&dict, 0, sizeof(dict)); memset(&remote, 0, sizeof(remote)); @@ -4217,31 +4193,17 @@ fil_open_single_table_tablespace( def.check_msg = fil_read_first_page( def.file, FALSE, &def.flags, &def.id, &def.lsn, &def.lsn, &def.crypt_data); - def.valid = !def.check_msg; if (table) { table->crypt_data = def.crypt_data; table->page_0_read = true; } - /* Validate this single-table-tablespace with SYS_TABLES, - but do not compare the DATA_DIR flag, in case the - tablespace was relocated. */ - - ulint newf = def.flags; - if (newf != mod_flags) { - if (FSP_FLAGS_HAS_DATA_DIR(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR); - } else if(FSP_FLAGS_HAS_DATA_DIR_ORACLE(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - } - } - - if (def.valid && def.id == id - && newf == mod_flags) { + def.valid = !def.check_msg && def.id == id + && fsp_flags_match(flags, def.flags); + if (def.valid) { valid_tablespaces_found++; } else { - def.valid = false; /* Do not use this tablespace. */ fil_report_bad_tablespace( def.filepath, def.check_msg, def.id, @@ -4254,30 +4216,18 @@ fil_open_single_table_tablespace( remote.check_msg = fil_read_first_page( remote.file, FALSE, &remote.flags, &remote.id, &remote.lsn, &remote.lsn, &remote.crypt_data); - remote.valid = !remote.check_msg; if (table) { table->crypt_data = remote.crypt_data; table->page_0_read = true; } - /* Validate this single-table-tablespace with SYS_TABLES, - but do not compare the DATA_DIR flag, in case the - tablespace was relocated. */ - ulint newf = remote.flags; - if (newf != mod_flags) { - if (FSP_FLAGS_HAS_DATA_DIR(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR); - } else if(FSP_FLAGS_HAS_DATA_DIR_ORACLE(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - } - } - - if (remote.valid && remote.id == id - && newf == mod_flags) { + /* Validate this single-table-tablespace with SYS_TABLES. */ + remote.valid = !remote.check_msg && remote.id == id + && fsp_flags_match(flags, remote.flags); + if (remote.valid) { valid_tablespaces_found++; } else { - remote.valid = false; /* Do not use this linked tablespace. */ fil_report_bad_tablespace( remote.filepath, remote.check_msg, remote.id, @@ -4291,30 +4241,19 @@ fil_open_single_table_tablespace( dict.check_msg = fil_read_first_page( dict.file, FALSE, &dict.flags, &dict.id, &dict.lsn, &dict.lsn, &dict.crypt_data); - dict.valid = !dict.check_msg; if (table) { table->crypt_data = dict.crypt_data; table->page_0_read = true; } - /* Validate this single-table-tablespace with SYS_TABLES, - but do not compare the DATA_DIR flag, in case the - tablespace was relocated. */ - ulint newf = dict.flags; - if (newf != mod_flags) { - if (FSP_FLAGS_HAS_DATA_DIR(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR); - } else if(FSP_FLAGS_HAS_DATA_DIR_ORACLE(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - } - } + /* Validate this single-table-tablespace with SYS_TABLES. */ + dict.valid = !dict.check_msg && dict.id == id + && fsp_flags_match(flags, dict.flags); - if (dict.valid && dict.id == id - && newf == mod_flags) { + if (dict.valid) { valid_tablespaces_found++; } else { - dict.valid = false; /* Do not use this tablespace. */ fil_report_bad_tablespace( dict.filepath, dict.check_msg, dict.id, @@ -4512,6 +4451,10 @@ cleanup_and_exit: mem_free(def.filepath); + if (err == DB_SUCCESS && !srv_read_only_mode) { + fsp_flags_try_adjust(id, flags & ~FSP_FLAGS_MEM_MASK); + } + return(err); } #endif /* !UNIV_HOTBACKUP */ @@ -4693,7 +4636,23 @@ fil_user_tablespace_restore_page( goto out; } - flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + + if (!fsp_flags_is_valid(flags)) { + ulint cflags = fsp_flags_convert_from_101(flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_WARN, + "Ignoring a doublewrite copy of page " + ULINTPF ":" ULINTPF + " due to invalid flags 0x%x", + fsp->id, page_no, int(flags)); + err = false; + goto out; + } + flags = cflags; + /* The flags on the page should be converted later. */ + } + zip_size = fsp_flags_get_zip_size(flags); page_size = fsp_flags_get_page_size(flags); @@ -5071,6 +5030,16 @@ will_not_choose: } mutex_exit(&fil_system->mutex); #endif /* UNIV_HOTBACKUP */ + /* Adjust the memory-based flags that would normally be set by + dict_tf_to_fsp_flags(). In recovery, we have no data dictionary. */ + if (FSP_FLAGS_HAS_PAGE_COMPRESSION(fsp->flags)) { + fsp->flags |= page_zip_level + << FSP_FLAGS_MEM_COMPRESSION_LEVEL; + } + remote.flags |= 1U << FSP_FLAGS_MEM_DATA_DIR; + /* We will leave atomic_writes at ATOMIC_WRITES_DEFAULT. + That will be adjusted in fil_space_for_table_exists_in_mem(). */ + ibool file_space_create_success = fil_space_create( tablename, fsp->id, fsp->flags, FIL_TABLESPACE, fsp->crypt_data, false); @@ -5380,13 +5349,12 @@ fil_report_missing_tablespace( name, space_id); } -/*******************************************************************//** -Returns TRUE if a matching tablespace exists in the InnoDB tablespace memory +/** Check if a matching tablespace exists in the InnoDB tablespace memory cache. Note that if we have not done a crash recovery at the database startup, there may be many tablespaces which are not yet in the memory cache. -@return TRUE if a matching tablespace exists in the memory cache */ +@return whether a matching tablespace exists in the memory cache */ UNIV_INTERN -ibool +bool fil_space_for_table_exists_in_mem( /*==============================*/ ulint id, /*!< in: space id */ @@ -5394,13 +5362,7 @@ fil_space_for_table_exists_in_mem( fil_space_create(). Either the standard 'dbname/tablename' format or table->dir_path_of_temp_table */ - ibool mark_space, /*!< in: in crash recovery, at database - startup we mark all spaces which have - an associated table in the InnoDB - data dictionary, so that - we can print a warning about orphaned - tablespaces */ - ibool print_error_if_does_not_exist, + bool print_error_if_does_not_exist, /*!< in: print detailed error information to the .err log if a matching tablespace is not found from @@ -5408,12 +5370,13 @@ fil_space_for_table_exists_in_mem( bool adjust_space, /*!< in: whether to adjust space id when find table space mismatch */ mem_heap_t* heap, /*!< in: heap memory */ - table_id_t table_id) /*!< in: table id */ + table_id_t table_id, /*!< in: table id */ + ulint table_flags) /*!< in: table flags */ { fil_space_t* fnamespace; fil_space_t* space; - ut_ad(fil_system); + const ulint expected_flags = dict_tf_to_fsp_flags(table_flags); mutex_enter(&fil_system->mutex); @@ -5425,42 +5388,31 @@ fil_space_for_table_exists_in_mem( directory path from the datadir to the file */ fnamespace = fil_space_get_by_name(name); - if (space && space == fnamespace) { - /* Found */ - - if (mark_space) { - space->mark = TRUE; - } - - mutex_exit(&fil_system->mutex); - - return(TRUE); - } - - /* Info from "fnamespace" comes from the ibd file itself, it can - be different from data obtained from System tables since it is - not transactional. If adjust_space is set, and the mismatching - space are between a user table and its temp table, we shall - adjust the ibd file name according to system table info */ - if (adjust_space - && space != NULL - && row_is_mysql_tmp_table_name(space->name) - && !row_is_mysql_tmp_table_name(name)) { + bool valid = space && !((space->flags ^ expected_flags) + & ~FSP_FLAGS_MEM_MASK); + if (!space) { + } else if (!valid || space == fnamespace) { + /* Found with the same file name, or got a flag mismatch. */ + goto func_exit; + } else if (adjust_space + && row_is_mysql_tmp_table_name(space->name) + && !row_is_mysql_tmp_table_name(name)) { + /* Info from fnamespace comes from the ibd file + itself, it can be different from data obtained from + System tables since renaming files is not + transactional. We shall adjust the ibd file name + according to system table info. */ mutex_exit(&fil_system->mutex); DBUG_EXECUTE_IF("ib_crash_before_adjust_fil_space", DBUG_SUICIDE();); - if (fnamespace) { - char* tmp_name; + char* tmp_name = dict_mem_create_temporary_tablename( + heap, name, table_id); - tmp_name = dict_mem_create_temporary_tablename( - heap, name, table_id); - - fil_rename_tablespace(fnamespace->name, fnamespace->id, - tmp_name, NULL); - } + fil_rename_tablespace(fnamespace->name, fnamespace->id, + tmp_name, NULL); DBUG_EXECUTE_IF("ib_crash_after_adjust_one_fil_space", DBUG_SUICIDE();); @@ -5473,16 +5425,12 @@ fil_space_for_table_exists_in_mem( mutex_enter(&fil_system->mutex); fnamespace = fil_space_get_by_name(name); ut_ad(space == fnamespace); - mutex_exit(&fil_system->mutex); - - return(TRUE); + goto func_exit; } if (!print_error_if_does_not_exist) { - - mutex_exit(&fil_system->mutex); - - return(FALSE); + valid = false; + goto func_exit; } if (space == NULL) { @@ -5509,10 +5457,8 @@ error_exit: fputs("InnoDB: Please refer to\n" "InnoDB: " REFMAN "innodb-troubleshooting-datadict.html\n" "InnoDB: for how to resolve the issue.\n", stderr); - - mutex_exit(&fil_system->mutex); - - return(FALSE); + valid = false; + goto func_exit; } if (0 != strcmp(space->name, name)) { @@ -5539,9 +5485,19 @@ error_exit: goto error_exit; } +func_exit: + if (valid) { + /* Adjust the flags that are in FSP_FLAGS_MEM_MASK. + FSP_SPACE_FLAGS will not be written back here. */ + space->flags = expected_flags; + } mutex_exit(&fil_system->mutex); - return(FALSE); + if (valid && !srv_read_only_mode) { + fsp_flags_try_adjust(id, expected_flags & ~FSP_FLAGS_MEM_MASK); + } + + return(valid); } /*******************************************************************//** @@ -7444,9 +7400,8 @@ fil_space_get_crypt_data( byte *page = static_cast<byte*>(ut_align(buf, UNIV_PAGE_SIZE)); fil_read(true, space_id, 0, 0, 0, UNIV_PAGE_SIZE, page, NULL, NULL); - ulint flags = fsp_header_get_flags(page); ulint offset = fsp_header_get_crypt_offset( - fsp_flags_get_zip_size(flags), NULL); + fsp_header_get_zip_size(page), NULL); space->crypt_data = fil_space_read_crypt_data(space_id, page, offset); ut_free(buf); diff --git a/storage/xtradb/fsp/fsp0fsp.cc b/storage/xtradb/fsp/fsp0fsp.cc index 4acfd134c1f..c32fddaabbe 100644 --- a/storage/xtradb/fsp/fsp0fsp.cc +++ b/storage/xtradb/fsp/fsp0fsp.cc @@ -663,6 +663,7 @@ fsp_header_init_fields( ulint space_id, /*!< in: space id */ ulint flags) /*!< in: tablespace flags (FSP_SPACE_FLAGS) */ { + flags &= ~FSP_FLAGS_MEM_MASK; ut_a(fsp_flags_is_valid(flags)); mach_write_to_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page, @@ -713,7 +714,7 @@ fsp_header_init( mlog_write_ulint(header + FSP_SIZE, size, MLOG_4BYTES, mtr); mlog_write_ulint(header + FSP_FREE_LIMIT, 0, MLOG_4BYTES, mtr); - mlog_write_ulint(header + FSP_SPACE_FLAGS, flags, + mlog_write_ulint(header + FSP_SPACE_FLAGS, flags & ~FSP_FLAGS_MEM_MASK, MLOG_4BYTES, mtr); mlog_write_ulint(header + FSP_FRAG_N_USED, 0, MLOG_4BYTES, mtr); diff --git a/storage/xtradb/include/dict0dict.ic b/storage/xtradb/include/dict0dict.ic index d9d40b809d1..2b63ddea51d 100644 --- a/storage/xtradb/include/dict0dict.ic +++ b/storage/xtradb/include/dict0dict.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -918,10 +918,13 @@ dict_tf_to_fsp_flags( ulint table_flags) /*!< in: dict_table_t::flags */ { ulint fsp_flags; - ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); - ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); + ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL( + table_flags); ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); + ut_ad((DICT_TF_GET_PAGE_COMPRESSION(table_flags) == 0) + == (page_compression_level == 0)); + DBUG_EXECUTE_IF("dict_tf_to_fsp_flags_failure", return(ULINT_UNDEFINED);); @@ -929,30 +932,23 @@ dict_tf_to_fsp_flags( fsp_flags = DICT_TF_HAS_ATOMIC_BLOBS(table_flags) ? 1 : 0; /* ZIP_SSIZE and ATOMIC_BLOBS are at the same position. */ - fsp_flags |= table_flags & DICT_TF_MASK_ZIP_SSIZE; - fsp_flags |= table_flags & DICT_TF_MASK_ATOMIC_BLOBS; - - /* In addition, tablespace flags also contain the page size. */ - fsp_flags |= fsp_flags_set_page_size(fsp_flags, UNIV_PAGE_SIZE); + fsp_flags |= table_flags + & (DICT_TF_MASK_ZIP_SSIZE | DICT_TF_MASK_ATOMIC_BLOBS); - /* The DATA_DIR flag is in a different position in fsp_flag */ - fsp_flags |= DICT_TF_HAS_DATA_DIR(table_flags) - ? FSP_FLAGS_MASK_DATA_DIR : 0; + fsp_flags |= FSP_FLAGS_PAGE_SSIZE(); - /* In addition, tablespace flags also contain if the page - compression is used for this table. */ - fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION(fsp_flags, page_compression); + if (page_compression_level) { + fsp_flags |= FSP_FLAGS_MASK_PAGE_COMPRESSION; + } - /* In addition, tablespace flags also contain page compression level - if page compression is used for this table. */ - fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(fsp_flags, page_compression_level); + ut_a(fsp_flags_is_valid(fsp_flags)); - /* In addition, tablespace flags also contain flag if atomic writes - is used for this table */ - fsp_flags |= FSP_FLAGS_SET_ATOMIC_WRITES(fsp_flags, atomic_writes); + if (DICT_TF_HAS_DATA_DIR(table_flags)) { + fsp_flags |= 1U << FSP_FLAGS_MEM_DATA_DIR; + } - ut_a(fsp_flags_is_valid(fsp_flags)); - ut_a(dict_tf_verify_flags(table_flags, fsp_flags)); + fsp_flags |= atomic_writes << FSP_FLAGS_MEM_ATOMIC_WRITES; + fsp_flags |= page_compression_level << FSP_FLAGS_MEM_COMPRESSION_LEVEL; return(fsp_flags); } diff --git a/storage/xtradb/include/dict0pagecompress.h b/storage/xtradb/include/dict0pagecompress.h index 19a2a6c52f3..6503c86ffa2 100644 --- a/storage/xtradb/include/dict0pagecompress.h +++ b/storage/xtradb/include/dict0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013 SkySQL Ab. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -57,17 +57,6 @@ dict_table_page_compression_level( __attribute__((const)); /********************************************************************//** -Verify that dictionary flags match tablespace flags -@return true if flags match, false if not */ -UNIV_INLINE -ibool -dict_tf_verify_flags( -/*=================*/ - ulint table_flags, /*!< in: dict_table_t::flags */ - ulint fsp_flags) /*!< in: fil_space_t::flags */ - __attribute__((const)); - -/********************************************************************//** Extract the atomic writes flag from table flags. @return true if atomic writes are used, false if not used */ UNIV_INLINE diff --git a/storage/xtradb/include/dict0pagecompress.ic b/storage/xtradb/include/dict0pagecompress.ic index 811976434a8..13c2b46c51c 100644 --- a/storage/xtradb/include/dict0pagecompress.ic +++ b/storage/xtradb/include/dict0pagecompress.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013 SkySQL Ab. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -25,92 +25,6 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com ***********************************************************************/ /********************************************************************//** -Verify that dictionary flags match tablespace flags -@return true if flags match, false if not */ -UNIV_INLINE -ibool -dict_tf_verify_flags( -/*=================*/ - ulint table_flags, /*!< in: dict_table_t::flags */ - ulint fsp_flags) /*!< in: fil_space_t::flags */ -{ - ulint table_unused = DICT_TF_GET_UNUSED(table_flags); - ulint compact = DICT_TF_GET_COMPACT(table_flags); - ulint ssize = DICT_TF_GET_ZIP_SSIZE(table_flags); - ulint atomic_blobs = DICT_TF_HAS_ATOMIC_BLOBS(table_flags); - ulint data_dir = DICT_TF_HAS_DATA_DIR(table_flags); - ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); - ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); - ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); - ulint post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(fsp_flags); - ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(fsp_flags); - ulint fsp_atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(fsp_flags); - ulint page_ssize = FSP_FLAGS_GET_PAGE_SSIZE(fsp_flags); - ulint fsp_unused = FSP_FLAGS_GET_UNUSED(fsp_flags); - ulint fsp_page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(fsp_flags); - ulint fsp_page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(fsp_flags); - ulint fsp_atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(fsp_flags); - - DBUG_EXECUTE_IF("dict_tf_verify_flags_failure", - return(ULINT_UNDEFINED);); - - ut_a(!table_unused); - ut_a(!fsp_unused); - ut_a(page_ssize == 0 || page_ssize != 0); /* silence compiler */ - ut_a(compact == 0 || compact == 1); /* silence compiler */ - ut_a(data_dir == 0 || data_dir == 1); /* silence compiler */ - ut_a(post_antelope == 0 || post_antelope == 1); /* silence compiler */ - - if (ssize != zip_ssize) { - fprintf(stderr, - "InnoDB: Error: table flags has zip_ssize %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has zip_ssize %ld\n", - ssize, zip_ssize); - return (FALSE); - } - if (atomic_blobs != fsp_atomic_blobs) { - fprintf(stderr, - "InnoDB: Error: table flags has atomic_blobs %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has atomic_blobs %ld\n", - atomic_blobs, fsp_atomic_blobs); - - return (FALSE); - } - if (page_compression != fsp_page_compression) { - fprintf(stderr, - "InnoDB: Error: table flags has page_compression %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file ahas page_compression %ld\n", - page_compression, fsp_page_compression); - - return (FALSE); - } - if (page_compression_level != fsp_page_compression_level) { - fprintf(stderr, - "InnoDB: Error: table flags has page_compression_level %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has page_compression_level %ld\n", - page_compression_level, fsp_page_compression_level); - - return (FALSE); - } - - if (atomic_writes != fsp_atomic_writes) { - fprintf(stderr, - "InnoDB: Error: table flags has atomic writes %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has atomic_writes %ld\n", - atomic_writes, fsp_atomic_writes); - - return (FALSE); - } - - return(TRUE); -} - -/********************************************************************//** Extract the page compression level from dict_table_t::flags. These flags are in memory, so assert that they are valid. @return page compression level, or 0 if not compressed */ diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 38cc09bced3..41e38794ea9 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -45,7 +45,6 @@ Created 10/25/1995 Heikki Tuuri // Forward declaration struct trx_t; -struct fil_space_t; typedef std::list<const char*> space_name_list_t; @@ -264,10 +263,6 @@ struct fil_space_t { an insert buffer merge request for a page because it actually was for the previous incarnation of the space */ - ibool mark; /*!< this is set to TRUE at database startup if - the space corresponds to a table in the InnoDB - data dictionary; so we can print a warning of - orphaned tablespaces */ ibool stop_ios;/*!< TRUE if we want to rename the .ibd file of tablespace and want to stop temporarily posting of new i/o @@ -296,7 +291,8 @@ struct fil_space_t { /*!< recovered tablespace size in pages; 0 if no size change was read from the redo log, or if the size change was implemented */ - ulint flags; /*!< tablespace flags; see + ulint flags; /*!< FSP_SPACE_FLAGS and FSP_FLAGS_MEM_ flags; + see fsp0fsp.h, fsp_flags_is_valid(), fsp_flags_get_zip_size() */ ulint n_reserved_extents; @@ -449,6 +445,7 @@ fil_node_create( ibool is_raw) /*!< in: TRUE if a raw device or a raw disk partition */ MY_ATTRIBUTE((nonnull, warn_unused_result)); + #ifdef UNIV_LOG_ARCHIVE /****************************************************************//** Drops files from the start of a file space, so that its size is cut by @@ -620,7 +617,7 @@ fil_read_first_page( ibool one_read_already, /*!< in: TRUE if min and max parameters below already contain sensible data */ - ulint* flags, /*!< out: tablespace flags */ + ulint* flags, /*!< out: FSP_SPACE_FLAGS */ ulint* space_id, /*!< out: tablespace ID */ lsn_t* min_flushed_lsn, /*!< out: min of flushed lsn values in data files */ @@ -832,6 +829,14 @@ fil_create_new_single_table_tablespace( ulint key_id) /*!< in: encryption key_id */ __attribute__((nonnull, warn_unused_result)); #ifndef UNIV_HOTBACKUP +/** Try to adjust FSP_SPACE_FLAGS if they differ from the expectations. +(Typically when upgrading from MariaDB 10.1.0..10.1.20.) +@param[in] space_id tablespace ID +@param[in] flags desired tablespace flags */ +UNIV_INTERN +void +fsp_flags_try_adjust(ulint space_id, ulint flags); + /********************************************************************//** Tries to open a single-table tablespace and optionally checks the space id is right in it. If does not succeed, prints an error message to the .err log. This @@ -860,7 +865,7 @@ fil_open_single_table_tablespace( bool validate, /*!< in: Do we validate tablespace? */ bool fix_dict, /*!< in: Can we fix the dictionary? */ ulint id, /*!< in: space id */ - ulint flags, /*!< in: tablespace flags */ + ulint flags, /*!< in: expected FSP_SPACE_FLAGS */ const char* tablename, /*!< in: table name in the databasename/tablename format */ const char* filepath, /*!< in: tablespace filepath */ @@ -901,25 +906,18 @@ fil_tablespace_exists_in_mem( /*=========================*/ ulint id); /*!< in: space id */ #ifndef UNIV_HOTBACKUP -/*******************************************************************//** -Returns TRUE if a matching tablespace exists in the InnoDB tablespace memory +/** Check if a matching tablespace exists in the InnoDB tablespace memory cache. Note that if we have not done a crash recovery at the database startup, there may be many tablespaces which are not yet in the memory cache. -@return TRUE if a matching tablespace exists in the memory cache */ +@return whether a matching tablespace exists in the memory cache */ UNIV_INTERN -ibool +bool fil_space_for_table_exists_in_mem( /*==============================*/ ulint id, /*!< in: space id */ const char* name, /*!< in: table name in the standard 'databasename/tablename' format */ - ibool mark_space, /*!< in: in crash recovery, at database - startup we mark all spaces which have - an associated table in the InnoDB - data dictionary, so that - we can print a warning about orphaned - tablespaces */ - ibool print_error_if_does_not_exist, + bool print_error_if_does_not_exist, /*!< in: print detailed error information to the .err log if a matching tablespace is not found from @@ -927,7 +925,8 @@ fil_space_for_table_exists_in_mem( bool adjust_space, /*!< in: whether to adjust space id when find table space mismatch */ mem_heap_t* heap, /*!< in: heap memory */ - table_id_t table_id); /*!< in: table id */ + table_id_t table_id, /*!< in: table id */ + ulint table_flags); /*!< in: table flags */ #else /* !UNIV_HOTBACKUP */ /********************************************************************//** Extends all tablespaces to the size stored in the space header. During the diff --git a/storage/xtradb/include/fil0pagecompress.h b/storage/xtradb/include/fil0pagecompress.h index 10db59fb218..1fe5cb66bf6 100644 --- a/storage/xtradb/include/fil0pagecompress.h +++ b/storage/xtradb/include/fil0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2016 MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017 MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -44,20 +44,11 @@ Returns the page compression flag of the space, or false if the space is not compressed. The tablespace must be cached in the memory cache. @return true if page compressed, false if not or space not found */ UNIV_INLINE -ibool +bool fil_space_is_page_compressed( /*=========================*/ ulint id); /*!< in: space id */ /*******************************************************************//** -Returns the page compression flag of the space, or false if the space -is not compressed. The tablespace must be cached in the memory cache. -@return true if page compressed, false if not or space not found */ -UNIV_INTERN -ibool -fil_space_get_page_compressed( -/*=========================*/ - fil_space_t* space); /*!< in: space id */ -/*******************************************************************//** Returns the atomic writes flag of the space, or false if the space is not using atomic writes. The tablespace must be cached in the memory cache. @return atomic write table option value */ diff --git a/storage/xtradb/include/fsp0fsp.h b/storage/xtradb/include/fsp0fsp.h index 551a8c9ef97..93f98eb0b0b 100644 --- a/storage/xtradb/include/fsp0fsp.h +++ b/storage/xtradb/include/fsp0fsp.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -51,28 +51,67 @@ to the two Barracuda row formats COMPRESSED and DYNAMIC. */ #define FSP_FLAGS_WIDTH_ATOMIC_BLOBS 1 /** Number of flag bits used to indicate the tablespace page size */ #define FSP_FLAGS_WIDTH_PAGE_SSIZE 4 -/** Width of the DATA_DIR flag. This flag indicates that the tablespace -is found in a remote location, not the default data directory. */ -#define FSP_FLAGS_WIDTH_DATA_DIR 1 -/** Number of flag bits used to indicate the page compression and compression level */ -#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION 1 -#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL 4 +/** Number of reserved bits */ +#define FSP_FLAGS_WIDTH_RESERVED 6 +/** Number of flag bits used to indicate the page compression */ +#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION 1 -/** Number of flag bits used to indicate atomic writes for this tablespace */ -#define FSP_FLAGS_WIDTH_ATOMIC_WRITES 2 - -/** Width of all the currently known tablespace flags */ +/** Width of all the currently known persistent tablespace flags */ #define FSP_FLAGS_WIDTH (FSP_FLAGS_WIDTH_POST_ANTELOPE \ + FSP_FLAGS_WIDTH_ZIP_SSIZE \ + FSP_FLAGS_WIDTH_ATOMIC_BLOBS \ + FSP_FLAGS_WIDTH_PAGE_SSIZE \ - + FSP_FLAGS_WIDTH_DATA_DIR \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES ) - -/** A mask of all the known/used bits in tablespace flags */ -#define FSP_FLAGS_MASK (~(~0 << FSP_FLAGS_WIDTH)) + + FSP_FLAGS_WIDTH_RESERVED \ + + FSP_FLAGS_WIDTH_PAGE_COMPRESSION) + +/** A mask of all the known/used bits in FSP_SPACE_FLAGS */ +#define FSP_FLAGS_MASK (~(~0U << FSP_FLAGS_WIDTH)) + +/* FSP_SPACE_FLAGS position and name in MySQL 5.6/MariaDB 10.0 or older +and MariaDB 10.1.20 or older MariaDB 10.1 and in MariaDB 10.1.21 +or newer. +MySQL 5.6 MariaDB 10.1.x MariaDB 10.1.21 +==================================================================== +Below flags in same offset +==================================================================== +0: POST_ANTELOPE 0:POST_ANTELOPE 0: POST_ANTELOPE +1..4: ZIP_SSIZE(0..5) 1..4:ZIP_SSIZE(0..5) 1..4: ZIP_SSIZE(0..5) +(NOTE: bit 4 is always 0) +5: ATOMIC_BLOBS 5:ATOMIC_BLOBS 5: ATOMIC_BLOBS +===================================================================== +Below note the order difference: +===================================================================== +6..9: PAGE_SSIZE(3..7) 6: COMPRESSION 6..9: PAGE_SSIZE(3..7) +10: DATA_DIR 7..10: COMP_LEVEL(0..9) 10: RESERVED (5.6 DATA_DIR) +===================================================================== +The flags below were in incorrect position in MariaDB 10.1, +or have been introduced in MySQL 5.7 or 8.0: +===================================================================== +11: UNUSED 11..12:ATOMIC_WRITES 11: RESERVED (5.7 SHARED) + 12: RESERVED (5.7 TEMPORARY) + 13..15:PAGE_SSIZE(3..7) 13: RESERVED (5.7 ENCRYPTION) + 14: RESERVED (8.0 SDI) + 15: RESERVED + 16: PAGE_SSIZE_msb(0) 16: COMPRESSION + 17: DATA_DIR 17: UNUSED + 18: UNUSED +===================================================================== +The flags below only exist in fil_space_t::flags, not in FSP_SPACE_FLAGS: +===================================================================== + 25: DATA_DIR + 26..27: ATOMIC_WRITES + 28..31: COMPRESSION_LEVEL +*/ + +/** A mask of the memory-only flags in fil_space_t::flags */ +#define FSP_FLAGS_MEM_MASK (~0U << FSP_FLAGS_MEM_DATA_DIR) + +/** Zero relative shift position of the DATA_DIR flag */ +#define FSP_FLAGS_MEM_DATA_DIR 25 +/** Zero relative shift position of the ATOMIC_WRITES field */ +#define FSP_FLAGS_MEM_ATOMIC_WRITES 26 +/** Zero relative shift position of the COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MEM_COMPRESSION_LEVEL 28 /** Zero relative shift position of the POST_ANTELOPE field */ #define FSP_FLAGS_POS_POST_ANTELOPE 0 @@ -82,29 +121,16 @@ is found in a remote location, not the default data directory. */ /** Zero relative shift position of the ATOMIC_BLOBS field */ #define FSP_FLAGS_POS_ATOMIC_BLOBS (FSP_FLAGS_POS_ZIP_SSIZE \ + FSP_FLAGS_WIDTH_ZIP_SSIZE) -/** Note that these need to be before the page size to be compatible with -dictionary */ -/** Zero relative shift position of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_POS_PAGE_COMPRESSION (FSP_FLAGS_POS_ATOMIC_BLOBS \ - + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) -/** Zero relative shift position of the PAGE_COMPRESSION_LEVEL field */ -#define FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL (FSP_FLAGS_POS_PAGE_COMPRESSION \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION) -/** Zero relative shift position of the ATOMIC_WRITES field */ -#define FSP_FLAGS_POS_ATOMIC_WRITES (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL) -/** Zero relative shift position of the PAGE_SSIZE field */ -#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_ATOMIC_WRITES \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES) -/** Zero relative shift position of the start of the DATA DIR bits */ -#define FSP_FLAGS_POS_DATA_DIR (FSP_FLAGS_POS_PAGE_SSIZE \ - + FSP_FLAGS_WIDTH_PAGE_SSIZE) -#define FSP_FLAGS_POS_DATA_DIR_ORACLE (FSP_FLAGS_POS_ATOMIC_BLOBS \ - + FSP_FLAGS_WIDTH_ATOMIC_BLOBS \ +/** Zero relative shift position of the start of the PAGE_SSIZE bits */ +#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_ATOMIC_BLOBS \ + + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) +/** Zero relative shift position of the start of the RESERVED bits +these are only used in MySQL 5.7 and used for compatibility. */ +#define FSP_FLAGS_POS_RESERVED (FSP_FLAGS_POS_PAGE_SSIZE \ + FSP_FLAGS_WIDTH_PAGE_SSIZE) -/** Zero relative shift position of the start of the UNUSED bits */ -#define FSP_FLAGS_POS_UNUSED (FSP_FLAGS_POS_DATA_DIR\ - + FSP_FLAGS_WIDTH_DATA_DIR) +/** Zero relative shift position of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_POS_PAGE_COMPRESSION (FSP_FLAGS_POS_RESERVED \ + + FSP_FLAGS_WIDTH_RESERVED) /** Bit mask of the POST_ANTELOPE field */ #define FSP_FLAGS_MASK_POST_ANTELOPE \ @@ -122,26 +148,23 @@ dictionary */ #define FSP_FLAGS_MASK_PAGE_SSIZE \ ((~(~0U << FSP_FLAGS_WIDTH_PAGE_SSIZE)) \ << FSP_FLAGS_POS_PAGE_SSIZE) -/** Bit mask of the DATA_DIR field */ -#define FSP_FLAGS_MASK_DATA_DIR \ - ((~(~0U << FSP_FLAGS_WIDTH_DATA_DIR)) \ - << FSP_FLAGS_POS_DATA_DIR) -/** Bit mask of the DATA_DIR field */ -#define FSP_FLAGS_MASK_DATA_DIR_ORACLE \ - ((~(~0U << FSP_FLAGS_WIDTH_DATA_DIR)) \ - << FSP_FLAGS_POS_DATA_DIR_ORACLE) +/** Bit mask of the RESERVED1 field */ +#define FSP_FLAGS_MASK_RESERVED \ + ((~(~0U << FSP_FLAGS_WIDTH_RESERVED)) \ + << FSP_FLAGS_POS_RESERVED) /** Bit mask of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_MASK_PAGE_COMPRESSION \ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION \ ((~(~0U << FSP_FLAGS_WIDTH_PAGE_COMPRESSION)) \ << FSP_FLAGS_POS_PAGE_COMPRESSION) -/** Bit mask of the PAGE_COMPRESSION_LEVEL field */ -#define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL \ - ((~(~0U << FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL)) \ - << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) -/** Bit mask of the ATOMIC_WRITES field */ -#define FSP_FLAGS_MASK_ATOMIC_WRITES \ - ((~(~0U << FSP_FLAGS_WIDTH_ATOMIC_WRITES)) \ - << FSP_FLAGS_POS_ATOMIC_WRITES) + +/** Bit mask of the in-memory ATOMIC_WRITES field */ +#define FSP_FLAGS_MASK_MEM_ATOMIC_WRITES \ + (3U << FSP_FLAGS_MEM_ATOMIC_WRITES) + +/** Bit mask of the in-memory COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MASK_MEM_COMPRESSION_LEVEL \ + (15U << FSP_FLAGS_MEM_COMPRESSION_LEVEL) + /** Return the value of the POST_ANTELOPE field */ #define FSP_FLAGS_GET_POST_ANTELOPE(flags) \ ((flags & FSP_FLAGS_MASK_POST_ANTELOPE) \ @@ -158,48 +181,78 @@ dictionary */ #define FSP_FLAGS_GET_PAGE_SSIZE(flags) \ ((flags & FSP_FLAGS_MASK_PAGE_SSIZE) \ >> FSP_FLAGS_POS_PAGE_SSIZE) -/** Return the value of the DATA_DIR field */ -#define FSP_FLAGS_HAS_DATA_DIR(flags) \ - ((flags & FSP_FLAGS_MASK_DATA_DIR) \ - >> FSP_FLAGS_POS_DATA_DIR) -#define FSP_FLAGS_HAS_DATA_DIR_ORACLE(flags) \ - ((flags & FSP_FLAGS_MASK_DATA_DIR_ORACLE) \ - >> FSP_FLAGS_POS_DATA_DIR_ORACLE) +/** @return the RESERVED flags */ +#define FSP_FLAGS_GET_RESERVED(flags) \ + ((flags & FSP_FLAGS_MASK_RESERVED) \ + >> FSP_FLAGS_POS_RESERVED) +/** @return the PAGE_COMPRESSION flag */ +#define FSP_FLAGS_HAS_PAGE_COMPRESSION(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION) + /** Return the contents of the UNUSED bits */ #define FSP_FLAGS_GET_UNUSED(flags) \ (flags >> FSP_FLAGS_POS_UNUSED) + +/** @return the PAGE_SSIZE flags for the current innodb_page_size */ +#define FSP_FLAGS_PAGE_SSIZE() \ + ((UNIV_PAGE_SIZE == UNIV_PAGE_SIZE_ORIG) ? \ + 0 : (UNIV_PAGE_SIZE_SHIFT - UNIV_ZIP_SIZE_SHIFT_MIN + 1) \ + << FSP_FLAGS_POS_PAGE_SSIZE) + +/** @return the value of the DATA_DIR field */ +#define FSP_FLAGS_HAS_DATA_DIR(flags) \ + (flags & 1U << FSP_FLAGS_MEM_DATA_DIR) +/** @return the COMPRESSION_LEVEL field */ +#define FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags) \ + ((flags & FSP_FLAGS_MASK_MEM_COMPRESSION_LEVEL) \ + >> FSP_FLAGS_MEM_COMPRESSION_LEVEL) +/** @return the ATOMIC_WRITES field */ +#define FSP_FLAGS_GET_ATOMIC_WRITES(flags) \ + ((flags & FSP_FLAGS_MASK_MEM_ATOMIC_WRITES) \ + >> FSP_FLAGS_MEM_ATOMIC_WRITES) + +/* Compatibility macros for MariaDB 10.1.20 or older 10.1 see +table above. */ +/** Zero relative shift position of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101 \ + (FSP_FLAGS_POS_ATOMIC_BLOBS \ + + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) +/** Zero relative shift position of the PAGE_COMPRESSION_LEVEL field */ +#define FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101 \ + (FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101 + 1) +/** Zero relative shift position of the ATOMIC_WRITES field */ +#define FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101 \ + (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101 + 4) +/** Zero relative shift position of the PAGE_SSIZE field */ +#define FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101 \ + (FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101 + 2) + +/** Bit mask of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION_MARIADB101 \ + (1U << FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101) +/** Bit mask of the PAGE_COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL_MARIADB101 \ + (15U << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101) +/** Bit mask of the ATOMIC_WRITES field */ +#define FSP_FLAGS_MASK_ATOMIC_WRITES_MARIADB101 \ + (3U << FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101) +/** Bit mask of the PAGE_SSIZE field */ +#define FSP_FLAGS_MASK_PAGE_SSIZE_MARIADB101 \ + (15U << FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101) + /** Return the value of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_GET_PAGE_COMPRESSION(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION) \ - >> FSP_FLAGS_POS_PAGE_COMPRESSION) +#define FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101) /** Return the value of the PAGE_COMPRESSION_LEVEL field */ -#define FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL) \ - >> FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) -/** Return the value of the ATOMIC_WRITES field */ -#define FSP_FLAGS_GET_ATOMIC_WRITES(flags) \ - ((flags & FSP_FLAGS_MASK_ATOMIC_WRITES) \ - >> FSP_FLAGS_POS_ATOMIC_WRITES) - -/** Set a PAGE_SSIZE into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_SSIZE(flags, ssize) \ - (flags | (ssize << FSP_FLAGS_POS_PAGE_SSIZE)) - -/** Set a PAGE_COMPRESSION into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_COMPRESSION(flags, compression) \ - (flags | (compression << FSP_FLAGS_POS_PAGE_COMPRESSION)) - -/** Set a PAGE_COMPRESSION_LEVEL into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(flags, level) \ - (flags | (level << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL)) - -/** Set a ATOMIC_WRITES into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_ATOMIC_WRITES(flags, atomics) \ - (flags | (atomics << FSP_FLAGS_POS_ATOMIC_WRITES)) +#define FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101) +/** Return the value of the PAGE_SSIZE field */ +#define FSP_FLAGS_GET_PAGE_SSIZE_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_SSIZE_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101) /* @} */ @@ -733,19 +786,193 @@ fseg_print( mtr_t* mtr); /*!< in/out: mini-transaction */ #endif /* UNIV_BTR_PRINT */ -/********************************************************************//** -Validate and return the tablespace flags, which are stored in the -tablespace header at offset FSP_SPACE_FLAGS. They should be 0 for -ROW_FORMAT=COMPACT and ROW_FORMAT=REDUNDANT. The newer row formats, -COMPRESSED and DYNAMIC, use a file format > Antelope so they should -have a file format number plus the DICT_TF_COMPACT bit set. -@return true if check ok */ +/** Validate the tablespace flags, which are stored in the +tablespace header at offset FSP_SPACE_FLAGS. +@param[in] flags the contents of FSP_SPACE_FLAGS +@return whether the flags are correct (not in the buggy 10.1) format */ +MY_ATTRIBUTE((warn_unused_result, const)) UNIV_INLINE bool -fsp_flags_is_valid( -/*===============*/ - ulint flags) /*!< in: tablespace flags */ - MY_ATTRIBUTE((warn_unused_result, const)); +fsp_flags_is_valid(ulint flags) +{ + DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", + return(false);); + if (flags == 0) { + return(true); + } + if (flags & ~FSP_FLAGS_MASK) { + return(false); + } + if ((flags & (FSP_FLAGS_MASK_POST_ANTELOPE | FSP_FLAGS_MASK_ATOMIC_BLOBS)) + == FSP_FLAGS_MASK_ATOMIC_BLOBS) { + /* If the "atomic blobs" flag (indicating + ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED) flag + is set, then the "post Antelope" (ROW_FORMAT!=REDUNDANT) flag + must also be set. */ + return(false); + } + /* Bits 10..14 should be 0b0000d where d is the DATA_DIR flag + of MySQL 5.6 and MariaDB 10.0, which we ignore. + In the buggy FSP_SPACE_FLAGS written by MariaDB 10.1.0 to 10.1.20, + bits 10..14 would be nonzero 0bsssaa where sss is + nonzero PAGE_SSIZE (3, 4, 6, or 7) + and aa is ATOMIC_WRITES (not 0b11). */ + if (FSP_FLAGS_GET_RESERVED(flags) & ~1) { + return(false); + } + + const ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); + if (ssize == 1 || ssize == 2 || ssize == 5 || ssize & 8) { + /* the page_size is not between 4k and 64k; + 16k should be encoded as 0, not 5 */ + return(false); + } + const ulint zssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); + if (zssize == 0) { + /* not ROW_FORMAT=COMPRESSED */ + } else if (zssize > (ssize ? ssize : 5)) { + /* invalid KEY_BLOCK_SIZE */ + return(false); + } else if (~flags & (FSP_FLAGS_MASK_POST_ANTELOPE + | FSP_FLAGS_MASK_ATOMIC_BLOBS)) { + /* both these flags should be set for + ROW_FORMAT=COMPRESSED */ + return(false); + } + + return(true); +} + +/** Convert FSP_SPACE_FLAGS from the buggy MariaDB 10.1.0..10.1.20 format. +@param[in] flags the contents of FSP_SPACE_FLAGS +@return the flags corrected from the buggy MariaDB 10.1 format +@retval ULINT_UNDEFINED if the flags are not in the buggy 10.1 format */ +MY_ATTRIBUTE((warn_unused_result, const)) +UNIV_INLINE +ulint +fsp_flags_convert_from_101(ulint flags) +{ + DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", + return(ULINT_UNDEFINED);); + if (flags == 0) { + return(flags); + } + + if (flags >> 18) { + /* The most significant FSP_SPACE_FLAGS bit that was ever set + by MariaDB 10.1.0 to 10.1.20 was bit 17 (misplaced DATA_DIR flag). + The flags must be less than 1<<18 in order to be valid. */ + return(ULINT_UNDEFINED); + } + + if ((flags & (FSP_FLAGS_MASK_POST_ANTELOPE | FSP_FLAGS_MASK_ATOMIC_BLOBS)) + == FSP_FLAGS_MASK_ATOMIC_BLOBS) { + /* If the "atomic blobs" flag (indicating + ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED) flag + is set, then the "post Antelope" (ROW_FORMAT!=REDUNDANT) flag + must also be set. */ + return(ULINT_UNDEFINED); + } + + /* Bits 6..10 denote compression in MariaDB 10.1.0 to 10.1.20. + They must be either 0b00000 or 0b00011 through 0b10011. + In correct versions, these bits would be + 0bd0sss where d is the DATA_DIR flag (garbage bit) and + sss is the PAGE_SSIZE (3, 4, 6, or 7). + + NOTE: MariaDB 10.1.0 to 10.1.20 can misinterpret + uncompressed data files with innodb_page_size=4k or 64k as + compressed innodb_page_size=16k files. Below is an exhaustive + state space analysis. + + -0by1zzz: impossible (the bit 4 must be clean; see above) + -0b101xx: DATA_DIR, innodb_page_size>4k: invalid (COMPRESSION_LEVEL>9) + +0bx0011: innodb_page_size=4k: + !!! Misinterpreted as COMPRESSION_LEVEL=9 or 1, COMPRESSION=1. + -0bx0010: impossible, because sss must be 0b011 or 0b1xx + -0bx0001: impossible, because sss must be 0b011 or 0b1xx + -0b10000: DATA_DIR, innodb_page_size=16: + invalid (COMPRESSION_LEVEL=8 but COMPRESSION=0) + +0b00111: no DATA_DIR, innodb_page_size=64k: + !!! Misinterpreted as COMPRESSION_LEVEL=3, COMPRESSION=1. + -0b00101: impossible, because sss must be 0 for 16k, not 0b101 + -0b001x0: no DATA_DIR, innodb_page_size=32k or 8k: + invalid (COMPRESSION_LEVEL=3 but COMPRESSION=0) + +0b00000: innodb_page_size=16k (looks like COMPRESSION=0) + ??? Could actually be compressed; see PAGE_SSIZE below */ + const ulint level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL_MARIADB101( + flags); + if (FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) != (level != 0) + || level > 9) { + /* The compression flags are not in the buggy MariaDB + 10.1 format. */ + return(ULINT_UNDEFINED); + } + if (!(~flags & FSP_FLAGS_MASK_ATOMIC_WRITES_MARIADB101)) { + /* The ATOMIC_WRITES flags cannot be 0b11. + (The bits 11..12 should actually never be 0b11, + because in MySQL they would be SHARED|TEMPORARY.) */ + return(ULINT_UNDEFINED); + } + + /* Bits 13..16 are the wrong position for PAGE_SSIZE, and they + should contain one of the values 3,4,6,7, that is, be of the form + 0b0011 or 0b01xx (except 0b0110). + In correct versions, these bits should be 0bc0se + where c is the MariaDB COMPRESSED flag + and e is the MySQL 5.7 ENCRYPTION flag + and s is the MySQL 8.0 SDI flag. MariaDB can only support s=0, e=0. + + Compressed innodb_page_size=16k tables with correct FSP_SPACE_FLAGS + will be properly rejected by older MariaDB 10.1.x because they + would read as PAGE_SSIZE>=8 which is not valid. */ + + const ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE_MARIADB101(flags); + if (ssize == 1 || ssize == 2 || ssize == 5 || ssize & 8) { + /* the page_size is not between 4k and 64k; + 16k should be encoded as 0, not 5 */ + return(ULINT_UNDEFINED); + } + const ulint zssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); + if (zssize == 0) { + /* not ROW_FORMAT=COMPRESSED */ + } else if (zssize > (ssize ? ssize : 5)) { + /* invalid KEY_BLOCK_SIZE */ + return(ULINT_UNDEFINED); + } else if (~flags & (FSP_FLAGS_MASK_POST_ANTELOPE + | FSP_FLAGS_MASK_ATOMIC_BLOBS)) { + /* both these flags should be set for + ROW_FORMAT=COMPRESSED */ + return(ULINT_UNDEFINED); + } + + flags = ((flags & 0x3f) | ssize << FSP_FLAGS_POS_PAGE_SSIZE + | FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) + << FSP_FLAGS_POS_PAGE_COMPRESSION); + ut_ad(fsp_flags_is_valid(flags)); + return(flags); +} + +/** Compare tablespace flags. +@param[in] expected expected flags from dict_tf_to_fsp_flags() +@param[in] actual flags read from FSP_SPACE_FLAGS +@return whether the flags match */ +MY_ATTRIBUTE((warn_unused_result)) +UNIV_INLINE +bool +fsp_flags_match(ulint expected, ulint actual) +{ + expected &= ~FSP_FLAGS_MEM_MASK; + ut_ad(fsp_flags_is_valid(expected)); + + if (actual == expected) { + return(true); + } + + actual = fsp_flags_convert_from_101(actual); + return(actual == expected); +} + /********************************************************************//** Determine if the tablespace is compressed from dict_table_t::flags. @return TRUE if compressed, FALSE if not compressed */ diff --git a/storage/xtradb/include/fsp0fsp.ic b/storage/xtradb/include/fsp0fsp.ic index ddcb87b0e57..ee4cb1f32c7 100644 --- a/storage/xtradb/include/fsp0fsp.ic +++ b/storage/xtradb/include/fsp0fsp.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2012, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -47,107 +47,6 @@ fsp_descr_page( } /********************************************************************//** -Validate and return the tablespace flags, which are stored in the -tablespace header at offset FSP_SPACE_FLAGS. They should be 0 for -ROW_FORMAT=COMPACT and ROW_FORMAT=REDUNDANT. The newer row formats, -COMPRESSED and DYNAMIC, use a file format > Antelope so they should -have a file format number plus the DICT_TF_COMPACT bit set. -@return true if check ok */ -UNIV_INLINE -bool -fsp_flags_is_valid( -/*===============*/ - ulint flags) /*!< in: tablespace flags */ -{ - ulint post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(flags); - ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); - ulint atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(flags); - ulint page_ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); - ulint unused = FSP_FLAGS_GET_UNUSED(flags); - ulint page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(flags); - ulint page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags); - ulint atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(flags); - - DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", return(false);); - - /* fsp_flags is zero unless atomic_blobs is set. */ - /* Make sure there are no bits that we do not know about. */ - if (unused != 0 || flags == 1) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted unused %lu\n", - flags, unused); - return(false); - } else if (post_antelope) { - /* The Antelope row formats REDUNDANT and COMPACT did - not use tablespace flags, so this flag and the entire - 4-byte field is zero for Antelope row formats. */ - - if (!atomic_blobs) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted atomic_blobs %lu\n", - flags, atomic_blobs); - return(false); - } - } - - if (!atomic_blobs) { - /* Barracuda row formats COMPRESSED and DYNAMIC build on - the page structure introduced for the COMPACT row format - by allowing long fields to be broken into prefix and - externally stored parts. */ - - if (post_antelope || zip_ssize != 0) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted zip_ssize %lu atomic_blobs %lu\n", - flags, zip_ssize, atomic_blobs); - return(false); - } - - } else if (!post_antelope || zip_ssize > PAGE_ZIP_SSIZE_MAX) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted zip_ssize %lu max %d\n", - flags, zip_ssize, PAGE_ZIP_SSIZE_MAX); - return(false); - } else if (page_ssize > UNIV_PAGE_SSIZE_MAX) { - - /* The page size field can be used for any row type, or it may - be zero for an original 16k page size. - Validate the page shift size is within allowed range. */ - - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted page_ssize %lu max %lu\n", - flags, page_ssize, UNIV_PAGE_SSIZE_MAX); - return(false); - - } else if (UNIV_PAGE_SIZE != UNIV_PAGE_SIZE_ORIG && !page_ssize) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted page_ssize %lu max %lu:%d\n", - flags, page_ssize, UNIV_PAGE_SIZE, UNIV_PAGE_SIZE_ORIG); - return(false); - } - -#if UNIV_FORMAT_MAX != UNIV_FORMAT_B -# error "UNIV_FORMAT_MAX != UNIV_FORMAT_B, Add more validations." -#endif - - /* Page compression level requires page compression and atomic blobs - to be set */ - if (page_compression_level || page_compression) { - if (!page_compression || !atomic_blobs) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted page_compression %lu\n" - "InnoDB: Error: page_compression_level %lu atomic_blobs %lu\n", - flags, page_compression, page_compression_level, atomic_blobs); - return(false); - } - } - - if (atomic_writes > ATOMIC_WRITES_OFF) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted atomic_writes %lu\n", - flags, atomic_writes); - return (false); - } - - /* The DATA_DIR field can be used for any row type so there is - nothing here to validate. */ - - return(true); -} - -/********************************************************************//** Determine if the tablespace is compressed from dict_table_t::flags. @return TRUE if compressed, FALSE if not compressed */ UNIV_INLINE @@ -191,10 +90,10 @@ UNIV_INLINE ulint fsp_flags_get_page_size( /*====================*/ - ulint flags) /*!< in: tablespace flags */ + ulint flags) /*!< in: tablespace flags */ { - ulint page_size = 0; - ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); + ulint page_size = 0; + ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); /* Convert from a 'log2 minus 9' to a page size in bytes. */ if (UNIV_UNLIKELY(ssize)) { @@ -211,50 +110,6 @@ fsp_flags_get_page_size( } #ifndef UNIV_INNOCHECKSUM - -/********************************************************************//** -Add the page size to the tablespace flags. -@return tablespace flags after page size is added */ -UNIV_INLINE -ulint -fsp_flags_set_page_size( -/*====================*/ - ulint flags, /*!< in: tablespace flags */ - ulint page_size) /*!< in: page size in bytes */ -{ - ulint ssize = 0; - ulint shift; - - /* Page size should be > UNIV_PAGE_SIZE_MIN */ - ut_ad(page_size >= UNIV_PAGE_SIZE_MIN); - ut_ad(page_size <= UNIV_PAGE_SIZE_MAX); - - if (page_size == UNIV_PAGE_SIZE_ORIG) { - ut_ad(0 == FSP_FLAGS_GET_PAGE_SSIZE(flags)); - return(flags); - } - - for (shift = UNIV_PAGE_SIZE_SHIFT_MAX; - shift >= UNIV_PAGE_SIZE_SHIFT_MIN; - shift--) { - ulint mask = (1 << shift); - if (page_size & mask) { - ut_ad(!(page_size & ~mask)); - ssize = shift - UNIV_ZIP_SIZE_SHIFT_MIN + 1; - break; - } - } - - ut_ad(ssize); - ut_ad(ssize <= UNIV_PAGE_SSIZE_MAX); - - flags = FSP_FLAGS_SET_PAGE_SSIZE(flags, ssize); - - ut_ad(fsp_flags_is_valid(flags)); - - return(flags); -} - /********************************************************************//** Calculates the descriptor index within a descriptor page. @return descriptor index */ diff --git a/storage/xtradb/include/fsp0pagecompress.h b/storage/xtradb/include/fsp0pagecompress.h index 5f943ee2b83..c623d11c326 100644 --- a/storage/xtradb/include/fsp0pagecompress.h +++ b/storage/xtradb/include/fsp0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -48,15 +48,6 @@ fsp_header_get_compression_level( const page_t* page); /*!< in: first page of a tablespace */ /********************************************************************//** -Determine if the tablespace is page compressed from dict_table_t::flags. -@return TRUE if page compressed, FALSE if not compressed */ -UNIV_INLINE -ibool -fsp_flags_is_page_compressed( -/*=========================*/ - ulint flags); /*!< in: tablespace flags */ - -/********************************************************************//** Extract the page compression level from tablespace flags. A tablespace has only one physical page compression level whether that page is compressed or not. diff --git a/storage/xtradb/include/fsp0pagecompress.ic b/storage/xtradb/include/fsp0pagecompress.ic index e879aa2c16e..48163277feb 100644 --- a/storage/xtradb/include/fsp0pagecompress.ic +++ b/storage/xtradb/include/fsp0pagecompress.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -26,18 +26,6 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@mariadb.com ***********************************************************************/ /********************************************************************//** -Determine if the tablespace is page compressed from dict_table_t::flags. -@return TRUE if page compressed, FALSE if not page compressed */ -UNIV_INLINE -ibool -fsp_flags_is_page_compressed( -/*=========================*/ - ulint flags) /*!< in: tablespace flags */ -{ - return(FSP_FLAGS_GET_PAGE_COMPRESSION(flags)); -} - -/********************************************************************//** Determine the tablespace is page compression level from dict_table_t::flags. @return page compression level or 0 if not compressed*/ UNIV_INLINE @@ -125,21 +113,15 @@ Extract the page compression from space. @return true if space is page compressed, false if space is not found or space is not page compressed. */ UNIV_INLINE -ibool +bool fil_space_is_page_compressed( /*=========================*/ ulint id) /*!< in: space id */ { - ulint flags; - - flags = fil_space_get_flags(id); - - if (flags && flags != ULINT_UNDEFINED) { - - return(fsp_flags_is_page_compressed(flags)); - } + ulint flags = fil_space_get_flags(id); - return(0); + return(flags != ULINT_UNDEFINED + && FSP_FLAGS_HAS_PAGE_COMPRESSION(flags)); } #endif /* UNIV_INNOCHECKSUM */ diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index d45ce907304..6170eb66195 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -371,8 +371,7 @@ public: m_space(ULINT_UNDEFINED), m_xdes(), m_xdes_page_no(ULINT_UNDEFINED), - m_space_flags(ULINT_UNDEFINED), - m_table_flags(ULINT_UNDEFINED) UNIV_NOTHROW { } + m_space_flags(ULINT_UNDEFINED) UNIV_NOTHROW { } /** Free any extent descriptor instance */ @@ -535,10 +534,6 @@ protected: /** Flags value read from the header page */ ulint m_space_flags; - - /** Derived from m_space_flags and row format type, the row format - type is determined from the page header. */ - ulint m_table_flags; }; /** Determine the page size to use for traversing the tablespace @@ -553,6 +548,19 @@ AbstractCallback::init( const page_t* page = block->frame; m_space_flags = fsp_header_get_flags(page); + if (!fsp_flags_is_valid(m_space_flags)) { + ulint cflags = fsp_flags_convert_from_101(m_space_flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Invalid FSP_SPACE_FLAGS=0x%x", + int(m_space_flags)); + return(DB_CORRUPTION); + } + m_space_flags = cflags; + } + + /* Clear the DATA_DIR flag, which is basically garbage. */ + m_space_flags &= ~(1U << FSP_FLAGS_POS_RESERVED); /* Since we don't know whether it is a compressed table or not, the data is always read into the block->frame. */ @@ -641,46 +649,6 @@ struct FetchIndexRootPages : public AbstractCallback { } /** - Check if the .ibd file row format is the same as the table's. - @param ibd_table_flags - determined from space and page. - @return DB_SUCCESS or error code. */ - dberr_t check_row_format(ulint ibd_table_flags) UNIV_NOTHROW - { - dberr_t err; - rec_format_t ibd_rec_format; - rec_format_t table_rec_format; - - if (!dict_tf_is_valid(ibd_table_flags)) { - - ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - ".ibd file has invlad table flags: %lx", - ibd_table_flags); - - return(DB_CORRUPTION); - } - - ibd_rec_format = dict_tf_get_rec_format(ibd_table_flags); - table_rec_format = dict_tf_get_rec_format(m_table->flags); - - if (table_rec_format != ibd_rec_format) { - - ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Table has %s row format, .ibd " - "file has %s row format.", - dict_tf_to_row_format_string(m_table->flags), - dict_tf_to_row_format_string(ibd_table_flags)); - - err = DB_CORRUPTION; - } else { - err = DB_SUCCESS; - } - - return(err); - } - - /** Called for each block as it is read from the file. @param offset - physical offset in the file @param block - block to convert, it is not from the buffer pool. @@ -743,12 +711,17 @@ FetchIndexRootPages::operator() ( m_indexes.push_back(Index(id, page_no)); if (m_indexes.size() == 1) { - - m_table_flags = dict_sys_tables_type_to_tf( - m_space_flags, - page_is_comp(page) ? DICT_N_COLS_COMPACT : 0); - - err = check_row_format(m_table_flags); + /* Check that the tablespace flags match the table flags. */ + ulint expected = dict_tf_to_fsp_flags(m_table->flags); + if (!fsp_flags_match(expected, m_space_flags)) { + ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, + ER_TABLE_SCHEMA_MISMATCH, + "Expected FSP_SPACE_FLAGS=0x%x, .ibd " + "file contains 0x%x.", + unsigned(expected), + unsigned(m_space_flags)); + return(DB_CORRUPTION); + } } } @@ -1968,21 +1941,14 @@ PageConverter::update_header( "- ignored"); } - ulint space_flags = fsp_header_get_flags(get_frame(block)); - - if (!fsp_flags_is_valid(space_flags)) { - - ib_logf(IB_LOG_LEVEL_ERROR, - "Unsupported tablespace format %lu", - (ulong) space_flags); - - return(DB_UNSUPPORTED); - } - mach_write_to_8( get_frame(block) + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, m_current_lsn); + /* Write back the adjusted flags. */ + mach_write_to_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + + get_frame(block), m_space_flags); + /* Write space_id to the tablespace header, page 0. */ mach_write_to_4( get_frame(block) + FSP_HEADER_OFFSET + FSP_SPACE_ID, diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index aff7b758249..c81b10b93f1 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -4362,6 +4362,7 @@ row_drop_table_for_mysql( switch (err) { ibool is_temp; + ulint table_flags; case DB_SUCCESS: /* Clone the name, in case it has been allocated @@ -4370,6 +4371,7 @@ row_drop_table_for_mysql( space_id = table->space; ibd_file_missing = table->ibd_file_missing; + table_flags = table->flags; is_temp = DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY); /* If there is a temp path then the temp flag is set. @@ -4385,9 +4387,9 @@ row_drop_table_for_mysql( } /* We do not allow temporary tables with a remote path. */ - ut_a(!(is_temp && DICT_TF_HAS_DATA_DIR(table->flags))); + ut_a(!(is_temp && DICT_TF_HAS_DATA_DIR(table_flags))); - if (space_id && DICT_TF_HAS_DATA_DIR(table->flags)) { + if (space_id && DICT_TF_HAS_DATA_DIR(table_flags)) { dict_get_and_save_data_dir_path(table, true); ut_a(table->data_dir_path); @@ -4453,8 +4455,9 @@ row_drop_table_for_mysql( if (err == DB_SUCCESS && space_id > TRX_SYS_SPACE) { if (!is_temp && !fil_space_for_table_exists_in_mem( - space_id, tablename, FALSE, - print_msg, false, NULL, 0)) { + space_id, tablename, + print_msg, false, NULL, 0, + table_flags)) { /* This might happen if we are dropping a discarded tablespace */ err = DB_SUCCESS; diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index bb07aea7720..70aec8767ad 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -703,8 +703,7 @@ create_log_files( sprintf(logfilename + dirnamelen, "ib_logfile%u", INIT_LOG_FILE0); fil_space_create( - logfilename, SRV_LOG_SPACE_FIRST_ID, - fsp_flags_set_page_size(0, UNIV_PAGE_SIZE), + logfilename, SRV_LOG_SPACE_FIRST_ID, 0, FIL_LOG, NULL /* no encryption yet */, true /* this is create */); @@ -1193,7 +1192,7 @@ check_first_page: crypt_data = fil_space_create_crypt_data(FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY); } - flags = fsp_flags_set_page_size(0, UNIV_PAGE_SIZE); + flags = FSP_FLAGS_PAGE_SSIZE(); fil_space_create(name, 0, flags, FIL_TABLESPACE, crypt_data, (*create_new_db) == true); @@ -1341,7 +1340,7 @@ srv_undo_tablespace_open( fil_set_max_space_id_if_bigger(space); /* Set the compressed page size to 0 (non-compressed) */ - flags = fsp_flags_set_page_size(0, UNIV_PAGE_SIZE); + flags = FSP_FLAGS_PAGE_SSIZE(); fil_space_create(name, space, flags, FIL_TABLESPACE, NULL /* no encryption */, true /* create */); @@ -2343,9 +2342,7 @@ innobase_start_or_create_for_mysql(void) sprintf(logfilename + dirnamelen, "ib_logfile%u", 0); fil_space_create(logfilename, - SRV_LOG_SPACE_FIRST_ID, - fsp_flags_set_page_size(0, UNIV_PAGE_SIZE), - FIL_LOG, + SRV_LOG_SPACE_FIRST_ID, 0, FIL_LOG, NULL /* no encryption yet */, true /* create */); @@ -2740,6 +2737,13 @@ files_checked: } if (!srv_read_only_mode) { + const ulint flags = FSP_FLAGS_PAGE_SSIZE(); + for (ulint id = 0; id <= srv_undo_tablespaces; id++) { + if (fil_space_get(id)) { + fsp_flags_try_adjust(id, flags); + } + } + /* Create the thread which watches the timeouts for lock waits */ thread_handles[2 + SRV_MAX_N_IO_THREADS] = os_thread_create( |