diff options
| author | Marko Mäkelä <marko.makela@mariadb.com> | 2017-05-05 10:25:29 +0300 |
|---|---|---|
| committer | Marko Mäkelä <marko.makela@mariadb.com> | 2017-05-05 10:38:53 +0300 |
| commit | f9cc391863ab962e8c8a8a8a676d730498eb8d6f (patch) | |
| tree | a602f36756fafd5ec7ab856563689f9e1a50a8f2 | |
| parent | db0917f68f2681882974afd53935aa8cba29c6b8 (diff) | |
| parent | 765a43605a42c069ede604826ede2d93d72c4fdd (diff) | |
| download | mariadb-git-f9cc391863ab962e8c8a8a8a676d730498eb8d6f.tar.gz | |
Merge 10.1 into 10.2
This only merges MDEV-12253, adapting it to MDEV-12602 which is already
present in 10.2 but not yet in the 10.1 revision that is being merged.
TODO: Error handling in crash recovery needs to be improved.
If a page cannot be decrypted (or read), we should cleanly abort
the startup. If innodb_force_recovery is specified, we should
ignore the problematic page and apply redo log to other pages.
Currently, the test encryption.innodb-redo-badkey randomly fails
like this (the last messages are from cmake -DWITH_ASAN):
2017-05-05 10:19:40 140037071685504 [Note] InnoDB: Starting crash recovery from checkpoint LSN=1635994
2017-05-05 10:19:40 140037071685504 [ERROR] InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 1
2017-05-05 10:19:40 140037071685504 [ERROR] InnoDB: Plugin initialization aborted at srv0start.cc[2201] with error Data structure corruption
2017-05-05 10:19:41 140037071685504 [Note] InnoDB: Starting shutdown...
i=================================================================
==5226==ERROR: AddressSanitizer: attempting free on address which was not malloc()-ed: 0x612000018588 in thread T0
#0 0x736750 in operator delete(void*) (/mariadb/server/build/sql/mysqld+0x736750)
#1 0x1e4833f in LatchCounter::~LatchCounter() /mariadb/server/storage/innobase/include/sync0types.h:599:4
#2 0x1e480b8 in LatchMeta<LatchCounter>::~LatchMeta() /mariadb/server/storage/innobase/include/sync0types.h:786:17
#3 0x1e35509 in sync_latch_meta_destroy() /mariadb/server/storage/innobase/sync/sync0debug.cc:1622:3
#4 0x1e35314 in sync_check_close() /mariadb/server/storage/innobase/sync/sync0debug.cc:1839:2
#5 0x1dfdc18 in innodb_shutdown() /mariadb/server/storage/innobase/srv/srv0start.cc:2888:2
#6 0x197e5e6 in innobase_init(void*) /mariadb/server/storage/innobase/handler/ha_innodb.cc:4475:3
188 files changed, 3487 insertions, 2903 deletions
diff --git a/mysql-test/std_data/keys.txt b/mysql-test/std_data/keys.txt index e511521d7ab..2d83b7fba60 100644 --- a/mysql-test/std_data/keys.txt +++ b/mysql-test/std_data/keys.txt @@ -9,4 +9,4 @@ 5;966050D7777350B6FD5CCB3E5F648DA45C63BEFB6DEDDFA13443F156B7D35C84 6;B5EA210C8C09EF20DB95EC584714A89F # and yet another - +10;770A8A65DA156D24EE2A093277530143 diff --git a/mysql-test/std_data/keys2.txt b/mysql-test/std_data/keys2.txt index 5b98fbeebd2..93fc537cd3e 100644 --- a/mysql-test/std_data/keys2.txt +++ b/mysql-test/std_data/keys2.txt @@ -1,7 +1,8 @@ -1;593E580927F47AB530D3B1237CDEF6D6 +1;770A8A65DA156D24EE2A093277530142 2;352E42F1B9DB5CB915C3262FE745520A 3;CFE065600F5EB57481075C65180C3F8A 4;205379930183490D3BECA139BDF4DB5B 5;E2D944D5D837A1DCB22FF7FD397892EE 6;BAFE99B0BB87F2CD33A6AF26A11F6BD1 19;678D6B0063824BACCE33224B385104B35F30FF5749F0EBC030A0955DBC7FAC34 +20;BAFE99B0BB87F2CD33A6AF26A11F6BD1 diff --git a/mysql-test/std_data/keys3.txt b/mysql-test/std_data/keys3.txt index 4f6c618d2ad..be30acc6add 100644 --- a/mysql-test/std_data/keys3.txt +++ b/mysql-test/std_data/keys3.txt @@ -1,4 +1,4 @@ -1;593E580927F47AB530D3B1237CDEF6D6 +1;770A8A65DA156D24EE2A093277530142 2;E4B00A45BF775B4E07D634EC5CA5912B 3;6E35ACB162B29D1FB9E178021DAF16ED 4;971A664A88EE0022D408E40BFAB17E79 diff --git a/mysql-test/suite/encryption/r/create_or_replace.result b/mysql-test/suite/encryption/r/create_or_replace.result index d52572d6d23..1671043b50d 100644 --- a/mysql-test/suite/encryption/r/create_or_replace.result +++ b/mysql-test/suite/encryption/r/create_or_replace.result @@ -1,4 +1,3 @@ -call mtr.add_suppression("InnoDB: Error: trying to do an operation on a dropped tablespace.*"); SET default_storage_engine = InnoDB; CREATE TABLE t1 (pk INT PRIMARY KEY, c VARCHAR(256)); CREATE TABLE t2 AS SELECT * FROM t1; diff --git a/mysql-test/suite/encryption/r/encrypt_and_grep.result b/mysql-test/suite/encryption/r/encrypt_and_grep.result index b1ffbdb8134..38dc11e7850 100644 --- a/mysql-test/suite/encryption/r/encrypt_and_grep.result +++ b/mysql-test/suite/encryption/r/encrypt_and_grep.result @@ -1,10 +1,13 @@ SET GLOBAL innodb_file_per_table = ON; +SET GLOBAL innodb_file_format = `Barracuda`; create table t1 (a varchar(255)) engine=innodb encrypted=yes; create table t2 (a varchar(255)) engine=innodb; +show warnings; +Level Code Message create table t3 (a varchar(255)) engine=innodb encrypted=no; -insert t1 values (repeat('foobar', 42)); -insert t2 values (repeat('temp', 42)); -insert t3 values (repeat('dummy', 42)); +insert t1 values (repeat('foobarsecret', 12)); +insert t2 values (repeat('tempsecret', 12)); +insert t3 values (repeat('dummysecret', 12)); # Wait max 10 min for key encryption threads to encrypt all spaces SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0; NAME @@ -17,15 +20,14 @@ test/t1 test/t2 innodb_system # t1 yes on expecting NOT FOUND -NOT FOUND /foobar/ in t1.ibd +NOT FOUND /foobarsecret/ in t1.ibd # t2 ... on expecting NOT FOUND -NOT FOUND /temp/ in t2.ibd +NOT FOUND /tempsecret/ in t2.ibd # t3 no on expecting FOUND -FOUND 42 /dummy/ in t3.ibd +FOUND 12 /dummysecret/ in t3.ibd # ibdata1 expecting NOT FOUND -NOT FOUND /foobar/ in ibdata1 +NOT FOUND /foobarsecret/ in ibdata1 # Now turn off encryption and wait for threads to decrypt everything -SET GLOBAL innodb_encryption_threads = 1; SET GLOBAL innodb_encrypt_tables = off; # Wait max 10 min for key encryption threads to decrypt all spaces SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0; @@ -39,15 +41,14 @@ SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_ NAME test/t1 # t1 yes on expecting NOT FOUND -NOT FOUND /foobar/ in t1.ibd -# t2 ... on expecting FOUND -NOT FOUND /temp/ in t2.ibd +NOT FOUND /foobarsecret/ in t1.ibd +# t2 ... default expecting FOUND +FOUND 12 /tempsecret/ in t2.ibd # t3 no on expecting FOUND -FOUND 42 /dummy/ in t3.ibd +FOUND 12 /dummysecret/ in t3.ibd # ibdata1 expecting NOT FOUND -NOT FOUND /foobar/ in ibdata1 +NOT FOUND /foobarsecret/ in ibdata1 # Now turn on encryption and wait for threads to encrypt all spaces -SET GLOBAL innodb_encryption_threads = 4; SET GLOBAL innodb_encrypt_tables = on; # Wait max 10 min for key encryption threads to encrypt all spaces SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0; @@ -61,11 +62,11 @@ test/t1 test/t2 innodb_system # t1 yes on expecting NOT FOUND -NOT FOUND /foobar/ in t1.ibd +NOT FOUND /foobarsecret/ in t1.ibd # t2 ... on expecting NOT FOUND -NOT FOUND /temp/ in t2.ibd +NOT FOUND /tempsecret/ in t2.ibd # t3 no on expecting FOUND -FOUND 42 /dummy/ in t3.ibd +FOUND 12 /dummysecret/ in t3.ibd # ibdata1 expecting NOT FOUND -NOT FOUND /foobar/ in ibdata1 +NOT FOUND /foobarsecret/ in ibdata1 drop table t1, t2, t3; diff --git a/mysql-test/suite/encryption/r/filekeys_emptyfile.result b/mysql-test/suite/encryption/r/filekeys_emptyfile.result index 19bca3c36c7..f94f11d9f08 100644 --- a/mysql-test/suite/encryption/r/filekeys_emptyfile.result +++ b/mysql-test/suite/encryption/r/filekeys_emptyfile.result @@ -1,7 +1,7 @@ call mtr.add_suppression("System key id 1 is missing at"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 1 /System key id 1 is missing at/ in mysqld.1.err +FOUND /System key id 1 is missing at/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins diff --git a/mysql-test/suite/encryption/r/filekeys_encfile_bad.result b/mysql-test/suite/encryption/r/filekeys_encfile_bad.result index 59124f2babd..6261bd459b8 100644 --- a/mysql-test/suite/encryption/r/filekeys_encfile_bad.result +++ b/mysql-test/suite/encryption/r/filekeys_encfile_bad.result @@ -1,7 +1,7 @@ call mtr.add_suppression("Cannot decrypt .*filekeys-data.enc. Wrong key"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 1 /Cannot decrypt .*filekeys-data.enc. Wrong key/ in mysqld.1.err +FOUND /Cannot decrypt .*filekeys-data.enc. Wrong key/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins diff --git a/mysql-test/suite/encryption/r/filekeys_encfile_badfile.result b/mysql-test/suite/encryption/r/filekeys_encfile_badfile.result index 7e244c2c381..98e2266f3f2 100644 --- a/mysql-test/suite/encryption/r/filekeys_encfile_badfile.result +++ b/mysql-test/suite/encryption/r/filekeys_encfile_badfile.result @@ -1,7 +1,7 @@ call mtr.add_suppression("File 'bad' not found"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 1 /File 'bad' not found/ in mysqld.1.err +FOUND /File 'bad' not found/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins diff --git a/mysql-test/suite/encryption/r/filekeys_encfile_no.result b/mysql-test/suite/encryption/r/filekeys_encfile_no.result index 59124f2babd..6261bd459b8 100644 --- a/mysql-test/suite/encryption/r/filekeys_encfile_no.result +++ b/mysql-test/suite/encryption/r/filekeys_encfile_no.result @@ -1,7 +1,7 @@ call mtr.add_suppression("Cannot decrypt .*filekeys-data.enc. Wrong key"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 1 /Cannot decrypt .*filekeys-data.enc. Wrong key/ in mysqld.1.err +FOUND /Cannot decrypt .*filekeys-data.enc. Wrong key/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins diff --git a/mysql-test/suite/encryption/r/filekeys_nofile.result b/mysql-test/suite/encryption/r/filekeys_nofile.result index 2caf258fef7..690f2e61df0 100644 --- a/mysql-test/suite/encryption/r/filekeys_nofile.result +++ b/mysql-test/suite/encryption/r/filekeys_nofile.result @@ -1,7 +1,7 @@ call mtr.add_suppression("file-key-management-filename is not set"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 1 /file-key-management-filename is not set/ in mysqld.1.err +FOUND /file-key-management-filename is not set/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins diff --git a/mysql-test/suite/encryption/r/filekeys_syntax.result b/mysql-test/suite/encryption/r/filekeys_syntax.result index 019446096b9..eb8119bc4f5 100644 --- a/mysql-test/suite/encryption/r/filekeys_syntax.result +++ b/mysql-test/suite/encryption/r/filekeys_syntax.result @@ -1,7 +1,7 @@ call mtr.add_suppression("File '.*keys.txt' not found"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 1 /File '.*keys.txt' not found/ in mysqld.1.err +FOUND /File '.*keys.txt' not found/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins @@ -12,7 +12,7 @@ ERROR HY000: Invalid key id at MYSQL_TMP_DIR/keys.txt line 2, column 2 call mtr.add_suppression("File '.*keys.txt' not found"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 1 /File '.*keys.txt' not found/ in mysqld.1.err +FOUND /File '.*keys.txt' not found/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins @@ -21,7 +21,7 @@ plugin_status call mtr.add_suppression("Invalid key id"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 1 /Invalid key id/ in mysqld.1.err +FOUND /Invalid key id/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins @@ -32,7 +32,7 @@ ERROR HY000: Invalid key id at MYSQL_TMP_DIR/keys.txt line 2, column 11 call mtr.add_suppression("Invalid key id"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 2 /Invalid key id/ in mysqld.1.err +FOUND /Invalid key id/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins @@ -41,7 +41,7 @@ plugin_status call mtr.add_suppression("Invalid key id"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 2 /Invalid key id/ in mysqld.1.err +FOUND /Invalid key id/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins @@ -52,7 +52,7 @@ ERROR HY000: Invalid key at MYSQL_TMP_DIR/keys.txt line 2, column 47 call mtr.add_suppression("Invalid key id"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 2 /Invalid key id/ in mysqld.1.err +FOUND /Invalid key id/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins @@ -61,7 +61,7 @@ plugin_status call mtr.add_suppression("Invalid key"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 3 /Invalid key/ in mysqld.1.err +FOUND /Invalid key/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins @@ -72,7 +72,7 @@ ERROR HY000: Invalid key at MYSQL_TMP_DIR/keys.txt line 2, column 33 call mtr.add_suppression("Invalid key"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 4 /Invalid key/ in mysqld.1.err +FOUND /Invalid key/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins @@ -81,7 +81,7 @@ plugin_status call mtr.add_suppression("Invalid key"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 4 /Invalid key/ in mysqld.1.err +FOUND /Invalid key/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins @@ -92,7 +92,7 @@ ERROR HY000: Syntax error at MYSQL_TMP_DIR/keys.txt line 2, column 2 call mtr.add_suppression("Invalid key"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 4 /Invalid key/ in mysqld.1.err +FOUND /Invalid key/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins @@ -101,7 +101,7 @@ plugin_status call mtr.add_suppression("Syntax error"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 1 /Syntax error/ in mysqld.1.err +FOUND /Syntax error/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins @@ -112,7 +112,7 @@ ERROR HY000: Syntax error at MYSQL_TMP_DIR/keys.txt line 2, column 1 call mtr.add_suppression("Syntax error"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 2 /Syntax error/ in mysqld.1.err +FOUND /Syntax error/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins @@ -121,7 +121,7 @@ plugin_status call mtr.add_suppression("Syntax error"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 2 /Syntax error/ in mysqld.1.err +FOUND /Syntax error/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins @@ -132,7 +132,7 @@ ERROR HY000: System key id 1 is missing at MYSQL_TMP_DIR/keys.txt line 1, column call mtr.add_suppression("Syntax error"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 2 /Syntax error/ in mysqld.1.err +FOUND /Syntax error/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins @@ -141,7 +141,7 @@ plugin_status call mtr.add_suppression("System key id 1"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 1 /System key id 1/ in mysqld.1.err +FOUND /System key id 1/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins diff --git a/mysql-test/suite/encryption/r/filekeys_tooshort.result b/mysql-test/suite/encryption/r/filekeys_tooshort.result index 781bde6fd49..efa66097563 100644 --- a/mysql-test/suite/encryption/r/filekeys_tooshort.result +++ b/mysql-test/suite/encryption/r/filekeys_tooshort.result @@ -1,7 +1,7 @@ call mtr.add_suppression("Cannot decrypt .*tooshort.enc. Not encrypted"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 1 /Cannot decrypt .*tooshort.enc. Not encrypted/ in mysqld.1.err +FOUND /Cannot decrypt .*tooshort.enc. Not encrypted/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins diff --git a/mysql-test/suite/encryption/r/filekeys_unencfile.result b/mysql-test/suite/encryption/r/filekeys_unencfile.result index 31668348607..1b9c092a713 100644 --- a/mysql-test/suite/encryption/r/filekeys_unencfile.result +++ b/mysql-test/suite/encryption/r/filekeys_unencfile.result @@ -1,7 +1,7 @@ call mtr.add_suppression("Cannot decrypt .*keys.txt. Not encrypted"); call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -FOUND 1 /Cannot decrypt .*keys.txt. Not encrypted/ in mysqld.1.err +FOUND /Cannot decrypt .*keys.txt. Not encrypted/ in mysqld.1.err create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; ERROR HY000: Can't create table `test`.`t1` (errno: 140 "Wrong create options") select plugin_status from information_schema.plugins diff --git a/mysql-test/suite/encryption/r/innochecksum.result b/mysql-test/suite/encryption/r/innochecksum.result index 7cd7af7b93b..50eafbf9f5e 100644 --- a/mysql-test/suite/encryption/r/innochecksum.result +++ b/mysql-test/suite/encryption/r/innochecksum.result @@ -1,3 +1,7 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; # Create and populate a tables CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; @@ -19,3 +23,5 @@ CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_CO # Write file to make mysql-test-run.pl start up the server again # Cleanup DROP TABLE t1, t2, t3, t4, t5; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change.result b/mysql-test/suite/encryption/r/innodb-bad-key-change.result index 798057e3fb5..63d41790320 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change.result @@ -1,9 +1,8 @@ -call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); -call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -call mtr.add_suppression("InnoDB: The page .* in file test/.* cannot be decrypted"); -call mtr.add_suppression("mysqld: File .* not found"); - +call mtr.add_suppression("Plugin 'file_key_management'"); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[12]\\.ibd' cannot be decrypted\\."); +call mtr.add_suppression("File '.*mysql-test.std_data.keysbad3\\.txt' not found"); # Start server with keys2.txt +SET GLOBAL innodb_file_per_table = ON; CREATE TABLE t1 (c VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=2; INSERT INTO t1 VALUES ('foobar'); ALTER TABLE t1 ADD COLUMN c2 INT; @@ -29,15 +28,13 @@ ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be SHOW WARNINGS; Level Code Message Warning 192 Table test/t1 in tablespace is encrypted but encryption service or used key_id is not available. Can't continue reading table. -Warning 192 Table test/t1 is encrypted but encryption service or used key_id 2 is not available. Can't continue reading table. +Warning 192 Table t1 in file ./test/t1.ibd is encrypted but encryption service or used key_id is not available. Can't continue reading table. Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB DROP TABLE t1; -Warnings: -Warning 192 Table in tablespace encrypted.However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match. Can't continue opening the table. SHOW WARNINGS; Level Code Message -Warning 192 Table in tablespace encrypted.However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match. Can't continue opening the table. -# Start server with keys.txt +# Start server with keys3.txt +SET GLOBAL innodb_default_encryption_key_id=5; CREATE TABLE t2 (c VARCHAR(8), id int not null primary key, b int, key(b)) ENGINE=InnoDB ENCRYPTED=YES; INSERT INTO t2 VALUES ('foobar',1,2); @@ -47,69 +44,60 @@ ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be SHOW WARNINGS; Level Code Message Warning 192 Table test/t2 in tablespace is encrypted but encryption service or used key_id is not available. Can't continue reading table. -Warning 192 Table test/t2 is encrypted but encryption service or used key_id is not available. Can't continue reading table. +Warning 192 Table t2 in file ./test/t2.ibd is encrypted but encryption service or used key_id is not available. Can't continue reading table. Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB SELECT * FROM t2 where id = 1; ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB SHOW WARNINGS; Level Code Message -Warning 1812 Tablespace is missing for table 'test/t2' -Warning 192 Table test/t2 is encrypted but encryption service or used key_id is not available. Can't continue reading table. +Warning 192 Table t2 in file ./test/t2.ibd is encrypted but encryption service or used key_id is not available. Can't continue reading table. Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB SELECT * FROM t2 where b = 1; ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB SHOW WARNINGS; Level Code Message -Warning 1812 Tablespace is missing for table 'test/t2' -Warning 192 Table test/t2 is encrypted but encryption service or used key_id is not available. Can't continue reading table. +Warning 192 Table t2 in file ./test/t2.ibd is encrypted but encryption service or used key_id is not available. Can't continue reading table. Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB INSERT INTO t2 VALUES ('tmp',3,3); ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB SHOW WARNINGS; Level Code Message -Warning 1812 Tablespace is missing for table 'test/t2' -Warning 192 Table test/t2 is encrypted but encryption service or used key_id is not available. Can't continue reading table. +Warning 192 Table t2 in file ./test/t2.ibd is encrypted but encryption service or used key_id is not available. Can't continue reading table. Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB DELETE FROM t2 where b = 3; ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB SHOW WARNINGS; Level Code Message -Warning 1812 Tablespace is missing for table 'test/t2' -Warning 192 Table test/t2 is encrypted but encryption service or used key_id is not available. Can't continue reading table. +Warning 192 Table t2 in file ./test/t2.ibd is encrypted but encryption service or used key_id is not available. Can't continue reading table. Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB DELETE FROM t2 where id = 3; ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB SHOW WARNINGS; Level Code Message -Warning 1812 Tablespace is missing for table 'test/t2' -Warning 192 Table test/t2 is encrypted but encryption service or used key_id is not available. Can't continue reading table. +Warning 192 Table t2 in file ./test/t2.ibd is encrypted but encryption service or used key_id is not available. Can't continue reading table. Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB UPDATE t2 set b = b +1; ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB SHOW WARNINGS; Level Code Message -Warning 1812 Tablespace is missing for table 'test/t2' -Warning 192 Table test/t2 is encrypted but encryption service or used key_id is not available. Can't continue reading table. +Warning 192 Table t2 in file ./test/t2.ibd is encrypted but encryption service or used key_id is not available. Can't continue reading table. Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB OPTIMIZE TABLE t2; Table Op Msg_type Msg_text -test.t2 optimize Warning Tablespace is missing for table 'test/t2' -test.t2 optimize Warning Table test/t2 is encrypted but encryption service or used key_id is not available. Can't continue reading table. +test.t2 optimize Warning Table t2 in file ./test/t2.ibd is encrypted but encryption service or used key_id is not available. Can't continue reading table. test.t2 optimize Error Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB test.t2 optimize error Corrupt SHOW WARNINGS; Level Code Message -ALTER TABLE t2 ADD COLUMN c INT; +ALTER TABLE t2 ADD COLUMN d INT; ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB SHOW WARNINGS; Level Code Message -Warning 1812 Tablespace is missing for table 'test/t2' -Warning 192 Table test/t2 is encrypted but encryption service or used key_id is not available. Can't continue reading table. +Warning 192 Table t2 in file ./test/t2.ibd is encrypted but encryption service or used key_id is not available. Can't continue reading table. Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB ANALYZE TABLE t2; Table Op Msg_type Msg_text -test.t2 analyze Warning Tablespace is missing for table 'test/t2' -test.t2 analyze Warning Table test/t2 is encrypted but encryption service or used key_id is not available. Can't continue reading table. +test.t2 analyze Warning Table t2 in file ./test/t2.ibd is encrypted but encryption service or used key_id is not available. Can't continue reading table. test.t2 analyze Error Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB test.t2 analyze error Corrupt SHOW WARNINGS; @@ -118,16 +106,8 @@ TRUNCATE TABLE t2; ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB SHOW WARNINGS; Level Code Message -Warning 1812 Tablespace is missing for table 'test/t2' -Warning 192 Table test/t2 is encrypted but encryption service or used key_id is not available. Can't continue reading table. +Warning 192 Table t2 in file ./test/t2.ibd is encrypted but encryption service or used key_id is not available. Can't continue reading table. Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB DROP TABLE t2; -ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB -SHOW WARNINGS; -Level Code Message -Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB -# Restart server with keys.txt -DROP TABLE t2; -SHOW WARNINGS; -Level Code Message +# Start server with keys2.txt diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change2.result b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result index adf984b9708..49129dc8fdd 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change2.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change2.result @@ -1,23 +1,57 @@ -call mtr.add_suppression("InnoDB: The page .* in file test/t1 cannot be decrypted"); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t1(new)?\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); -CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTION_KEY_ID=4; +call mtr.add_suppression("InnoDB: Tablespace for table \`test\`.\`t1\` is set as discarded."); +SET GLOBAL innodb_file_per_table = ON; +CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; INSERT INTO t1 VALUES (1,'foo'),(2,'bar'); -select * from t1; +SELECT * FROM t1; ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB -show warnings; +SHOW WARNINGS; Level Code Message -Warning 192 Table test/t1 is encrypted but encryption service or used key_id is not available. Can't continue reading table. -Warning 192 Table test/t1 is encrypted but encryption service or used key_id is not available. Can't continue reading table. +Warning 192 Table test/t1 in tablespace is encrypted but encryption service or used key_id is not available. Can't continue reading table. +Warning 192 Table t1 in file ./test/t1.ibd is encrypted but encryption service or used key_id is not available. Can't continue reading table. Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB -alter table t1 discard tablespace; +ALTER TABLE t1 engine=InnoDB; ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB -show warnings; +SHOW WARNINGS; Level Code Message +Warning 192 Table t1 in file ./test/t1.ibd is encrypted but encryption service or used key_id is not available. Can't continue reading table. Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB -alter table t1 engine=InnoDB; -ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB -show warnings; +OPTIMIZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 optimize Warning Table t1 in file ./test/t1.ibd is encrypted but encryption service or used key_id is not available. Can't continue reading table. +test.t1 optimize Error Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB +test.t1 optimize error Corrupt +SHOW WARNINGS; Level Code Message -Warning 192 Table test/t1 is encrypted but encryption service or used key_id is not available. Can't continue reading table. -Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB -DROP TABLE t1; +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check Warning Table t1 in file ./test/t1.ibd is encrypted but encryption service or used key_id is not available. Can't continue reading table. +test.t1 check Error Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB +test.t1 check error Corrupt +SHOW WARNINGS; +Level Code Message +FLUSH TABLES t1 FOR EXPORT; +backup: t1 +UNLOCK TABLES; +ALTER TABLE t1 DISCARD TABLESPACE; +Warnings: +Warning 192 Table test/t1 in tablespace is encrypted but encryption service or used key_id is not available. Can't continue reading table. +Warning 1812 Tablespace is missing for table 'test/t1' +restore: t1 .ibd and .cfg files +ALTER TABLE t1 IMPORT TABLESPACE; +Warnings: +Warning 1814 Tablespace has been discarded for table `t1` +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `pk` int(11) NOT NULL, + `f` varchar(8) DEFAULT NULL, + PRIMARY KEY (`pk`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 `ENCRYPTED`=YES `ENCRYPTION_KEY_ID`=4 +RENAME TABLE t1 TO t1new; +ALTER TABLE t1new RENAME TO t2new; +ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB +DROP TABLE t1new; +DROP TABLE t1, t1new; +ERROR 42S02: Unknown table 'test.t1,test.t1new' diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change3.result b/mysql-test/suite/encryption/r/innodb-bad-key-change3.result index 79155bf43b9..82046cbf3ad 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change3.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change3.result @@ -1,5 +1,6 @@ -call mtr.add_suppression("InnoDB: Tablespace for table .* is set as discarded."); -call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* because the .ibd file is missing. Please refer to .* for how to resolve the issue."); +call mtr.add_suppression("InnoDB: Tablespace for table \`test\`.\`t1\` is set as discarded."); +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(255)) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4; SHOW WARNINGS; @@ -13,14 +14,12 @@ t1 CREATE TABLE `t1` ( ) ENGINE=InnoDB DEFAULT CHARSET=latin1 `PAGE_COMPRESSED`=1 `ENCRYPTED`=YES `ENCRYPTION_KEY_ID`=4 INSERT INTO t1 VALUES (1,'foobar'),(2,'barfoo'); FLUSH TABLE t1 FOR EXPORT; -# List before copying files -t1.cfg -t1.frm -t1.ibd backup: t1 UNLOCK TABLES; ALTER TABLE t1 DISCARD TABLESPACE; restore: t1 .ibd and .cfg files +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; ALTER TABLE t1 IMPORT TABLESPACE; ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB SHOW CREATE TABLE t1; diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change4.result b/mysql-test/suite/encryption/r/innodb-bad-key-change4.result index 69f11d8745b..cec5e2b7eae 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change4.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change4.result @@ -1,11 +1,20 @@ -call mtr.add_suppression("InnoDB: The page .* in file test/t1 cannot be decrypted"); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t1\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); -CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTION_KEY_ID=4; +SET GLOBAL innodb_file_per_table = ON; +CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; INSERT INTO t1 VALUES (1,'foo'),(2,'bar'); +OPTIMIZE TABLE t1; +Table Op Msg_type Msg_text +test.t1 optimize Warning Table test/t1 in tablespace is encrypted but encryption service or used key_id is not available. Can't continue reading table. +test.t1 optimize Warning Table t1 in file ./test/t1.ibd is encrypted but encryption service or used key_id is not available. Can't continue reading table. +test.t1 optimize Error Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB +test.t1 optimize error Corrupt +SHOW WARNINGS; +Level Code Message CHECK TABLE t1; Table Op Msg_type Msg_text -test.t1 check Warning Table test/t1 in tablespace # is encrypted but encryption service or used key_id is not available. Can't continue reading table. -test.t1 check Warning Table test/t1 is encrypted but encryption service or used key_id is not available. Can't continue checking table. +test.t1 check Warning Table t1 in file ./test/t1.ibd is encrypted but encryption service or used key_id is not available. Can't continue reading table. +test.t1 check Error Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB test.t1 check error Corrupt SHOW WARNINGS; Level Code Message diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change5.result b/mysql-test/suite/encryption/r/innodb-bad-key-change5.result deleted file mode 100644 index 13b74f3b23d..00000000000 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change5.result +++ /dev/null @@ -1,20 +0,0 @@ -call mtr.add_suppression("InnoDB: The page .* in file test/t1 cannot be decrypted"); -call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); -CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTION_KEY_ID=4; -INSERT INTO t1 VALUES (1,'foo'),(2,'bar'); -OPTIMIZE TABLE t1; -Table Op Msg_type Msg_text -test.t1 optimize Warning Table test/t1 is encrypted but encryption service or used key_id is not available. Can't continue checking table. -test.t1 optimize Warning InnoDB: Cannot defragment table test/t1: returned error code 192 - -test.t1 optimize note Table does not support optimize, doing recreate + analyze instead -test.t1 optimize error Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB -test.t1 optimize status Operation failed -Warnings: -Warning 192 Table test/t1 is encrypted but encryption service or used key_id is not available. Can't continue reading table. -Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB -SHOW WARNINGS; -Level Code Message -Warning 192 Table test/t1 is encrypted but encryption service or used key_id is not available. Can't continue reading table. -Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB -DROP TABLE t1; diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-shutdown.result b/mysql-test/suite/encryption/r/innodb-bad-key-shutdown.result deleted file mode 100644 index 447329a74da..00000000000 --- a/mysql-test/suite/encryption/r/innodb-bad-key-shutdown.result +++ /dev/null @@ -1,16 +0,0 @@ -call mtr.add_suppression("InnoDB: The page .* in file test/t1 cannot be decrypted"); -call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); -# -# Restart the server with key 4 in the key file -# -CREATE TABLE t1 (i INT, KEY(i)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; -INSERT INTO t1 VALUES (1); -# -# Restart the server with a different value for key 4 in the key file -# -SELECT * FROM t1; -ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB -SELECT * FROM t1; -i -1 -DROP TABLE t1; diff --git a/mysql-test/suite/encryption/r/innodb-compressed-blob.result b/mysql-test/suite/encryption/r/innodb-compressed-blob.result new file mode 100644 index 00000000000..5753188b168 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb-compressed-blob.result @@ -0,0 +1,20 @@ +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); +# Restart mysqld --file-key-management-filename=keys2.txt +SET GLOBAL innodb_file_per_table = ON; +set GLOBAL innodb_default_encryption_key_id=4; +create table t1(a int not null primary key, b blob, index(b(10))) engine=innodb row_format=compressed; +create table t2(a int not null primary key, b blob, index(b(10))) engine=innodb row_format=compressed encrypted=yes; +create table t3(a int not null primary key, b blob, index(b(10))) engine=innodb row_format=compressed encrypted=no; +insert into t1 values (1, repeat('secret',6000)); +insert into t2 values (1, repeat('secret',6000)); +insert into t3 values (1, repeat('secret',6000)); +# Restart mysqld --file-key-management-filename=keys3.txt +select count(*) from t1 FORCE INDEX (b) where b like 'secret%'; +ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB +select count(*) from t2 FORCE INDEX (b) where b like 'secret%'; +ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB +select count(*) from t3 FORCE INDEX (b) where b like 'secret%'; +count(*) +1 +# Restart mysqld --file-key-management-filename=keys2.txt +drop table t1,t2,t3; diff --git a/mysql-test/suite/encryption/r/innodb-discard-import-change.result b/mysql-test/suite/encryption/r/innodb-discard-import-change.result index 51670d89e52..d2196794fa7 100644 --- a/mysql-test/suite/encryption/r/innodb-discard-import-change.result +++ b/mysql-test/suite/encryption/r/innodb-discard-import-change.result @@ -1,4 +1,6 @@ call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded"); +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_compression_algorithm = 1; create table t1(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=4; create table t2(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; diff --git a/mysql-test/suite/encryption/r/innodb-discard-import.result b/mysql-test/suite/encryption/r/innodb-discard-import.result index 91314a77177..06f4abab9f4 100644 --- a/mysql-test/suite/encryption/r/innodb-discard-import.result +++ b/mysql-test/suite/encryption/r/innodb-discard-import.result @@ -1,5 +1,7 @@ call mtr.add_suppression("InnoDB: Tablespace for table .* is set as discarded."); call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* because the .ibd file is missing. Please refer to .* for how to resolve the issue."); +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_compression_algorithm = 1; create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=4; show warnings; diff --git a/mysql-test/suite/encryption/r/innodb-encryption-alter.result b/mysql-test/suite/encryption/r/innodb-encryption-alter.result index 06e4172bbae..2003df0f4f4 100644 --- a/mysql-test/suite/encryption/r/innodb-encryption-alter.result +++ b/mysql-test/suite/encryption/r/innodb-encryption-alter.result @@ -1,3 +1,7 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_encrypt_tables = ON; SET GLOBAL innodb_encryption_threads = 4; CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT, c VARCHAR(256)) ENGINE=INNODB ENCRYPTED=NO ENCRYPTION_KEY_ID=4; @@ -49,3 +53,5 @@ Error 1005 Can't create table `test`.`#sql-temporary` (errno: 140 "Wrong create Warning 1030 Got error 140 "Wrong create options" from storage engine InnoDB set innodb_default_encryption_key_id = 1; drop table t1,t2; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html diff --git a/mysql-test/suite/encryption/r/innodb-encryption-disable.result b/mysql-test/suite/encryption/r/innodb-encryption-disable.result index d19124ab602..90668a3a395 100644 --- a/mysql-test/suite/encryption/r/innodb-encryption-disable.result +++ b/mysql-test/suite/encryption/r/innodb-encryption-disable.result @@ -1,4 +1,4 @@ -call mtr.add_suppression("InnoDB: The page .* in file test/t[15] cannot be decrypted"); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[15]\\.ibd' cannot be decrypted\\."); call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); create table t5 ( `intcol1` int(32) DEFAULT NULL, @@ -16,20 +16,8 @@ CREATE TABLE `t1` ( `charcol3` varchar(128) DEFAULT NULL ) ENGINE=InnoDB; insert into t1 values (1,2,'maria','db','encryption'); -select * from t1; -intcol1 intcol2 charcol1 charcol2 charcol3 -1 2 maria db encryption -select * from t5; -intcol1 intcol2 charcol1 charcol2 charcol3 -1 2 maria db encryption alter table t1 encrypted='yes' `encryption_key_id`=1; select * from t1; -intcol1 intcol2 charcol1 charcol2 charcol3 -1 2 maria db encryption -select * from t5; -intcol1 intcol2 charcol1 charcol2 charcol3 -1 2 maria db encryption -select * from t1; ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB select * from t5; ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB diff --git a/mysql-test/suite/encryption/r/innodb-force-corrupt.result b/mysql-test/suite/encryption/r/innodb-force-corrupt.result new file mode 100644 index 00000000000..67917ca5f82 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb-force-corrupt.result @@ -0,0 +1,23 @@ +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); +SET GLOBAL innodb_file_per_table = ON; +set global innodb_compression_algorithm = 1; +# Create and populate tables to be corrupted +CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT,c char(200)) ENGINE=InnoDB encrypted=yes; +CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT,c char(200)) ENGINE=InnoDB row_format=compressed encrypted=yes; +CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB page_compressed=yes encrypted=yes; +BEGIN; +INSERT INTO t1 (b,c) VALUES ('corrupt me','secret'); +INSERT INTO t1 (b,c) VALUES ('corrupt me','moresecretmoresecret'); +INSERT INTO t2 select * from t1; +INSERT INTO t3 select * from t1; +COMMIT; +# Backup tables before corrupting +# Corrupt tables +SELECT * FROM t1; +ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB +SELECT * FROM t2; +ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB +SELECT * FROM t3; +ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB +# Restore the original tables +DROP TABLE t1,t2,t3; diff --git a/mysql-test/suite/encryption/r/innodb-log-encrypt-crash.result b/mysql-test/suite/encryption/r/innodb-log-encrypt-crash.result new file mode 100644 index 00000000000..5310fb6ace2 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb-log-encrypt-crash.result @@ -0,0 +1,19 @@ +call mtr.add_suppression("InnoDB: New log files created, LSN=.*"); +call mtr.add_suppression("InnoDB: Creating foreign key constraint system tables."); +call mtr.add_suppression("InnoDB: Error: Table .*"); +CREATE TABLE t1 ( +pk bigint auto_increment, +col_int int, +col_int_key int, +col_char char(12), +col_char_key char(12), +primary key (pk), +key (`col_int_key` ), +key (`col_char_key` ) +) ENGINE=InnoDB; +CREATE TABLE t2 LIKE t1; +INSERT INTO t1 VALUES (NULL,1,1,'foo','foo'),(NULL,2,2,'bar','bar'),(NULL,3,3,'baz','baz'),(NULL,4,4,'qux','qux'); +INSERT INTO t2 +SELECT NULL, a1.col_int, a1.col_int_key, a1.col_char, a1.col_char_key +FROM t1 a1, t1 a2, t1 a3, t1 a4, t1 a5, t1 a6, t1 a7, t1 a8, t1 a9, t1 a10; +DROP TABLE t1, t2; diff --git a/mysql-test/suite/encryption/r/innodb-log-encrypt.result b/mysql-test/suite/encryption/r/innodb-log-encrypt.result new file mode 100644 index 00000000000..747f87d4cb7 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb-log-encrypt.result @@ -0,0 +1,57 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +SET GLOBAL innodb_file_per_table = ON; +create table t1(c1 bigint not null, b char(200), c varchar(200)) engine=innodb encrypted=yes encryption_key_id=1; +show warnings; +Level Code Message +create procedure innodb_insert_proc (repeat_count int) +begin +declare current_num int; +set current_num = 0; +while current_num < repeat_count do +insert into t1 values(current_num, substring(MD5(RAND()), -64), REPEAT('privatejanprivate',10)); +set current_num = current_num + 1; +end while; +end// +commit; +begin; +call innodb_insert_proc(2000); +commit; +update t1 set c1 = c1 +1; +select count(*) from t1; +count(*) +2000 +# Kill the server +# ibdata1 yes on expecting NOT FOUND +NOT FOUND /privatejanprivate/ in ibdata1 +# t1 yes on expecting NOT FOUND +NOT FOUND /privatejanprivate/ in t1.ibd +# log0 yes on expecting NOT FOUND +NOT FOUND /privatejanprivate/ in ib_logfile0 +# log1 yes on expecting NOT FOUND +NOT FOUND /privatejanprivate/ in ib_logfile1 +# Restart mysqld --innodb_encrypt_log=0 +insert into t1 values(5000, substring(MD5(RAND()), -64), REPEAT('publicmessage',10)); +insert into t1 values(5001, substring(MD5(RAND()), -64), REPEAT('publicmessage',10)); +insert into t1 values(5002, substring(MD5(RAND()), -64), REPEAT('publicmessage',10)); +insert into t1 values(5003, substring(MD5(RAND()), -64), REPEAT('publicmessage',10)); +insert into t1 values(5004, substring(MD5(RAND()), -64), REPEAT('publicmessage',10)); +# ibdata1 yes on expecting NOT FOUND +NOT FOUND /privatejanprivate/ in ibdata1 +# t1 yes on expecting NOT FOUND +NOT FOUND /privatejanprivate/ in t1.ibd +# log0 yes on expecting NOT FOUND +NOT FOUND /privatejanprivate/ in ib_logfile0 +# log1 yes on expecting NOT FOUND +NOT FOUND /privatejanprivate/ in ib_logfile1 +# ibdata1 yes on expecting NOT FOUND +NOT FOUND /publicmessage/ in ibdata1 +# t1 yes on expecting NOT FOUND +NOT FOUND /publicmessage/ in t1.ibd +# log0 no on expecting FOUND/NOTFOUND depending where insert goes +FOUND 48 /publicmessage/ in ib_logfile0 +# log1 no on expecting FOUND/NOTFOUND depending where insert goes +NOT FOUND /publicmessage/ in ib_logfile1 +drop procedure innodb_insert_proc; +drop table t1; diff --git a/mysql-test/suite/encryption/r/innodb-missing-key.result b/mysql-test/suite/encryption/r/innodb-missing-key.result index 2d2bc91c321..3eb48409f13 100644 --- a/mysql-test/suite/encryption/r/innodb-missing-key.result +++ b/mysql-test/suite/encryption/r/innodb-missing-key.result @@ -1,5 +1,4 @@ -call mtr.add_suppression("InnoDB: The page .* in file test/t. cannot be decrypted"); - +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); # Start server with keys2.txt CREATE TABLE t1(a int not null primary key auto_increment, b varchar(128)) engine=innodb ENCRYPTED=YES ENCRYPTION_KEY_ID=19; CREATE TABLE t2(a int not null primary key auto_increment, b varchar(128)) engine=innodb ENCRYPTED=YES ENCRYPTION_KEY_ID=1; diff --git a/mysql-test/suite/encryption/r/innodb-page_encryption-32k.result b/mysql-test/suite/encryption/r/innodb-page_encryption-32k.result index 72cd3e08931..6aa98a2fc68 100644 --- a/mysql-test/suite/encryption/r/innodb-page_encryption-32k.result +++ b/mysql-test/suite/encryption/r/innodb-page_encryption-32k.result @@ -1,13 +1,9 @@ +call mtr.add_suppression("InnoDB: Warning: innodb_page_size has been changed from default value *"); +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact encrypted=yes encryption_key_id=1; create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic encrypted=yes encryption_key_id=3; -ERROR HY000: Can't create table `test`.`innodb_dynamic` (errno: 140 "Wrong create options") -show warnings; -Level Code Message -Warning 140 InnoDB: ENCRYPTION_KEY_ID 3 not available -Error 1005 Can't create table `test`.`innodb_dynamic` (errno: 140 "Wrong create options") -Warning 1030 Got error 140 "Wrong create options" from storage engine InnoDB -create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic encrypted=yes encryption_key_id=33; create table innodb_redundant(c1 bigint not null, b char(200)) engine=innodb row_format=redundant encrypted=yes encryption_key_id=4; show create table innodb_compact; Table Create Table @@ -20,7 +16,7 @@ Table Create Table innodb_dynamic CREATE TABLE `innodb_dynamic` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC `encrypted`=yes `encryption_key_id`=33 +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC `encrypted`=yes `encryption_key_id`=3 show create table innodb_redundant; Table Create Table innodb_redundant CREATE TABLE `innodb_redundant` ( @@ -76,6 +72,9 @@ innodb_compact CREATE TABLE `innodb_compact` ( `b` char(200) DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT alter table innodb_dynamic engine=innodb encrypted=DEFAULT encryption_key_id=DEFAULT; +Warnings: +Warning 1478 InnoDB: ROW_FORMAT=DYNAMIC requires innodb_file_format > Antelope. +Warning 1478 InnoDB: assuming ROW_FORMAT=COMPACT. show create table innodb_dynamic; Table Create Table innodb_dynamic CREATE TABLE `innodb_dynamic` ( diff --git a/mysql-test/suite/encryption/r/innodb-page_encryption.result b/mysql-test/suite/encryption/r/innodb-page_encryption.result index 1069b8652da..051fd602db5 100644 --- a/mysql-test/suite/encryption/r/innodb-page_encryption.result +++ b/mysql-test/suite/encryption/r/innodb-page_encryption.result @@ -1,3 +1,7 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +SET GLOBAL innodb_file_per_table = ON; create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; show warnings; Level Code Message @@ -118,6 +122,10 @@ variable_value >= 0 SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; variable_value >= 0 1 +SET GLOBAL innodb_file_format = `Barracuda`; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +SET GLOBAL innodb_file_per_table = ON; update innodb_normal set c1 = c1 +1; update innodb_compact set c1 = c1 + 1; update innodb_compressed set c1 = c1 + 1; @@ -193,6 +201,10 @@ innodb_redundant CREATE TABLE `innodb_redundant` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=REDUNDANT +SET GLOBAL innodb_file_format = `Barracuda`; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +SET GLOBAL innodb_file_per_table = ON; show create table innodb_compact; Table Create Table innodb_compact CREATE TABLE `innodb_compact` ( @@ -269,3 +281,5 @@ drop table innodb_compressed; drop table innodb_dynamic; drop table innodb_redundant; drop table innodb_defkey; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html diff --git a/mysql-test/suite/encryption/r/innodb-page_encryption_compression.result b/mysql-test/suite/encryption/r/innodb-page_encryption_compression.result index a5bd7da8421..e4555bd6c8c 100644 --- a/mysql-test/suite/encryption/r/innodb-page_encryption_compression.result +++ b/mysql-test/suite/encryption/r/innodb-page_encryption_compression.result @@ -1,3 +1,5 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb page_compressed=1; show warnings; @@ -36,73 +38,14 @@ set current_num = current_num + 1; end while; end// commit; -set autocommit=0; -call innodb_insert_proc(5000); -commit; -set autocommit=1; +begin; +call innodb_insert_proc(2000); insert into innodb_compact select * from innodb_normal; insert into innodb_dynamic select * from innodb_normal; -update innodb_normal set c1 = c1 + 1; -update innodb_compact set c1 = c1 + 1; -update innodb_dynamic set c1 = c1 + 1; -select count(*) from innodb_normal; -count(*) -5000 -select count(*) from innodb_compact where c1 < 1500000; -count(*) -5000 -select count(*) from innodb_dynamic where c1 < 1500000; -count(*) -5000 -select count(*) from innodb_compact t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -count(*) -5000 -select count(*) from innodb_dynamic t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -count(*) -5000 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -variable_value >= 0 -1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; -variable_value >= 0 -1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_compressed'; -variable_value >= 0 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decompressed'; -variable_value >= 0 +commit; +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; -update innodb_normal set c1 = c1 + 1; -update innodb_compact set c1 = c1 + 1; -update innodb_dynamic set c1 = c1 + 1; -select count(*) from innodb_normal; -count(*) -5000 -select count(*) from innodb_compact where c1 < 1500000; -count(*) -5000 -select count(*) from innodb_dynamic where c1 < 1500000; -count(*) -5000 -select count(*) from innodb_compact t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -count(*) -5000 -select count(*) from innodb_dynamic t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -count(*) -5000 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -variable_value >= 0 -1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; -variable_value >= 0 -1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_compressed'; -variable_value >= 0 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decompressed'; -variable_value >= 0 alter table innodb_normal engine=innodb page_compressed=DEFAULT; show create table innodb_normal; Table Create Table @@ -124,55 +67,9 @@ innodb_dynamic CREATE TABLE `innodb_dynamic` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC -show create table innodb_normal; -Table Create Table -innodb_normal CREATE TABLE `innodb_normal` ( - `c1` bigint(20) NOT NULL, - `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 -show create table innodb_compact; -Table Create Table -innodb_compact CREATE TABLE `innodb_compact` ( - `c1` bigint(20) NOT NULL, - `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT -show create table innodb_dynamic; -Table Create Table -innodb_dynamic CREATE TABLE `innodb_dynamic` ( - `c1` bigint(20) NOT NULL, - `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC -update innodb_normal set c1 = c1 + 1; -update innodb_compact set c1 = c1 + 1; -update innodb_dynamic set c1 = c1 + 1; -select count(*) from innodb_normal; -count(*) -5000 -select count(*) from innodb_compact where c1 < 1500000; -count(*) -5000 -select count(*) from innodb_dynamic where c1 < 1500000; -count(*) -5000 -select count(*) from innodb_compact t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -count(*) -5000 -select count(*) from innodb_dynamic t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -count(*) -5000 -SELECT variable_value = 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -variable_value = 0 -1 -SELECT variable_value = 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; -variable_value = 0 -1 -SELECT variable_value = 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_compressed'; -variable_value = 0 -SELECT variable_value = 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decompressed'; -variable_value = 0 drop procedure innodb_insert_proc; drop table innodb_normal; drop table innodb_compact; drop table innodb_dynamic; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html diff --git a/mysql-test/suite/encryption/r/innodb-page_encryption_log_encryption.result b/mysql-test/suite/encryption/r/innodb-page_encryption_log_encryption.result index ce74ac52537..78ae0ec6b00 100644 --- a/mysql-test/suite/encryption/r/innodb-page_encryption_log_encryption.result +++ b/mysql-test/suite/encryption/r/innodb-page_encryption_log_encryption.result @@ -1,6 +1,5 @@ -call mtr.add_suppression("KeyID 0 not found or with error. Check the key and the log*"); -call mtr.add_suppression("Disabling redo log encryp*"); -call mtr.add_suppression("InnoDB: Redo log crypto: Can't initialize to key version*"); +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; show warnings; Level Code Message @@ -50,92 +49,33 @@ set current_num = current_num + 1; end while; end// commit; -set autocommit=0; +begin; call innodb_insert_proc(2000); -commit; -set autocommit=1; insert into innodb_compact select * from innodb_normal; insert into innodb_compressed select * from innodb_normal; insert into innodb_dynamic select * from innodb_normal; insert into innodb_redundant select * from innodb_normal; -update innodb_normal set c1 = c1 +1; -update innodb_compact set c1 = c1 + 1; -update innodb_compressed set c1 = c1 + 1; -update innodb_dynamic set c1 = c1 + 1; -update innodb_redundant set c1 = c1 + 1; -select count(*) from innodb_compact where c1 < 1500000; -count(*) -2000 -select count(*) from innodb_compressed where c1 < 1500000; -count(*) -2000 -select count(*) from innodb_dynamic where c1 < 1500000; -count(*) -2000 -select count(*) from innodb_redundant where c1 < 1500000; -count(*) -2000 -select count(*) from innodb_compact t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -count(*) -2000 -select count(*) from innodb_dynamic t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -count(*) -2000 -select count(*) from innodb_compressed t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -count(*) -2000 -select count(*) from innodb_redundant t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -count(*) -2000 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -variable_value >= 0 +commit; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; +variable_value > 0 1 SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; variable_value >= 0 1 +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; update innodb_normal set c1 = c1 +1; update innodb_compact set c1 = c1 + 1; update innodb_compressed set c1 = c1 + 1; update innodb_dynamic set c1 = c1 + 1; update innodb_redundant set c1 = c1 + 1; -select count(*) from innodb_compact where c1 < 1500000; -count(*) -2000 -select count(*) from innodb_compressed where c1 < 1500000; -count(*) -2000 -select count(*) from innodb_dynamic where c1 < 1500000; -count(*) -2000 -select count(*) from innodb_redundant where c1 < 1500000; -count(*) -2000 -select count(*) from innodb_compact t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -count(*) -2000 -select count(*) from innodb_dynamic t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -count(*) -2000 -select count(*) from innodb_compressed t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -count(*) -2000 -select count(*) from innodb_redundant t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -count(*) -2000 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -variable_value >= 0 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; +variable_value > 0 1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; -variable_value >= 0 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; +variable_value > 0 1 +SET GLOBAL innodb_encrypt_tables=OFF; alter table innodb_compact engine=innodb encrypted=DEFAULT encryption_key_id=DEFAULT; show create table innodb_compact; Table Create Table @@ -164,63 +104,6 @@ innodb_redundant CREATE TABLE `innodb_redundant` ( `c1` bigint(20) NOT NULL, `b` char(200) DEFAULT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=REDUNDANT -show create table innodb_compact; -Table Create Table -innodb_compact CREATE TABLE `innodb_compact` ( - `c1` bigint(20) NOT NULL, - `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPACT -show create table innodb_compressed; -Table Create Table -innodb_compressed CREATE TABLE `innodb_compressed` ( - `c1` bigint(20) NOT NULL, - `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPRESSED -show create table innodb_dynamic; -Table Create Table -innodb_dynamic CREATE TABLE `innodb_dynamic` ( - `c1` bigint(20) NOT NULL, - `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=DYNAMIC -show create table innodb_redundant; -Table Create Table -innodb_redundant CREATE TABLE `innodb_redundant` ( - `c1` bigint(20) NOT NULL, - `b` char(200) DEFAULT NULL -) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=REDUNDANT -update innodb_normal set c1 = c1 +1; -update innodb_compact set c1 = c1 + 1; -update innodb_compressed set c1 = c1 + 1; -update innodb_dynamic set c1 = c1 + 1; -update innodb_redundant set c1 = c1 + 1; -select count(*) from innodb_compact where c1 < 1500000; -count(*) -2000 -select count(*) from innodb_compressed where c1 < 1500000; -count(*) -2000 -select count(*) from innodb_dynamic where c1 < 1500000; -count(*) -2000 -select count(*) from innodb_redundant where c1 < 1500000; -count(*) -2000 -select count(*) from innodb_compact t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -count(*) -2000 -select count(*) from innodb_dynamic t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -count(*) -2000 -select count(*) from innodb_compressed t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -count(*) -2000 -select count(*) from innodb_redundant t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -count(*) -2000 SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; variable_value >= 0 1 @@ -233,6 +116,10 @@ drop table innodb_compact; drop table innodb_compressed; drop table innodb_dynamic; drop table innodb_redundant; +SET GLOBAL innodb_file_format = `Barracuda`; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +SET GLOBAL innodb_file_per_table = ON; CREATE TABLE t1 (pk INT PRIMARY KEY) ENGINE=InnoDB ENCRYPTION_KEY_ID=2 ENCRYPTED=YES; INSERT INTO t1 VALUES (1),(2); # Restarting server... diff --git a/mysql-test/suite/encryption/r/innodb-redo-badkey.result b/mysql-test/suite/encryption/r/innodb-redo-badkey.result new file mode 100644 index 00000000000..e12dad6d0d7 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb-redo-badkey.result @@ -0,0 +1,45 @@ +call mtr.add_suppression("InnoDB: Block in space_id .* in file .* encrypted."); +call mtr.add_suppression("Plugin 'file_key_management' .*"); +call mtr.add_suppression("Plugin 'InnoDB' init function returned error."); +call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed."); +call mtr.add_suppression("InnoDB: Read operation failed for tablespace .*"); +call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*"); +call mtr.add_suppression("InnoDB: Recovery read page .*"); +call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t4 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); +call mtr.add_suppression("InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE .*"); +call mtr.add_suppression("InnoDB: Plugin initialization aborted .*"); +call mtr.add_suppression("InnoDB: ############### CORRUPT LOG RECORD FOUND ##################"); +# Restart mysqld --file-key-management-filename=keys2.txt +# Wait max 10 min for key encryption threads to encrypt all spaces +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +create table t1(a int not null primary key auto_increment, c char(250), b blob, index(b(10))) engine=innodb row_format=compressed encrypted=yes encryption_key_id=4; +create table t2(a int not null primary key auto_increment, c char(250), b blob, index(b(10))) engine=innodb row_format=compressed; +create table t3(a int not null primary key auto_increment, c char(250), b blob, index(b(10))) engine=innodb encrypted=yes encryption_key_id=4; +create table t4(a int not null primary key auto_increment, c char(250), b blob, index(b(10))) engine=innodb; +begin; +insert into t2 select * from t1; +insert into t3 select * from t1; +insert into t4 select * from t1; +commit; +SET GLOBAL innodb_flush_log_at_trx_commit=1; +begin; +update t1 set c = repeat('secret3', 20); +update t2 set c = repeat('secret4', 20); +update t3 set c = repeat('secret4', 20); +update t4 set c = repeat('secret4', 20); +insert into t1 (c,b) values (repeat('secret5',20), repeat('secret6',6000)); +insert into t2 (c,b) values (repeat('secret7',20), repeat('secret8',6000)); +insert into t3 (c,b) values (repeat('secret9',20), repeat('secre10',6000)); +insert into t4 (c,b) values (repeat('secre11',20), repeat('secre12',6000)); +COMMIT; +# Kill the server +# restart +# Kill the server +# Restart mysqld --innodb-force-recovery=1 +# Kill the server +# Restart mysqld --file-key-management-filename=keys2.txt +drop table t1, t2,t3,t4; diff --git a/mysql-test/suite/encryption/r/innodb-redo-nokeys.result b/mysql-test/suite/encryption/r/innodb-redo-nokeys.result new file mode 100644 index 00000000000..dcbe1f5a395 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb-redo-nokeys.result @@ -0,0 +1,37 @@ +call mtr.add_suppression("InnoDB: Block in space_id .*"); +call mtr.add_suppression("mysqld: File .*"); +call mtr.add_suppression("Plugin 'file_key_management' .*"); +call mtr.add_suppression("InnoDB: cannot enable encryption, encryption plugin is not available"); +call mtr.add_suppression("Plugin 'InnoDB' init function returned error."); +call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed."); +call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t4 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); +# Restart mysqld --file-key-management-filename=keys2.txt +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +create table t1(a int not null primary key auto_increment, c char(200), b blob, index(b(10))) engine=innodb row_format=compressed encrypted=yes encryption_key_id=20; +create table t2(a int not null primary key auto_increment, c char(200), b blob, index(b(10))) engine=innodb row_format=compressed; +create table t3(a int not null primary key auto_increment, c char(200), b blob, index(b(10))) engine=innodb encrypted=yes encryption_key_id=20; +create table t4(a int not null primary key auto_increment, c char(200), b blob, index(b(10))) engine=innodb; +begin; +insert into t2 select * from t1; +insert into t3 select * from t1; +insert into t4 select * from t1; +commit; +SET GLOBAL innodb_flush_log_at_trx_commit=1; +begin; +update t1 set c = repeat('secret3', 20); +update t2 set c = repeat('secret4', 20); +update t3 set c = repeat('secret4', 20); +update t4 set c = repeat('secret4', 20); +insert into t1 (c,b) values (repeat('secret5',20), repeat('secret6',6000)); +insert into t2 (c,b) values (repeat('secret7',20), repeat('secret8',6000)); +insert into t3 (c,b) values (repeat('secret9',20), repeat('secre10',6000)); +insert into t4 (c,b) values (repeat('secre11',20), repeat('secre12',6000)); +COMMIT; +# Kill the server +# restart +# Restart mysqld --file-key-management-filename=keys2.txt +drop table t1, t2,t3,t4; diff --git a/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result b/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result index 60b9ff57605..ec92825ac8e 100644 --- a/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result +++ b/mysql-test/suite/encryption/r/innodb_encryption-page-compression.result @@ -1,3 +1,5 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_encryption_threads = 4; SET GLOBAL innodb_encrypt_tables = on; set global innodb_compression_algorithm = 1; @@ -95,13 +97,8 @@ set current_num = current_num + 1; end while; end// commit; -set autocommit=0; -call innodb_insert_proc(5000); -commit; -set autocommit=1; -select count(*) from innodb_normal; -count(*) -5000 +begin; +call innodb_insert_proc(2000); insert into innodb_page_compressed1 select * from innodb_normal; insert into innodb_page_compressed2 select * from innodb_normal; insert into innodb_page_compressed3 select * from innodb_normal; @@ -114,120 +111,45 @@ insert into innodb_page_compressed9 select * from innodb_normal; commit; select count(*) from innodb_page_compressed1 where c1 < 500000; count(*) -5000 -select count(*) from innodb_page_compressed2 where c1 < 500000; -count(*) -5000 -select count(*) from innodb_page_compressed3 where c1 < 500000; -count(*) -5000 -select count(*) from innodb_page_compressed4 where c1 < 500000; -count(*) -5000 -select count(*) from innodb_page_compressed5 where c1 < 500000; -count(*) -5000 -select count(*) from innodb_page_compressed6 where c1 < 500000; -count(*) -5000 -select count(*) from innodb_page_compressed7 where c1 < 500000; -count(*) -5000 -select count(*) from innodb_page_compressed8 where c1 < 500000; -count(*) -5000 -select count(*) from innodb_page_compressed9 where c1 < 500000; -count(*) -5000 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -variable_value >= 0 -1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; -variable_value >= 0 -1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed'; -variable_value >= 0 -1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed'; -variable_value >= 0 -1 -select count(*) from innodb_page_compressed1 where c1 < 500000; -count(*) -5000 +2000 select count(*) from innodb_page_compressed2 where c1 < 500000; count(*) -5000 +2000 select count(*) from innodb_page_compressed3 where c1 < 500000; count(*) -5000 +2000 select count(*) from innodb_page_compressed4 where c1 < 500000; count(*) -5000 +2000 select count(*) from innodb_page_compressed5 where c1 < 500000; count(*) -5000 +2000 select count(*) from innodb_page_compressed6 where c1 < 500000; count(*) -5000 +2000 select count(*) from innodb_page_compressed7 where c1 < 500000; count(*) -5000 +2000 select count(*) from innodb_page_compressed8 where c1 < 500000; count(*) -5000 +2000 select count(*) from innodb_page_compressed9 where c1 < 500000; count(*) -5000 -update innodb_page_compressed1 set c1 = c1 + 1; -update innodb_page_compressed2 set c1 = c1 + 1; -update innodb_page_compressed3 set c1 = c1 + 1; -update innodb_page_compressed4 set c1 = c1 + 1; -update innodb_page_compressed5 set c1 = c1 + 1; -update innodb_page_compressed6 set c1 = c1 + 1; -update innodb_page_compressed7 set c1 = c1 + 1; -update innodb_page_compressed8 set c1 = c1 + 1; -update innodb_page_compressed9 set c1 = c1 + 1; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -variable_value >= 0 +2000 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; +variable_value > 0 1 SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; variable_value >= 0 1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed'; -variable_value >= 0 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed'; +variable_value > 0 1 SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed'; variable_value >= 0 1 SET GLOBAL innodb_encryption_threads = 4; SET GLOBAL innodb_encrypt_tables = off; -select count(*) from innodb_page_compressed1 where c1 < 500000; -count(*) -5000 -select count(*) from innodb_page_compressed2 where c1 < 500000; -count(*) -5000 -select count(*) from innodb_page_compressed3 where c1 < 500000; -count(*) -5000 -select count(*) from innodb_page_compressed4 where c1 < 500000; -count(*) -5000 -select count(*) from innodb_page_compressed5 where c1 < 500000; -count(*) -5000 -select count(*) from innodb_page_compressed6 where c1 < 500000; -count(*) -5000 -select count(*) from innodb_page_compressed7 where c1 < 500000; -count(*) -5000 -select count(*) from innodb_page_compressed8 where c1 < 500000; -count(*) -5000 -select count(*) from innodb_page_compressed9 where c1 < 500000; -count(*) -5000 update innodb_page_compressed1 set c1 = c1 + 1; update innodb_page_compressed2 set c1 = c1 + 1; update innodb_page_compressed3 set c1 = c1 + 1; @@ -240,11 +162,11 @@ update innodb_page_compressed9 set c1 = c1 + 1; SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; variable_value >= 0 1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; -variable_value >= 0 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; +variable_value > 0 1 -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed'; -variable_value >= 0 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed'; +variable_value > 0 1 SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed'; variable_value >= 0 diff --git a/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result b/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result index 91581eb8837..d08a6e943f4 100644 --- a/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result +++ b/mysql-test/suite/encryption/r/innodb_encryption_discard_import.result @@ -1,5 +1,6 @@ -call mtr.add_suppression("InnoDB: Tablespace for table .* is set as discarded."); -call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* because the .ibd file is missing. Please refer to .* for how to resolve the issue."); +call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded."); +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; CREATE TABLE t1 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes; CREATE TABLE t2 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB; CREATE TABLE t3 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB row_format=compressed encrypted=yes; diff --git a/mysql-test/suite/encryption/r/innodb_encryption_filekeys.result b/mysql-test/suite/encryption/r/innodb_encryption_filekeys.result index 54cdf842b20..576b44fe897 100644 --- a/mysql-test/suite/encryption/r/innodb_encryption_filekeys.result +++ b/mysql-test/suite/encryption/r/innodb_encryption_filekeys.result @@ -1,4 +1,8 @@ call mtr.add_suppression("trying to do an operation on a dropped tablespace .*"); +SET GLOBAL innodb_file_format = `Barracuda`; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_encrypt_tables = OFF; SET GLOBAL innodb_encryption_threads = 4; CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT, c VARCHAR(256)) ENGINE=INNODB; @@ -61,5 +65,7 @@ COUNT(1) SELECT COUNT(1) FROM t5; COUNT(1) 400 +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html drop table t1,t2,t3,t4, t5; set GLOBAL innodb_default_encryption_key_id=1; diff --git a/mysql-test/suite/encryption/r/innodb_encryption_is.result b/mysql-test/suite/encryption/r/innodb_encryption_is.result index 0ce4a11f1ea..591c5a84ccc 100644 --- a/mysql-test/suite/encryption/r/innodb_encryption_is.result +++ b/mysql-test/suite/encryption/r/innodb_encryption_is.result @@ -1,3 +1,7 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +SET GLOBAL innodb_file_per_table = ON; CREATE TABLE t1 (c VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=1; CREATE TABLE t2 (c VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=2; INSERT INTO t1 VALUES ('foobar'); @@ -10,3 +14,5 @@ NAME ENCRYPTION_SCHEME MIN_KEY_VERSION CURRENT_KEY_VERSION CURRENT_KEY_ID test/t1 1 1 1 1 test/t2 1 1 1 2 DROP TABLE t1, t2; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html diff --git a/mysql-test/suite/encryption/r/innodb_encryption_row_compressed.result b/mysql-test/suite/encryption/r/innodb_encryption_row_compressed.result index 3d616ee71e3..e49e38a8f3f 100644 --- a/mysql-test/suite/encryption/r/innodb_encryption_row_compressed.result +++ b/mysql-test/suite/encryption/r/innodb_encryption_row_compressed.result @@ -1,3 +1,7 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +SET GLOBAL innodb_file_per_table = ON; create table innodb_compressed1(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed encrypted=yes; create table innodb_compressed2(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed key_block_size=1 encrypted=yes; create table innodb_compressed3(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed key_block_size=2 encrypted=yes; @@ -23,6 +27,10 @@ NOT FOUND /private/ in innodb_compressed2.ibd NOT FOUND /private/ in innodb_compressed3.ibd # t4 yes on expecting NOT FOUND NOT FOUND /private/ in innodb_compressed4.ibd +SET GLOBAL innodb_file_format = `Barracuda`; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +SET GLOBAL innodb_file_per_table = ON; select * from innodb_compressed1 where d = 20; c1 d a b 1 20 private evenmoreprivate @@ -147,3 +155,5 @@ drop table innodb_compressed1; drop table innodb_compressed2; drop table innodb_compressed3; drop table innodb_compressed4; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html diff --git a/mysql-test/suite/encryption/r/innodb_encryption_tables.result b/mysql-test/suite/encryption/r/innodb_encryption_tables.result index e7bcc207612..da62c0a2f0e 100644 --- a/mysql-test/suite/encryption/r/innodb_encryption_tables.result +++ b/mysql-test/suite/encryption/r/innodb_encryption_tables.result @@ -1,3 +1,7 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +SET GLOBAL innodb_file_per_table = ON; create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact; create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic; @@ -101,6 +105,10 @@ variable_value >= 0 SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed'; variable_value >= 0 1 +SET GLOBAL innodb_file_format = `Barracuda`; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +SET GLOBAL innodb_file_per_table = ON; update innodb_normal set c1 = c1 + 1; update innodb_compact set c1 = c1 + 1; update innodb_dynamic set c1 = c1 + 1; @@ -155,3 +163,5 @@ drop table innodb_compact; drop table innodb_dynamic; drop table innodb_compressed; drop table innodb_redundant; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html diff --git a/mysql-test/suite/encryption/r/innodb_first_page.result b/mysql-test/suite/encryption/r/innodb_first_page.result index aaef462e639..c2bbdb5695b 100644 --- a/mysql-test/suite/encryption/r/innodb_first_page.result +++ b/mysql-test/suite/encryption/r/innodb_first_page.result @@ -1,2 +1,5 @@ +call mtr.add_suppression("InnoDB: New log files created, LSN"); +call mtr.add_suppression("InnoDB: Creating foreign key constraint system tables"); +shutdown; create table t1 (a int); drop table t1; diff --git a/mysql-test/suite/encryption/r/innodb_lotoftables.result b/mysql-test/suite/encryption/r/innodb_lotoftables.result index cf5724b527a..a5a0f539c16 100644 --- a/mysql-test/suite/encryption/r/innodb_lotoftables.result +++ b/mysql-test/suite/encryption/r/innodb_lotoftables.result @@ -1,4 +1,5 @@ SET GLOBAL innodb_fast_shutdown=0; +SET GLOBAL innodb_file_per_table = ON; SHOW VARIABLES LIKE 'innodb_encrypt%'; Variable_name Value innodb_encrypt_log OFF diff --git a/mysql-test/suite/encryption/r/innodb_onlinealter_encryption.result b/mysql-test/suite/encryption/r/innodb_onlinealter_encryption.result index 26765229a2e..6f2de43fed7 100644 --- a/mysql-test/suite/encryption/r/innodb_onlinealter_encryption.result +++ b/mysql-test/suite/encryption/r/innodb_onlinealter_encryption.result @@ -1,3 +1,5 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; CREATE TABLE t1 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes; CREATE TABLE t2 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB; CREATE TABLE t3 (id INT, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes; @@ -10,8 +12,8 @@ begin declare current_num int; set current_num = 0; while current_num < repeat_count do -insert into t1 values (current_num,repeat('foobar',42)); -insert into t2 values (current_num,repeat('temp', 42)); +insert into t1 values (current_num,repeat('foobar',12)); +insert into t2 values (current_num,repeat('tempsecret', 12)); insert into t3 values (current_num,repeat('barfoo',42)); insert into t4 values (current_num,repeat('repeat',42)); insert into t5 values (current_num,substring('A BC DEF GHIJ KLM NOPQRS TUV WXYZ 012 3456789', rand()*36+1, 100), repeat('author new',22)); @@ -29,7 +31,7 @@ set autocommit=1; # t1 yes on expecting NOT FOUND NOT FOUND /foobar/ in t1.ibd # t2 ... on expecting NOT FOUND -NOT FOUND /temp/ in t2.ibd +NOT FOUND /tempsecret/ in t2.ibd # t3 ... on expecting NOT FOUND NOT FOUND /barfoo/ in t3.ibd # t4 ... on expecting NOT FOUND @@ -124,7 +126,7 @@ t7 CREATE TABLE `t7` ( # t1 yes on expecting NOT FOUND NOT FOUND /foobar/ in t1.ibd # t2 ... on expecting NOT FOUND -NOT FOUND /temp/ in t2.ibd +NOT FOUND /tempsecret/ in t2.ibd # t3 ... on expecting NOT FOUND NOT FOUND /barfoo/ in t3.ibd # t4 ... on expecting NOT FOUND diff --git a/mysql-test/suite/encryption/r/innodb_page_encryption_key_change.result b/mysql-test/suite/encryption/r/innodb_page_encryption_key_change.result index d3406e67f18..22038c0e933 100644 --- a/mysql-test/suite/encryption/r/innodb_page_encryption_key_change.result +++ b/mysql-test/suite/encryption/r/innodb_page_encryption_key_change.result @@ -1,4 +1,8 @@ # Restart mysqld --loose-file-key-management-filename=keys2.txt +SET GLOBAL innodb_file_format = `Barracuda`; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +SET GLOBAL innodb_file_per_table = ON; create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; show warnings; Level Code Message @@ -102,6 +106,10 @@ variable_value >= 0 SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; variable_value >= 0 1 +SET GLOBAL innodb_file_format = `Barracuda`; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +SET GLOBAL innodb_file_per_table = ON; alter table innodb_compact engine=innodb encryption_key_id = 2; alter table innodb_compressed engine=innodb encryption_key_id = 3; alter table innodb_dynamic engine=innodb encryption_key_id = 4; @@ -147,3 +155,5 @@ drop table innodb_compact; drop table innodb_compressed; drop table innodb_dynamic; drop table innodb_redundant; +Warnings: +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html diff --git a/mysql-test/suite/encryption/r/innodb_scrub.result b/mysql-test/suite/encryption/r/innodb_scrub.result index e3dd48b2c51..6e8febc762d 100644 --- a/mysql-test/suite/encryption/r/innodb_scrub.result +++ b/mysql-test/suite/encryption/r/innodb_scrub.result @@ -1,6 +1,6 @@ create table snapshot_status engine = myisam select * from information_schema.global_status -where variable_name like 'innodb_scrub_background%'; +where variable_name like 'innodb_scrub%'; # MDEV-8139 Fix scrubbing tests # FIXME: Add index(b) to each table; ensure that undo logs are scrubbed. create table delete_3 ( diff --git a/mysql-test/suite/encryption/r/tempfiles.result b/mysql-test/suite/encryption/r/tempfiles.result index e335e644400..a0b7596dd5a 100644 --- a/mysql-test/suite/encryption/r/tempfiles.result +++ b/mysql-test/suite/encryption/r/tempfiles.result @@ -26,7 +26,6 @@ i 10 j 10 drop table t1; set global binlog_cache_size=8192; -connect con1, localhost, root; create table t1 (a text) engine=innodb; start transaction; insert t1 select repeat(seq, 1000) from seq_1_to_15; @@ -35,8 +34,6 @@ start transaction; insert t1 select repeat(seq, 1000) from seq_1_to_8; commit; drop table t1; -disconnect con1; -connect con2, localhost, root; create table t1 (a text) engine=innodb; start transaction; insert t1 select repeat(seq, 1000) from seq_1_to_15; @@ -46,6 +43,4 @@ rollback to savepoint foo; insert t1 select repeat(seq, 1000) from seq_31_to_40; commit; drop table t1; -disconnect con2; -connection default; set global binlog_cache_size=default; diff --git a/mysql-test/suite/encryption/t/create_or_replace.test b/mysql-test/suite/encryption/t/create_or_replace.test index 98d5b7cc017..3b2970e5162 100644 --- a/mysql-test/suite/encryption/t/create_or_replace.test +++ b/mysql-test/suite/encryption/t/create_or_replace.test @@ -7,9 +7,6 @@ # # MDEV-8164: Server crashes in pfs_mutex_enter_func after fil_crypt_is_closing or alike # - -call mtr.add_suppression("InnoDB: Error: trying to do an operation on a dropped tablespace.*"); - SET default_storage_engine = InnoDB; CREATE TABLE t1 (pk INT PRIMARY KEY, c VARCHAR(256)); diff --git a/mysql-test/suite/encryption/t/encrypt_and_grep.opt b/mysql-test/suite/encryption/t/encrypt_and_grep.opt index 5c9aaf65b06..c50ec719307 100644 --- a/mysql-test/suite/encryption/t/encrypt_and_grep.opt +++ b/mysql-test/suite/encryption/t/encrypt_and_grep.opt @@ -1,7 +1,7 @@ --innodb-encrypt-tables=ON --innodb-encrypt-log=ON --innodb-encryption-rotate-key-age=15 ---innodb-encryption-threads=1 +--innodb-encryption-threads=4 --innodb-tablespaces-encryption diff --git a/mysql-test/suite/encryption/t/encrypt_and_grep.test b/mysql-test/suite/encryption/t/encrypt_and_grep.test index 2ef69db237d..c5a10620f6e 100644 --- a/mysql-test/suite/encryption/t/encrypt_and_grep.test +++ b/mysql-test/suite/encryption/t/encrypt_and_grep.test @@ -14,15 +14,19 @@ --let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd --let SEARCH_RANGE = 10000000 +--disable_warnings SET GLOBAL innodb_file_per_table = ON; +SET GLOBAL innodb_file_format = `Barracuda`; +--enable_warnings create table t1 (a varchar(255)) engine=innodb encrypted=yes; create table t2 (a varchar(255)) engine=innodb; +show warnings; create table t3 (a varchar(255)) engine=innodb encrypted=no; -insert t1 values (repeat('foobar', 42)); -insert t2 values (repeat('temp', 42)); -insert t3 values (repeat('dummy', 42)); +insert t1 values (repeat('foobarsecret', 12)); +insert t2 values (repeat('tempsecret', 12)); +insert t3 values (repeat('dummysecret', 12)); --echo # Wait max 10 min for key encryption threads to encrypt all spaces --let $wait_timeout= 600 @@ -34,19 +38,19 @@ SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_ --source include/shutdown_mysqld.inc ---let SEARCH_PATTERN=foobar +--let SEARCH_PATTERN=foobarsecret --echo # t1 yes on expecting NOT FOUND -- let SEARCH_FILE=$t1_IBD -- source include/search_pattern_in_file.inc ---let SEARCH_PATTERN=temp +--let SEARCH_PATTERN=tempsecret --echo # t2 ... on expecting NOT FOUND -- let SEARCH_FILE=$t2_IBD -- source include/search_pattern_in_file.inc ---let SEARCH_PATTERN=dummy +--let SEARCH_PATTERN=dummysecret --echo # t3 no on expecting FOUND -- let SEARCH_FILE=$t3_IBD -- source include/search_pattern_in_file.inc ---let SEARCH_PATTERN=foobar +--let SEARCH_PATTERN=foobarsecret --echo # ibdata1 expecting NOT FOUND -- let SEARCH_FILE=$ib1_IBD -- source include/search_pattern_in_file.inc @@ -54,12 +58,12 @@ SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_ -- source include/start_mysqld.inc --echo # Now turn off encryption and wait for threads to decrypt everything -SET GLOBAL innodb_encryption_threads = 1; + SET GLOBAL innodb_encrypt_tables = off; --echo # Wait max 10 min for key encryption threads to decrypt all spaces --let $wait_timeout= 600 ---let $wait_condition=SELECT COUNT(*) = 5 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND KEY_ROTATION_PAGE_NUMBER IS NULL; +--let $wait_condition=SELECT COUNT(*) = 5 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND CURRENT_KEY_VERSION = 0; --source include/wait_condition.inc SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0; @@ -67,27 +71,27 @@ SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_ --source include/shutdown_mysqld.inc ---let SEARCH_PATTERN=foobar +--let SEARCH_PATTERN=foobarsecret --echo # t1 yes on expecting NOT FOUND -- let SEARCH_FILE=$t1_IBD -- source include/search_pattern_in_file.inc ---let SEARCH_PATTERN=temp ---echo # t2 ... on expecting FOUND +--let SEARCH_PATTERN=tempsecret +--echo # t2 ... default expecting FOUND -- let SEARCH_FILE=$t2_IBD -- source include/search_pattern_in_file.inc ---let SEARCH_PATTERN=dummy +--let SEARCH_PATTERN=dummysecret --echo # t3 no on expecting FOUND -- let SEARCH_FILE=$t3_IBD -- source include/search_pattern_in_file.inc ---let SEARCH_PATTERN=foobar +--let SEARCH_PATTERN=foobarsecret --echo # ibdata1 expecting NOT FOUND -- let SEARCH_FILE=$ib1_IBD -- source include/search_pattern_in_file.inc + -- source include/start_mysqld.inc --echo # Now turn on encryption and wait for threads to encrypt all spaces -SET GLOBAL innodb_encryption_threads = 4; SET GLOBAL innodb_encrypt_tables = on; --echo # Wait max 10 min for key encryption threads to encrypt all spaces @@ -100,19 +104,19 @@ SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_ --source include/shutdown_mysqld.inc ---let SEARCH_PATTERN=foobar +--let SEARCH_PATTERN=foobarsecret --echo # t1 yes on expecting NOT FOUND -- let SEARCH_FILE=$t1_IBD -- source include/search_pattern_in_file.inc ---let SEARCH_PATTERN=temp +--let SEARCH_PATTERN=tempsecret --echo # t2 ... on expecting NOT FOUND -- let SEARCH_FILE=$t2_IBD -- source include/search_pattern_in_file.inc ---let SEARCH_PATTERN=dummy +--let SEARCH_PATTERN=dummysecret --echo # t3 no on expecting FOUND -- let SEARCH_FILE=$t3_IBD -- source include/search_pattern_in_file.inc ---let SEARCH_PATTERN=foobar +--let SEARCH_PATTERN=foobarsecret --echo # ibdata1 expecting NOT FOUND -- let SEARCH_FILE=$ib1_IBD -- source include/search_pattern_in_file.inc diff --git a/mysql-test/suite/encryption/t/filekeys_badtest.inc b/mysql-test/suite/encryption/t/filekeys_badtest.inc index 60ac9f0e798..7c0858af20a 100644 --- a/mysql-test/suite/encryption/t/filekeys_badtest.inc +++ b/mysql-test/suite/encryption/t/filekeys_badtest.inc @@ -1,5 +1,5 @@ -- source include/not_embedded.inc --- source include/have_innodb.inc +-- source include/have_xtradb.inc -- source filekeys_plugin.inc --eval call mtr.add_suppression("$SEARCH_PATTERN") @@ -7,6 +7,7 @@ call mtr.add_suppression("Plugin 'file_key_management' init function returned er call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); --let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err +--let SEARCH_RANGE= -10000 --source include/search_pattern_in_file.inc --error ER_CANT_CREATE_TABLE diff --git a/mysql-test/suite/encryption/t/filekeys_goodtest.inc b/mysql-test/suite/encryption/t/filekeys_goodtest.inc index 12a79c13d2a..146a570412c 100644 --- a/mysql-test/suite/encryption/t/filekeys_goodtest.inc +++ b/mysql-test/suite/encryption/t/filekeys_goodtest.inc @@ -1,4 +1,4 @@ --- source include/have_innodb.inc +-- source include/have_xtradb.inc -- source filekeys_plugin.inc create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; diff --git a/mysql-test/suite/encryption/t/innochecksum.test b/mysql-test/suite/encryption/t/innochecksum.test index 39e86d49674..a176cad6c0e 100644 --- a/mysql-test/suite/encryption/t/innochecksum.test +++ b/mysql-test/suite/encryption/t/innochecksum.test @@ -15,9 +15,12 @@ if (!$INNOCHECKSUM) { --disable_query_log let $innodb_compression_algorithm_orig=`SELECT @@innodb_compression_algorithm`; -let MYSQLD_DATADIR= `SELECT @@datadir`; +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; --enable_query_log +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; # zlib set global innodb_compression_algorithm = 1; @@ -93,4 +96,6 @@ DROP TABLE t1, t2, t3, t4, t5; # reset system --disable_query_log EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig; +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; --enable_query_log diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change.test b/mysql-test/suite/encryption/t/innodb-bad-key-change.test index e8b5de99d80..91c7c1142c2 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change.test @@ -2,26 +2,22 @@ -- source include/have_file_key_management_plugin.inc # embedded does not support restart -- source include/not_embedded.inc --- source include/not_valgrind.inc -# Avoid CrashReporter popup on Mac --- source include/not_crashrep.inc --- source include/not_windows.inc # # MDEV-8588: Assertion failure in file ha_innodb.cc line 21140 if at least one encrypted # table exists and encryption service is not available. # -call mtr.add_suppression("Plugin 'file_key_management' init function returned error"); -call mtr.add_suppression("Plugin 'file_key_management' registration.*failed"); -call mtr.add_suppression("InnoDB: The page .* in file test/.* cannot be decrypted"); -call mtr.add_suppression("mysqld: File .* not found"); +call mtr.add_suppression("Plugin 'file_key_management'"); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[12]\\.ibd' cannot be decrypted\\."); +call mtr.add_suppression("File '.*mysql-test.std_data.keysbad3\\.txt' not found"); ---echo --echo # Start server with keys2.txt -- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt -- source include/restart_mysqld.inc +SET GLOBAL innodb_file_per_table = ON; + CREATE TABLE t1 (c VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=2; INSERT INTO t1 VALUES ('foobar'); ALTER TABLE t1 ADD COLUMN c2 INT; @@ -41,14 +37,15 @@ SELECT * FROM t1; --error ER_GET_ERRMSG SELECT * FROM t1; ---replace_regex /tablespace [0-9]*/tablespace / +--replace_regex /tablespace [0-9]*/tablespace / /key_id [0-9]*/key_id / SHOW WARNINGS; -- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keysbad3.txt -- source include/restart_mysqld.inc ---replace_regex /tablespace [0-9]*/tablespace / + +--replace_regex /tablespace [1-9][0-9]*/tablespace / DROP TABLE t1; ---replace_regex /tablespace [0-9]*/tablespace / +--replace_regex /tablespace [1-9][0-9]*/tablespace / /key_id [1-9][0-9]*/key_id / SHOW WARNINGS; # @@ -56,10 +53,11 @@ SHOW WARNINGS; # line 2856 on querying a table using wrong default encryption key # ---echo # Start server with keys.txt --- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys.txt +--echo # Start server with keys3.txt +-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt -- source include/restart_mysqld.inc +SET GLOBAL innodb_default_encryption_key_id=5; CREATE TABLE t2 (c VARCHAR(8), id int not null primary key, b int, key(b)) ENGINE=InnoDB ENCRYPTED=YES; INSERT INTO t2 VALUES ('foobar',1,2); @@ -70,58 +68,51 @@ INSERT INTO t2 VALUES ('foobar',1,2); --error ER_GET_ERRMSG SELECT * FROM t2; ---replace_regex /tablespace [0-9]*/tablespace / +--replace_regex /tablespace [1-9][0-9]*/tablespace / /key_id [1-9][0-9]*/key_id / SHOW WARNINGS; --error ER_GET_ERRMSG SELECT * FROM t2 where id = 1; ---replace_regex /tablespace [0-9]*/tablespace / +--replace_regex /tablespace [1-9][0-9]*/tablespace / /key_id [1-9][0-9]*/key_id / SHOW WARNINGS; --error ER_GET_ERRMSG SELECT * FROM t2 where b = 1; ---replace_regex /tablespace [0-9]*/tablespace / +--replace_regex /tablespace [1-9][0-9]*/tablespace / /key_id [1-9][0-9]*/key_id / SHOW WARNINGS; +--replace_regex /tablespace [1-9][0-9]*/tablespace / --error ER_GET_ERRMSG INSERT INTO t2 VALUES ('tmp',3,3); ---replace_regex /tablespace [0-9]*/tablespace / +--replace_regex /tablespace [1-9][0-9]*/tablespace / /key_id [1-9][0-9]*/key_id / SHOW WARNINGS; --error ER_GET_ERRMSG DELETE FROM t2 where b = 3; ---replace_regex /tablespace [0-9]*/tablespace / +--replace_regex /tablespace [1-9][0-9]*/tablespace / /key_id [1-9][0-9]*/key_id / SHOW WARNINGS; --error ER_GET_ERRMSG DELETE FROM t2 where id = 3; ---replace_regex /tablespace [0-9]*/tablespace / +--replace_regex /tablespace [1-9][0-9]*/tablespace / /key_id [1-9][0-9]*/key_id / SHOW WARNINGS; --error ER_GET_ERRMSG UPDATE t2 set b = b +1; ---replace_regex /tablespace [0-9]*/tablespace / +--replace_regex /tablespace [1-9][0-9]*/tablespace / /key_id [1-9][0-9]*/key_id / SHOW WARNINGS; OPTIMIZE TABLE t2; ---replace_regex /tablespace [0-9]*/tablespace / +--replace_regex /tablespace [1-9][0-9]*/tablespace / /key_id [1-9][0-9]*/key_id / SHOW WARNINGS; --error ER_GET_ERRMSG -ALTER TABLE t2 ADD COLUMN c INT; ---replace_regex /tablespace [0-9]*/tablespace / +ALTER TABLE t2 ADD COLUMN d INT; +--replace_regex /tablespace [1-9][0-9]*/tablespace / /key_id [1-9][0-9]*/key_id / SHOW WARNINGS; ANALYZE TABLE t2; ---replace_regex /tablespace [0-9]*/tablespace / +--replace_regex /tablespace [1-9][0-9]*/tablespace / /key_id [1-9][0-9]*/key_id / SHOW WARNINGS; --error ER_GET_ERRMSG TRUNCATE TABLE t2; ---replace_regex /tablespace [0-9]*/tablespace / +--replace_regex /tablespace [1-9][0-9]*/tablespace / /key_id [1-9][0-9]*/key_id / SHOW WARNINGS; ---replace_regex /.*tablespace [0-9]*// ---error ER_GET_ERRMSG + DROP TABLE t2; ---replace_regex /tablespace [0-9]*/tablespace / -SHOW WARNINGS; --echo ---echo # Restart server with keys.txt --- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys.txt +--echo # Start server with keys2.txt +-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt -- source include/restart_mysqld.inc - ---replace_regex /.*tablespace [0-9]*// -DROP TABLE t2; ---replace_regex /tablespace [0-9]*/tablespace / -SHOW WARNINGS; diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change2.test b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test index c7c9e66dcf8..ee857a515da 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change2.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change2.test @@ -1,49 +1,98 @@ --source include/have_innodb.inc # embedded does not support restart -- source include/not_embedded.inc --- source include/not_valgrind.inc -# Avoid CrashReporter popup on Mac --- source include/not_crashrep.inc -- source filekeys_plugin_exists.inc # # MDEV-8750: Server crashes in page_cur_is_after_last on altering table using a wrong encryption key +# MDEV-8769: Server crash at file btr0btr.ic line 122 when defragmenting encrypted table using incorrect keys +# MDEV-8768: Server crash at file btr0btr.ic line 122 when checking encrypted table using incorrect keys +# MDEV-8727: Server/InnoDB hangs on shutdown after trying to read an encrypted table with a wrong key # -call mtr.add_suppression("InnoDB: The page .* in file test/t1 cannot be decrypted"); +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t1(new)?\\.ibd' cannot be decrypted\\."); # Suppression for builds where file_key_management plugin is linked statically call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); +call mtr.add_suppression("InnoDB: Tablespace for table \`test\`.\`t1\` is set as discarded."); ---write_file $MYSQLTEST_VARDIR/keys1.txt -1;770A8A65DA156D24EE2A093277530142 -4;770A8A65DA156D24EE2A093277530143 +--let $restart_parameters=--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt +--source include/restart_mysqld.inc + +SET GLOBAL innodb_file_per_table = ON; + +CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; +INSERT INTO t1 VALUES (1,'foo'),(2,'bar'); + +--let $restart_parameters=--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt +--source include/restart_mysqld.inc + +--error ER_GET_ERRMSG +SELECT * FROM t1; +--replace_regex /tablespace [0-9]*/tablespace / /key_id [0-9]*/key_id / +SHOW WARNINGS; +--error ER_GET_ERRMSG +ALTER TABLE t1 engine=InnoDB; +--replace_regex /tablespace [0-9]*/tablespace / /key_id [0-9]*/key_id / +SHOW WARNINGS; + +OPTIMIZE TABLE t1; +--replace_regex /tablespace [0-9]*/tablespace / /key_id [0-9]*/key_id / +SHOW WARNINGS; + +CHECK TABLE t1; +--replace_regex /tablespace [0-9]*/tablespace / /key_id [0-9]*/key_id / +SHOW WARNINGS; + +--let $restart_parameters=--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt +--source include/restart_mysqld.inc + +let MYSQLD_DATADIR =`SELECT @@datadir`; +let MYSQLD_TMPDIR = `SELECT @@tmpdir`; + +FLUSH TABLES t1 FOR EXPORT; +perl; +do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl"; +ib_backup_tablespaces("test", "t1"); EOF +UNLOCK TABLES; ---let $restart_parameters= --innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt +--let $restart_parameters=--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt --source include/restart_mysqld.inc -CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTION_KEY_ID=4; -INSERT INTO t1 VALUES (1,'foo'),(2,'bar'); +# Discard should pass even with incorrect keys +--replace_regex /tablespace [0-9]*/tablespace / /key_id [0-9]*/key_id / +ALTER TABLE t1 DISCARD TABLESPACE; ---write_file $MYSQLTEST_VARDIR/keys2.txt -1;770A8A65DA156D24EE2A093277530142 -4;770A8A65DA156D24EE2A093277530144 +perl; +do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl"; +ib_discard_tablespaces("test", "t1"); +ib_restore_tablespaces("test", "t1"); EOF ---let $restart_parameters= --innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys2.txt +--let $restart_parameters=--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt --source include/restart_mysqld.inc +ALTER TABLE t1 IMPORT TABLESPACE; +SHOW CREATE TABLE t1; + +--let $restart_parameters=--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt +--source include/restart_mysqld.inc + +# Rename table should pass even with incorrect keys +RENAME TABLE t1 TO t1new; + +# Alter table rename is not allowed with incorrect keys +--replace_regex /tablespace [0-9]*/tablespace / /key_id [0-9]*/key_id / --error ER_GET_ERRMSG -select * from t1; -show warnings; ---error ER_GET_ERRMSG -alter table t1 discard tablespace; -show warnings; ---error ER_GET_ERRMSG -alter table t1 engine=InnoDB; -show warnings; +ALTER TABLE t1new RENAME TO t2new; ---let $restart_parameters= --innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt +# Drop should pass even with incorrect keys +--replace_regex /tablespace [0-9]*/tablespace / /key_id [0-9]*/key_id / +DROP TABLE t1new; + +# +# Reset environment +# +--let $restart_parameters= --source include/restart_mysqld.inc -DROP TABLE t1; ---remove_file $MYSQLTEST_VARDIR/keys1.txt ---remove_file $MYSQLTEST_VARDIR/keys2.txt +--error ER_BAD_TABLE_ERROR +DROP TABLE t1, t1new; diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change3.test b/mysql-test/suite/encryption/t/innodb-bad-key-change3.test index dbd04748143..0ac0fc84906 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change3.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change3.test @@ -9,8 +9,12 @@ # MDEV-8772: Assertion failure in file ha_innodb.cc line 20027 when importing page compressed and encrypted tablespace using incorrect keys # -call mtr.add_suppression("InnoDB: Tablespace for table .* is set as discarded."); -call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* because the .ibd file is missing. Please refer to .* for how to resolve the issue."); +call mtr.add_suppression("InnoDB: Tablespace for table \`test\`.\`t1\` is set as discarded."); + +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; +--enable_query_log --let $MYSQLD_TMPDIR = `SELECT @@tmpdir` --let $MYSQLD_DATADIR = `SELECT @@datadir` @@ -29,7 +33,11 @@ EOF --enable_reconnect --source include/wait_until_connected_again.inc +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; +--enable_warnings CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(255)) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4; SHOW WARNINGS; @@ -37,8 +45,6 @@ SHOW CREATE TABLE t1; INSERT INTO t1 VALUES (1,'foobar'),(2,'barfoo'); let MYSQLD_DATADIR =`SELECT @@datadir`; FLUSH TABLE t1 FOR EXPORT; ---echo # List before copying files ---list_files $MYSQLD_DATADIR/test perl; do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl"; ib_backup_tablespaces("test", "t1"); @@ -67,6 +73,11 @@ EOF --source include/wait_until_connected_again.inc --source include/restart_mysqld.inc +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +--enable_warnings + --error ER_GET_ERRMSG ALTER TABLE t1 IMPORT TABLESPACE; SHOW CREATE TABLE t1; @@ -95,6 +106,12 @@ EOF DROP TABLE t1; # reset system +--disable_warnings +--disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; +--enable_query_log +--enable_warnings --remove_file $MYSQLTEST_VARDIR/keys1.txt --remove_file $MYSQLTEST_VARDIR/keys2.txt diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change5.opt b/mysql-test/suite/encryption/t/innodb-bad-key-change4.opt index 6426bac41a0..6426bac41a0 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change5.opt +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change4.opt diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change4.test b/mysql-test/suite/encryption/t/innodb-bad-key-change4.test index 0459c433ece..36421d72c63 100644 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change4.test +++ b/mysql-test/suite/encryption/t/innodb-bad-key-change4.test @@ -1,43 +1,38 @@ --source include/have_innodb.inc # embedded does not support restart -- source include/not_embedded.inc --- source include/not_valgrind.inc -# Avoid CrashReporter popup on Mac --- source include/not_crashrep.inc -- source filekeys_plugin_exists.inc # +# MDEV-8769: Server crash at file btr0btr.ic line 122 when defragmenting encrypted table using incorrect keys # MDEV-8768: Server crash at file btr0btr.ic line 122 when checking encrypted table using incorrect keys # -call mtr.add_suppression("InnoDB: The page .* in file test/t1 cannot be decrypted"); + +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t1\\.ibd' cannot be decrypted\\."); # Suppression for builds where file_key_management plugin is linked statically call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); ---write_file $MYSQLTEST_VARDIR/keys1.txt -1;770A8A65DA156D24EE2A093277530142 -4;770A8A65DA156D24EE2A093277530143 -EOF - ---let $restart_parameters= --innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt +--let $restart_parameters=--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt --source include/restart_mysqld.inc -CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTION_KEY_ID=4; -INSERT INTO t1 VALUES (1,'foo'),(2,'bar'); +SET GLOBAL innodb_file_per_table = ON; ---write_file $MYSQLTEST_VARDIR/keys2.txt -1;770A8A65DA156D24EE2A093277530142 -4;770A8A65DA156D24EE2A093277530144 -EOF +CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; +INSERT INTO t1 VALUES (1,'foo'),(2,'bar'); ---let $restart_parameters= --innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys2.txt +--let $restart_parameters=--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt --source include/restart_mysqld.inc ---replace_regex /tablespace [0-9]*/tablespace #/ +--replace_regex /tablespace [0-9]*/tablespace / /key_id [0-9]*/key_id / +OPTIMIZE TABLE t1; +--replace_regex /tablespace [0-9]*/tablespace / /key_id [0-9]*/key_id / +SHOW WARNINGS; + +--replace_regex /tablespace [0-9]*/tablespace / /key_id [0-9]*/key_id / CHECK TABLE t1; +--replace_regex /tablespace [0-9]*/tablespace / /key_id [0-9]*/key_id / SHOW WARNINGS; ---let $restart_parameters= --innodb-encrypt-tables --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt +--let $restart_parameters=--plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt --source include/restart_mysqld.inc DROP TABLE t1; ---remove_file $MYSQLTEST_VARDIR/keys1.txt ---remove_file $MYSQLTEST_VARDIR/keys2.txt diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-change5.test b/mysql-test/suite/encryption/t/innodb-bad-key-change5.test deleted file mode 100644 index b205d203374..00000000000 --- a/mysql-test/suite/encryption/t/innodb-bad-key-change5.test +++ /dev/null @@ -1,42 +0,0 @@ ---source include/have_innodb.inc -# embedded does not support restart --- source include/not_embedded.inc --- source include/not_valgrind.inc -# Avoid CrashReporter popup on Mac --- source include/not_crashrep.inc --- source filekeys_plugin_exists.inc -# -# MDEV-8769: Server crash at file btr0btr.ic line 122 when defragmenting encrypted table using incorrect keys -# -call mtr.add_suppression("InnoDB: The page .* in file test/t1 cannot be decrypted"); -# Suppression for builds where file_key_management plugin is linked statically -call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); - ---write_file $MYSQLTEST_VARDIR/keys1.txt -1;770A8A65DA156D24EE2A093277530142 -4;770A8A65DA156D24EE2A093277530143 -EOF - ---let $restart_parameters= --innodb-encrypt-tables --innodb-defragment=1 --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt ---source include/restart_mysqld.inc - -CREATE TABLE t1 (pk INT PRIMARY KEY, f VARCHAR(8)) ENGINE=InnoDB ENCRYPTION_KEY_ID=4; -INSERT INTO t1 VALUES (1,'foo'),(2,'bar'); - ---write_file $MYSQLTEST_VARDIR/keys2.txt -1;770A8A65DA156D24EE2A093277530142 -4;770A8A65DA156D24EE2A093277530144 -EOF - ---let $restart_parameters= --innodb-encrypt-tables --innodb-defragment=1 --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys2.txt ---source include/restart_mysqld.inc - -OPTIMIZE TABLE t1; -SHOW WARNINGS; - ---let $restart_parameters= --innodb-encrypt-tables --innodb-defragment=1 --innodb-stats-persistent --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt ---source include/restart_mysqld.inc - -DROP TABLE t1; ---remove_file $MYSQLTEST_VARDIR/keys1.txt ---remove_file $MYSQLTEST_VARDIR/keys2.txt diff --git a/mysql-test/suite/encryption/t/innodb-bad-key-shutdown.test b/mysql-test/suite/encryption/t/innodb-bad-key-shutdown.test deleted file mode 100644 index d1b009fad93..00000000000 --- a/mysql-test/suite/encryption/t/innodb-bad-key-shutdown.test +++ /dev/null @@ -1,52 +0,0 @@ -# -# MDEV-8727: Server/InnoDB hangs on shutdown after trying to read an encrypted table with a wrong key -# - ---source include/have_innodb.inc ---source include/not_embedded.inc - -call mtr.add_suppression("InnoDB: The page .* in file test/t1 cannot be decrypted"); -# Suppression for builds where file_key_management plugin is linked statically -call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); - ---echo # ---echo # Restart the server with key 4 in the key file ---echo # - ---write_file $MYSQLTEST_VARDIR/keys1.txt -1;770A8A65DA156D24EE2A093277530142 -4;18420B5CBA31CCDFFE9716E91EB61374D05914F3ADE23E03 -EOF - ---let $restart_parameters= --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt ---source include/restart_mysqld.inc - -CREATE TABLE t1 (i INT, KEY(i)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; -INSERT INTO t1 VALUES (1); - ---echo # ---echo # Restart the server with a different value for key 4 in the key file ---echo # - ---write_file $MYSQLTEST_VARDIR/keys2.txt -1;770A8A65DA156D24EE2A093277530142 -4;22222222222222222222222222222222 -EOF - ---let $restart_parameters= --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys2.txt ---source include/restart_mysqld.inc - ---error ER_GET_ERRMSG -SELECT * FROM t1; - ---let $restart_parameters= --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt ---source include/restart_mysqld.inc - -SELECT * FROM t1; -DROP TABLE t1; - ---let $restart_parameters= ---source include/restart_mysqld.inc - ---remove_file $MYSQLTEST_VARDIR/keys2.txt ---remove_file $MYSQLTEST_VARDIR/keys1.txt diff --git a/mysql-test/suite/encryption/t/innodb-compressed-blob.opt b/mysql-test/suite/encryption/t/innodb-compressed-blob.opt new file mode 100644 index 00000000000..36dcb6c6f26 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-compressed-blob.opt @@ -0,0 +1,4 @@ +--innodb-tablespaces-encryption +--innodb-encrypt-tables=on +--innodb-encryption-threads=2 +--max_allowed_packet=64K diff --git a/mysql-test/suite/encryption/t/innodb-compressed-blob.test b/mysql-test/suite/encryption/t/innodb-compressed-blob.test new file mode 100644 index 00000000000..6256aaf5c33 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-compressed-blob.test @@ -0,0 +1,38 @@ +-- source include/have_innodb.inc +-- source include/have_file_key_management_plugin.inc + +# embedded does not support restart +-- source include/not_embedded.inc + +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); + +--echo # Restart mysqld --file-key-management-filename=keys2.txt +-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt +-- source include/restart_mysqld.inc + +SET GLOBAL innodb_file_per_table = ON; + +set GLOBAL innodb_default_encryption_key_id=4; +create table t1(a int not null primary key, b blob, index(b(10))) engine=innodb row_format=compressed; +create table t2(a int not null primary key, b blob, index(b(10))) engine=innodb row_format=compressed encrypted=yes; +create table t3(a int not null primary key, b blob, index(b(10))) engine=innodb row_format=compressed encrypted=no; + +insert into t1 values (1, repeat('secret',6000)); +insert into t2 values (1, repeat('secret',6000)); +insert into t3 values (1, repeat('secret',6000)); + +--echo # Restart mysqld --file-key-management-filename=keys3.txt +-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys3.txt +-- source include/restart_mysqld.inc + +--error ER_GET_ERRMSG +select count(*) from t1 FORCE INDEX (b) where b like 'secret%'; +--error ER_GET_ERRMSG +select count(*) from t2 FORCE INDEX (b) where b like 'secret%'; +select count(*) from t3 FORCE INDEX (b) where b like 'secret%'; + +--echo # Restart mysqld --file-key-management-filename=keys2.txt +-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt +-- source include/restart_mysqld.inc + +drop table t1,t2,t3; diff --git a/mysql-test/suite/encryption/t/innodb-discard-import-change.test b/mysql-test/suite/encryption/t/innodb-discard-import-change.test index 2cb6865ed95..a278a8fba29 100644 --- a/mysql-test/suite/encryption/t/innodb-discard-import-change.test +++ b/mysql-test/suite/encryption/t/innodb-discard-import-change.test @@ -6,9 +6,17 @@ call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded"); +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; let $innodb_compression_algo = `SELECT @@innodb_compression_algorithm`; +--enable_query_log +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_compression_algorithm = 1; +--enable_warnings create table t1(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=4; create table t2(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; @@ -113,6 +121,11 @@ SELECT COUNT(*) FROM t5; DROP TABLE t1,t2,t3,t4,t5,t6; # reset system +--disable_warnings --disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algo; --enable_query_log +--enable_warnings + diff --git a/mysql-test/suite/encryption/t/innodb-discard-import.test b/mysql-test/suite/encryption/t/innodb-discard-import.test index e105cf82b67..9feaacc41e5 100644 --- a/mysql-test/suite/encryption/t/innodb-discard-import.test +++ b/mysql-test/suite/encryption/t/innodb-discard-import.test @@ -13,8 +13,17 @@ call mtr.add_suppression("InnoDB: Tablespace for table .* is set as discarded."); call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* because the .ibd file is missing. Please refer to .* for how to resolve the issue."); +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; let $innodb_compression_algo = `SELECT @@innodb_compression_algorithm`; +--enable_query_log + +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_compression_algorithm = 1; +--enable_warnings --let $MYSQLD_TMPDIR = `SELECT @@tmpdir` --let $MYSQLD_DATADIR = `SELECT @@datadir` @@ -117,6 +126,10 @@ DROP PROCEDURE innodb_insert_proc; DROP TABLE t1,t2,t3,t4; # reset system +--disable_warnings --disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algo; --enable_query_log +--enable_warnings diff --git a/mysql-test/suite/encryption/t/innodb-encryption-alter.test b/mysql-test/suite/encryption/t/innodb-encryption-alter.test index cb7d5a47975..316ece1c16b 100644 --- a/mysql-test/suite/encryption/t/innodb-encryption-alter.test +++ b/mysql-test/suite/encryption/t/innodb-encryption-alter.test @@ -5,9 +5,15 @@ # MDEV-8817: Failing assertion: new_state->key_version != ENCRYPTION_KEY_VERSION_INVALID # +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; let $encrypt_tables = `SELECT @@innodb_encrypt_tables`; let $threads = `SELECT @@innodb_encryption_threads`; +--enable_query_log +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_encrypt_tables = ON; SET GLOBAL innodb_encryption_threads = 4; @@ -85,6 +91,8 @@ drop table t1,t2; # reset system --disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; EVAL SET GLOBAL innodb_encrypt_tables = $encrypt_tables; EVAL SET GLOBAL innodb_encryption_threads = $threads; --enable_query_log diff --git a/mysql-test/suite/encryption/t/innodb-encryption-disable.test b/mysql-test/suite/encryption/t/innodb-encryption-disable.test index fed9878ffbc..8c72cf6a3b2 100644 --- a/mysql-test/suite/encryption/t/innodb-encryption-disable.test +++ b/mysql-test/suite/encryption/t/innodb-encryption-disable.test @@ -1,22 +1,18 @@ -- source include/have_innodb.inc # embedded does not support restart -- source include/not_embedded.inc --- source include/not_valgrind.inc -# Avoid CrashReporter popup on Mac --- source include/not_crashrep.inc -- source filekeys_plugin_exists.inc # # MDEV-9559: Server without encryption configs crashes if selecting from an implicitly encrypted table # -call mtr.add_suppression("InnoDB: The page .* in file test/t[15] cannot be decrypted"); -call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); ---write_file $MYSQLTEST_VARDIR/keys1.txt -1;770A8A65DA156D24EE2A093277530142 -EOF +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[15]\\.ibd' cannot be decrypted\\."); + +# Suppression for builds where file_key_management plugin is linked statically +call mtr.add_suppression("Couldn't load plugins from 'file_key_management"); ---let $restart_parameters= --innodb-encrypt-tables --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt +--let $restart_parameters=--innodb-encrypt-tables=ON --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt --source include/restart_mysqld.inc create table t5 ( @@ -38,16 +34,9 @@ CREATE TABLE `t1` ( ) ENGINE=InnoDB; insert into t1 values (1,2,'maria','db','encryption'); - -select * from t1; -select * from t5; - alter table t1 encrypted='yes' `encryption_key_id`=1; -select * from t1; -select * from t5; - ---let $restart_parameters= --innodb-encrypt-tables=OFF +--let $restart_parameters=--innodb-encrypt-tables=OFF --source include/restart_mysqld.inc --error ER_GET_ERRMSG @@ -55,10 +44,8 @@ select * from t1; --error ER_GET_ERRMSG select * from t5; ---let $restart_parameters= --innodb-encrypt-tables --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQLTEST_VARDIR/keys1.txt +--let $restart_parameters=--innodb-encrypt-tables=ON --plugin-load-add=file_key_management.so --file-key-management --file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt --source include/restart_mysqld.inc drop table t1; drop table t5; - ---remove_file $MYSQLTEST_VARDIR/keys1.txt diff --git a/mysql-test/suite/encryption/t/innodb-force-corrupt.test b/mysql-test/suite/encryption/t/innodb-force-corrupt.test new file mode 100644 index 00000000000..4d3bfc2d1e9 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-force-corrupt.test @@ -0,0 +1,84 @@ +# +# MDEV-11759: Encryption code in MariaDB 10.1/10.2 causes compatibility problems +# + +-- source include/have_innodb.inc +-- source include/have_file_key_management_plugin.inc +# Don't test under embedded +-- source include/not_embedded.inc + +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); + +SET GLOBAL innodb_file_per_table = ON; +set global innodb_compression_algorithm = 1; + +--echo # Create and populate tables to be corrupted +CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT,c char(200)) ENGINE=InnoDB encrypted=yes; +CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT,c char(200)) ENGINE=InnoDB row_format=compressed encrypted=yes; +CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB page_compressed=yes encrypted=yes; + +BEGIN; +INSERT INTO t1 (b,c) VALUES ('corrupt me','secret'); +--disable_query_log +--let $i = 100 +while ($i) +{ + INSERT INTO t1 (b,c) VALUES (REPEAT('abcabcabc', 100),'secretsecret'); + dec $i; +} +--enable_query_log + +INSERT INTO t1 (b,c) VALUES ('corrupt me','moresecretmoresecret'); +INSERT INTO t2 select * from t1; +INSERT INTO t3 select * from t1; +COMMIT; + +let INNODB_PAGE_SIZE=`select @@innodb_page_size`; +let MYSQLD_DATADIR=`select @@datadir`; + +--source include/shutdown_mysqld.inc + +--echo # Backup tables before corrupting +--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_DATADIR/test/t1.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t2.ibd $MYSQLD_DATADIR/test/t2.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t3.ibd $MYSQLD_DATADIR/test/t3.ibd.backup + +--echo # Corrupt tables + +perl; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +EOF + +--source include/start_mysqld.inc + +--error ER_GET_ERRMSG +SELECT * FROM t1; +--error ER_GET_ERRMSG +SELECT * FROM t2; +--error ER_GET_ERRMSG +SELECT * FROM t3; + +--source include/shutdown_mysqld.inc + +--echo # Restore the original tables +--move_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd +--move_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd +--move_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd + +--source include/start_mysqld.inc + +DROP TABLE t1,t2,t3; diff --git a/mysql-test/suite/encryption/t/innodb-log-encrypt-crash.opt b/mysql-test/suite/encryption/t/innodb-log-encrypt-crash.opt new file mode 100644 index 00000000000..e76aa060879 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-log-encrypt-crash.opt @@ -0,0 +1,6 @@ +--innodb-encrypt-log=ON +--plugin-load-add=$FILE_KEY_MANAGEMENT_SO +--loose-file-key-management +--loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/logkey.txt +--file-key-management-encryption-algorithm=aes_cbc +--innodb-buffer-pool-size=128M diff --git a/mysql-test/suite/encryption/t/innodb-log-encrypt-crash.test b/mysql-test/suite/encryption/t/innodb-log-encrypt-crash.test new file mode 100644 index 00000000000..cb9a69d036b --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-log-encrypt-crash.test @@ -0,0 +1,41 @@ +-- source include/have_innodb.inc +-- source include/not_embedded.inc +# test takes very long time on debug build +-- source include/not_debug.inc +-- source include/big_test.inc +-- source filekeys_plugin.inc + +call mtr.add_suppression("InnoDB: New log files created, LSN=.*"); +call mtr.add_suppression("InnoDB: Creating foreign key constraint system tables."); +call mtr.add_suppression("InnoDB: Error: Table .*"); + +# +# MDEV-9422: Checksum errors on restart when killing busy instance that uses encrypted XtraDB tables +# + +CREATE TABLE t1 ( + pk bigint auto_increment, + col_int int, + col_int_key int, + col_char char(12), + col_char_key char(12), + primary key (pk), + key (`col_int_key` ), + key (`col_char_key` ) +) ENGINE=InnoDB; +CREATE TABLE t2 LIKE t1; + +INSERT INTO t1 VALUES (NULL,1,1,'foo','foo'),(NULL,2,2,'bar','bar'),(NULL,3,3,'baz','baz'),(NULL,4,4,'qux','qux'); +INSERT INTO t2 + SELECT NULL, a1.col_int, a1.col_int_key, a1.col_char, a1.col_char_key + FROM t1 a1, t1 a2, t1 a3, t1 a4, t1 a5, t1 a6, t1 a7, t1 a8, t1 a9, t1 a10; + +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--shutdown_server 0 +--source include/wait_until_disconnected.inc + +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--enable_reconnect +--source include/wait_until_connected_again.inc + +DROP TABLE t1, t2; diff --git a/mysql-test/suite/encryption/t/innodb-log-encrypt.opt b/mysql-test/suite/encryption/t/innodb-log-encrypt.opt new file mode 100644 index 00000000000..24046fe779a --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-log-encrypt.opt @@ -0,0 +1,6 @@ +--innodb-encrypt-log=ON +--innodb-encrypt-tables=FORCE +--plugin-load-add=$FILE_KEY_MANAGEMENT_SO +--loose-file-key-management +--loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/logkey.txt +--file-key-management-encryption-algorithm=aes_cbc diff --git a/mysql-test/suite/encryption/t/innodb-log-encrypt.test b/mysql-test/suite/encryption/t/innodb-log-encrypt.test new file mode 100644 index 00000000000..0555f9d49aa --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-log-encrypt.test @@ -0,0 +1,97 @@ +-- source include/have_innodb.inc +-- source include/not_embedded.inc +-- source filekeys_plugin.inc + +# +# MDEV-9011: Redo log encryption does not work +# + +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + +create table t1(c1 bigint not null, b char(200), c varchar(200)) engine=innodb encrypted=yes encryption_key_id=1; +show warnings; + +delimiter //; +create procedure innodb_insert_proc (repeat_count int) +begin + declare current_num int; + set current_num = 0; + while current_num < repeat_count do + insert into t1 values(current_num, substring(MD5(RAND()), -64), REPEAT('privatejanprivate',10)); + set current_num = current_num + 1; + end while; +end// +delimiter ;// +commit; + +begin; +call innodb_insert_proc(2000); +commit; + +update t1 set c1 = c1 +1; +select count(*) from t1; + +--let $MYSQLD_DATADIR=`select @@datadir` +--let ib1_IBD = $MYSQLD_DATADIR/ibdata1 +--let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd +--let log0 = $MYSQLD_DATADIR/ib_logfile0 +--let log1 = $MYSQLD_DATADIR/ib_logfile1 +--let SEARCH_RANGE = 10000000 +--let SEARCH_PATTERN=privatejanprivate + +-- source include/kill_mysqld.inc + +--echo # ibdata1 yes on expecting NOT FOUND +-- let SEARCH_FILE=$ib1_IBD +-- source include/search_pattern_in_file.inc +--echo # t1 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t1_IBD +-- source include/search_pattern_in_file.inc +--echo # log0 yes on expecting NOT FOUND +-- let SEARCH_FILE=$log0 +-- source include/search_pattern_in_file.inc +--echo # log1 yes on expecting NOT FOUND +-- let SEARCH_FILE=$log1 +-- source include/search_pattern_in_file.inc + +--echo # Restart mysqld --innodb_encrypt_log=0 +-- let $restart_parameters=--innodb_encrypt_log=0 +-- source include/start_mysqld.inc + +insert into t1 values(5000, substring(MD5(RAND()), -64), REPEAT('publicmessage',10)); +insert into t1 values(5001, substring(MD5(RAND()), -64), REPEAT('publicmessage',10)); +insert into t1 values(5002, substring(MD5(RAND()), -64), REPEAT('publicmessage',10)); +insert into t1 values(5003, substring(MD5(RAND()), -64), REPEAT('publicmessage',10)); +insert into t1 values(5004, substring(MD5(RAND()), -64), REPEAT('publicmessage',10)); + +--let SEARCH_PATTERN=privatejanprivate +--echo # ibdata1 yes on expecting NOT FOUND +-- let SEARCH_FILE=$ib1_IBD +-- source include/search_pattern_in_file.inc +--echo # t1 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t1_IBD +-- source include/search_pattern_in_file.inc +--echo # log0 yes on expecting NOT FOUND +-- let SEARCH_FILE=$log0 +-- source include/search_pattern_in_file.inc +--echo # log1 yes on expecting NOT FOUND +-- let SEARCH_FILE=$log1 +-- source include/search_pattern_in_file.inc + +--let SEARCH_PATTERN=publicmessage +--echo # ibdata1 yes on expecting NOT FOUND +-- let SEARCH_FILE=$ib1_IBD +-- source include/search_pattern_in_file.inc +--echo # t1 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t1_IBD +-- source include/search_pattern_in_file.inc +--echo # log0 no on expecting FOUND/NOTFOUND depending where insert goes +-- let SEARCH_FILE=$log0 +-- source include/search_pattern_in_file.inc +--echo # log1 no on expecting FOUND/NOTFOUND depending where insert goes +-- let SEARCH_FILE=$log1 +-- source include/search_pattern_in_file.inc + +drop procedure innodb_insert_proc; +drop table t1; diff --git a/mysql-test/suite/encryption/t/innodb-missing-key.test b/mysql-test/suite/encryption/t/innodb-missing-key.test index 84ca92010e9..8091d23cf1c 100644 --- a/mysql-test/suite/encryption/t/innodb-missing-key.test +++ b/mysql-test/suite/encryption/t/innodb-missing-key.test @@ -6,9 +6,9 @@ # # MDEV-11004: Unable to start (Segfault or os error 2) when encryption key missing # -call mtr.add_suppression("InnoDB: The page .* in file test/t. cannot be decrypted"); ---echo +call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\."); + --echo # Start server with keys2.txt -- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt -- source include/restart_mysqld.inc diff --git a/mysql-test/suite/encryption/t/innodb-page_encryption-32k.test b/mysql-test/suite/encryption/t/innodb-page_encryption-32k.test index 03c9c93ff06..325f0a86521 100644 --- a/mysql-test/suite/encryption/t/innodb-page_encryption-32k.test +++ b/mysql-test/suite/encryption/t/innodb-page_encryption-32k.test @@ -5,12 +5,20 @@ --source include/have_innodb_32k.inc --source include/have_file_key_management_plugin.inc +call mtr.add_suppression("InnoDB: Warning: innodb_page_size has been changed from default value *"); + +--disable_query_log +# These values can change during the test +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; +--enable_query_log + +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact encrypted=yes encryption_key_id=1; ---error ER_CANT_CREATE_TABLE create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic encrypted=yes encryption_key_id=3; -show warnings; -create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic encrypted=yes encryption_key_id=33; create table innodb_redundant(c1 bigint not null, b char(200)) engine=innodb row_format=redundant encrypted=yes encryption_key_id=4; show create table innodb_compact; @@ -80,3 +88,9 @@ drop table innodb_normal; drop table innodb_compact; drop table innodb_dynamic; drop table innodb_redundant; + +# reset system +--disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; +--enable_query_log diff --git a/mysql-test/suite/encryption/t/innodb-page_encryption.test b/mysql-test/suite/encryption/t/innodb-page_encryption.test index df2d1d52aaa..463df756933 100644 --- a/mysql-test/suite/encryption/t/innodb-page_encryption.test +++ b/mysql-test/suite/encryption/t/innodb-page_encryption.test @@ -1,6 +1,14 @@ -- source include/have_innodb.inc -- source include/have_file_key_management_plugin.inc +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; +--enable_query_log + +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; show warnings; create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact encrypted=yes encryption_key_id=1; @@ -77,6 +85,9 @@ SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_ --source include/restart_mysqld.inc +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + update innodb_normal set c1 = c1 +1; update innodb_compact set c1 = c1 + 1; update innodb_compressed set c1 = c1 + 1; @@ -113,6 +124,9 @@ show create table innodb_redundant; --source include/restart_mysqld.inc +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + show create table innodb_compact; show create table innodb_compressed; show create table innodb_dynamic; @@ -147,3 +161,9 @@ drop table innodb_compressed; drop table innodb_dynamic; drop table innodb_redundant; drop table innodb_defkey; + +# reset system +--disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; +--enable_query_log diff --git a/mysql-test/suite/encryption/t/innodb-page_encryption_compression.test b/mysql-test/suite/encryption/t/innodb-page_encryption_compression.test index 2fe29e0cb5c..fc11aa1676d 100644 --- a/mysql-test/suite/encryption/t/innodb-page_encryption_compression.test +++ b/mysql-test/suite/encryption/t/innodb-page_encryption_compression.test @@ -1,15 +1,19 @@ -- source include/have_innodb.inc --- source include/not_embedded.inc -- source include/have_file_key_management_plugin.inc --- source include/big_test.inc -# Test heavy not tested on valgrind --- source include/not_valgrind.inc +-- source include/not_embedded.inc +--disable_query_log let $innodb_compression_algorithm_orig=`SELECT @@innodb_compression_algorithm`; -let $encryption = `SELECT @@innodb_encrypt_tables`; +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; +--enable_query_log +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; # zlib set global innodb_compression_algorithm = 1; +--enable_warnings create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb page_compressed=1; show warnings; @@ -35,50 +39,24 @@ end// delimiter ;// commit; -set autocommit=0; -call innodb_insert_proc(5000); -commit; -set autocommit=1; - +begin; +call innodb_insert_proc(2000); insert into innodb_compact select * from innodb_normal; insert into innodb_dynamic select * from innodb_normal; +commit; -update innodb_normal set c1 = c1 + 1; -update innodb_compact set c1 = c1 + 1; -update innodb_dynamic set c1 = c1 + 1; -select count(*) from innodb_normal; -select count(*) from innodb_compact where c1 < 1500000; -select count(*) from innodb_dynamic where c1 < 1500000; -select count(*) from innodb_compact t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -select count(*) from innodb_dynamic t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; - -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_compressed'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decompressed'; +let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_COMPRESSED'; +--source include/wait_condition.inc +--let $restart_parameters=--innodb-encrypt-tables=OFF --source include/restart_mysqld.inc +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; # zlib set global innodb_compression_algorithm = 1; - -update innodb_normal set c1 = c1 + 1; -update innodb_compact set c1 = c1 + 1; -update innodb_dynamic set c1 = c1 + 1; -select count(*) from innodb_normal; -select count(*) from innodb_compact where c1 < 1500000; -select count(*) from innodb_dynamic where c1 < 1500000; -select count(*) from innodb_compact t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -select count(*) from innodb_dynamic t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; - -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_compressed'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decompressed'; +--enable_warnings alter table innodb_normal engine=innodb page_compressed=DEFAULT; show create table innodb_normal; @@ -87,27 +65,8 @@ show create table innodb_compact; alter table innodb_dynamic engine=innodb encrypted=DEFAULT encryption_key_id=DEFAULT page_compressed=DEFAULT; show create table innodb_dynamic; ---source include/restart_mysqld.inc - -show create table innodb_normal; -show create table innodb_compact; -show create table innodb_dynamic; - -update innodb_normal set c1 = c1 + 1; -update innodb_compact set c1 = c1 + 1; -update innodb_dynamic set c1 = c1 + 1; -select count(*) from innodb_normal; -select count(*) from innodb_compact where c1 < 1500000; -select count(*) from innodb_dynamic where c1 < 1500000; -select count(*) from innodb_compact t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -select count(*) from innodb_dynamic t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; - -SELECT variable_value = 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_compressed'; -SELECT variable_value = 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decompressed'; +let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_DECOMPRESSED'; +--source include/wait_condition.inc drop procedure innodb_insert_proc; drop table innodb_normal; @@ -117,5 +76,6 @@ drop table innodb_dynamic; # reset system --disable_query_log EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig; -EVAL SET GLOBAL innodb_encrypt_tables = $encryption; +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; --enable_query_log diff --git a/mysql-test/suite/encryption/t/innodb-page_encryption_log_encryption.test b/mysql-test/suite/encryption/t/innodb-page_encryption_log_encryption.test index f7eab67154d..79dc447e352 100644 --- a/mysql-test/suite/encryption/t/innodb-page_encryption_log_encryption.test +++ b/mysql-test/suite/encryption/t/innodb-page_encryption_log_encryption.test @@ -2,9 +2,14 @@ -- source include/not_embedded.inc -- source include/have_file_key_management_plugin.inc -call mtr.add_suppression("KeyID 0 not found or with error. Check the key and the log*"); -call mtr.add_suppression("Disabling redo log encryp*"); -call mtr.add_suppression("InnoDB: Redo log crypto: Can't initialize to key version*"); +#call mtr.add_suppression("KeyID 0 not found or with error. Check the key and the log.*"); +#call mtr.add_suppression("Disabling redo log encryp.*"); +#call mtr.add_suppression("InnoDB: Redo log crypto: Can't initialize to key version.*"); + +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +--enable_warnings create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; show warnings; @@ -35,61 +40,41 @@ end// delimiter ;// commit; -set autocommit=0; +begin; call innodb_insert_proc(2000); -commit; -set autocommit=1; - insert into innodb_compact select * from innodb_normal; insert into innodb_compressed select * from innodb_normal; insert into innodb_dynamic select * from innodb_normal; insert into innodb_redundant select * from innodb_normal; +commit; -update innodb_normal set c1 = c1 +1; -update innodb_compact set c1 = c1 + 1; -update innodb_compressed set c1 = c1 + 1; -update innodb_dynamic set c1 = c1 + 1; -update innodb_redundant set c1 = c1 + 1; -select count(*) from innodb_compact where c1 < 1500000; -select count(*) from innodb_compressed where c1 < 1500000; -select count(*) from innodb_dynamic where c1 < 1500000; -select count(*) from innodb_redundant where c1 < 1500000; -select count(*) from innodb_compact t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -select count(*) from innodb_dynamic t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -select count(*) from innodb_compressed t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -select count(*) from innodb_redundant t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; +let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED'; +--source include/wait_condition.inc # Note there that these variables are updated only when real I/O is done, thus they are not reliable -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; --source include/restart_mysqld.inc +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +--enable_warnings + update innodb_normal set c1 = c1 +1; update innodb_compact set c1 = c1 + 1; update innodb_compressed set c1 = c1 + 1; update innodb_dynamic set c1 = c1 + 1; update innodb_redundant set c1 = c1 + 1; -select count(*) from innodb_compact where c1 < 1500000; -select count(*) from innodb_compressed where c1 < 1500000; -select count(*) from innodb_dynamic where c1 < 1500000; -select count(*) from innodb_redundant where c1 < 1500000; -select count(*) from innodb_compact t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -select count(*) from innodb_dynamic t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -select count(*) from innodb_compressed t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -select count(*) from innodb_redundant t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; +let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED'; +--source include/wait_condition.inc + +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; +SET GLOBAL innodb_encrypt_tables=OFF; alter table innodb_compact engine=innodb encrypted=DEFAULT encryption_key_id=DEFAULT; show create table innodb_compact; alter table innodb_compressed engine=innodb encrypted=DEFAULT encryption_key_id=DEFAULT; @@ -99,30 +84,8 @@ show create table innodb_dynamic; alter table innodb_redundant engine=innodb encrypted=DEFAULT encryption_key_id=DEFAULT; show create table innodb_redundant; ---source include/restart_mysqld.inc - -show create table innodb_compact; -show create table innodb_compressed; -show create table innodb_dynamic; -show create table innodb_redundant; - -update innodb_normal set c1 = c1 +1; -update innodb_compact set c1 = c1 + 1; -update innodb_compressed set c1 = c1 + 1; -update innodb_dynamic set c1 = c1 + 1; -update innodb_redundant set c1 = c1 + 1; -select count(*) from innodb_compact where c1 < 1500000; -select count(*) from innodb_compressed where c1 < 1500000; -select count(*) from innodb_dynamic where c1 < 1500000; -select count(*) from innodb_redundant where c1 < 1500000; -select count(*) from innodb_compact t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -select count(*) from innodb_dynamic t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -select count(*) from innodb_compressed t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; -select count(*) from innodb_redundant t1, innodb_normal t2 where -t1.c1 = t2.c1 and t1.b = t2.b; +let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_DECRYPTED'; +--source include/wait_condition.inc SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; @@ -133,10 +96,12 @@ drop table innodb_compact; drop table innodb_compressed; drop table innodb_dynamic; drop table innodb_redundant; - # # MDEV-8143: InnoDB: Database page corruption on disk or a failed file read # +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + CREATE TABLE t1 (pk INT PRIMARY KEY) ENGINE=InnoDB ENCRYPTION_KEY_ID=2 ENCRYPTED=YES; INSERT INTO t1 VALUES (1),(2); diff --git a/mysql-test/suite/encryption/t/innodb-redo-badkey.opt b/mysql-test/suite/encryption/t/innodb-redo-badkey.opt new file mode 100644 index 00000000000..343128e8803 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-redo-badkey.opt @@ -0,0 +1,5 @@ +--innodb-change-buffering=all +--innodb-encrypt-tables=on +--innodb-tablespaces-encryption +--innodb-encryption-threads=2 +--innodb-default-encryption-key-id=4 diff --git a/mysql-test/suite/encryption/t/innodb-redo-badkey.test b/mysql-test/suite/encryption/t/innodb-redo-badkey.test new file mode 100644 index 00000000000..159646541c7 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-redo-badkey.test @@ -0,0 +1,100 @@ +-- source include/have_innodb.inc +-- source include/have_file_key_management_plugin.inc +# embedded does not support restart +-- source include/not_embedded.inc + +call mtr.add_suppression("InnoDB: Block in space_id .* in file .* encrypted."); +call mtr.add_suppression("Plugin 'file_key_management' .*"); +call mtr.add_suppression("Plugin 'InnoDB' init function returned error."); +call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed."); +call mtr.add_suppression("InnoDB: Read operation failed for tablespace .*"); +call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*"); +call mtr.add_suppression("InnoDB: Recovery read page .*"); +call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t4 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); +call mtr.add_suppression("InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE .*"); +call mtr.add_suppression("InnoDB: Plugin initialization aborted .*"); +call mtr.add_suppression("InnoDB: ############### CORRUPT LOG RECORD FOUND ##################"); + +--echo # Restart mysqld --file-key-management-filename=keys2.txt +-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt +-- source include/restart_mysqld.inc + +--echo # Wait max 10 min for key encryption threads to encrypt all spaces +--let $wait_timeout= 600 +--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 +--source include/wait_condition.inc + +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +--enable_warnings + +create table t1(a int not null primary key auto_increment, c char(250), b blob, index(b(10))) engine=innodb row_format=compressed encrypted=yes encryption_key_id=4; +create table t2(a int not null primary key auto_increment, c char(250), b blob, index(b(10))) engine=innodb row_format=compressed; +create table t3(a int not null primary key auto_increment, c char(250), b blob, index(b(10))) engine=innodb encrypted=yes encryption_key_id=4; +create table t4(a int not null primary key auto_increment, c char(250), b blob, index(b(10))) engine=innodb; + +begin; +--disable_query_log +--let $i = 20 +begin; +while ($i) +{ + insert into t1(c,b) values (repeat('secret1',20), repeat('secret2',6000)); + dec $i; +} +--enable_query_log + +insert into t2 select * from t1; +insert into t3 select * from t1; +insert into t4 select * from t1; +commit; + +--source ../../suite/innodb/include/no_checkpoint_start.inc + +# +# We test redo log page read at recv_read_page using +# incorrect keys from std_data/keys.txt. If checkpoint +# happens we will skip this test. If no checkpoint +# happens, InnoDB refuses to start as used +# encryption key is incorrect. +# +SET GLOBAL innodb_flush_log_at_trx_commit=1; +begin; +update t1 set c = repeat('secret3', 20); +update t2 set c = repeat('secret4', 20); +update t3 set c = repeat('secret4', 20); +update t4 set c = repeat('secret4', 20); +insert into t1 (c,b) values (repeat('secret5',20), repeat('secret6',6000)); +insert into t2 (c,b) values (repeat('secret7',20), repeat('secret8',6000)); +insert into t3 (c,b) values (repeat('secret9',20), repeat('secre10',6000)); +insert into t4 (c,b) values (repeat('secre11',20), repeat('secre12',6000)); +COMMIT; +let $cleanup= drop table t1,t2,t3,t4; +--let CLEANUP_IF_CHECKPOINT= $cleanup; +--source ../../suite/innodb/include/no_checkpoint_end.inc + +--echo # restart +--error 1 +-- source include/start_mysqld.inc +--source include/kill_mysqld.inc + +# +# Now test with innodb-force-recovery=1 i.e. ignore corrupt pages +# + +--echo # Restart mysqld --innodb-force-recovery=1 +-- let $restart_parameters=--innodb-force-recovery=1 +--error 1 +-- source include/start_mysqld.inc + +--source include/kill_mysqld.inc + +--echo # Restart mysqld --file-key-management-filename=keys2.txt +-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt +-- source include/start_mysqld.inc + +drop table t1, t2,t3,t4; diff --git a/mysql-test/suite/encryption/t/innodb-redo-nokeys.opt b/mysql-test/suite/encryption/t/innodb-redo-nokeys.opt new file mode 100644 index 00000000000..21afc19fc5d --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-redo-nokeys.opt @@ -0,0 +1,3 @@ +--innodb-change-buffering=none +--innodb-encrypt-tables=on +--innodb-default-encryption-key-id=20 diff --git a/mysql-test/suite/encryption/t/innodb-redo-nokeys.test b/mysql-test/suite/encryption/t/innodb-redo-nokeys.test new file mode 100644 index 00000000000..e55e2ade153 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-redo-nokeys.test @@ -0,0 +1,80 @@ +-- source include/have_innodb.inc +-- source include/have_file_key_management_plugin.inc +# embedded does not support restart +-- source include/not_embedded.inc + +call mtr.add_suppression("InnoDB: Block in space_id .*"); +call mtr.add_suppression("mysqld: File .*"); +call mtr.add_suppression("Plugin 'file_key_management' .*"); +call mtr.add_suppression("InnoDB: cannot enable encryption, encryption plugin is not available"); +call mtr.add_suppression("Plugin 'InnoDB' init function returned error."); +call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed."); +call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t2 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t3 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t4 cannot be decrypted."); +call mtr.add_suppression("InnoDB: The page \[page id: space=[0-9]+, page number=[0-9]+\] in file test/t1 cannot be decrypted."); + +--echo # Restart mysqld --file-key-management-filename=keys2.txt +-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt +-- source include/restart_mysqld.inc + +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +--enable_warnings + +create table t1(a int not null primary key auto_increment, c char(200), b blob, index(b(10))) engine=innodb row_format=compressed encrypted=yes encryption_key_id=20; +create table t2(a int not null primary key auto_increment, c char(200), b blob, index(b(10))) engine=innodb row_format=compressed; +create table t3(a int not null primary key auto_increment, c char(200), b blob, index(b(10))) engine=innodb encrypted=yes encryption_key_id=20; +create table t4(a int not null primary key auto_increment, c char(200), b blob, index(b(10))) engine=innodb; + +begin; +--disable_query_log +--let $i = 20 +begin; +while ($i) +{ + insert into t1(c,b) values (repeat('secret1',20), repeat('secret2',6000)); + dec $i; +} +--enable_query_log + +insert into t2 select * from t1; +insert into t3 select * from t1; +insert into t4 select * from t1; +commit; + +--source ../../suite/innodb/include/no_checkpoint_start.inc +# +# We test redo log page read at recv_read_page using +# keys that are not in std_data/keys.txt. If checkpoint +# happens we will skip this test. If no checkpoint +# happens, InnoDB refuses to start as used +# encryption key is not found. +# +SET GLOBAL innodb_flush_log_at_trx_commit=1; +begin; +update t1 set c = repeat('secret3', 20); +update t2 set c = repeat('secret4', 20); +update t3 set c = repeat('secret4', 20); +update t4 set c = repeat('secret4', 20); +insert into t1 (c,b) values (repeat('secret5',20), repeat('secret6',6000)); +insert into t2 (c,b) values (repeat('secret7',20), repeat('secret8',6000)); +insert into t3 (c,b) values (repeat('secret9',20), repeat('secre10',6000)); +insert into t4 (c,b) values (repeat('secre11',20), repeat('secre12',6000)); +COMMIT; +let $cleanup= drop table t1,t2,t3,t4; +--let CLEANUP_IF_CHECKPOINT= $cleanup; +--source ../../suite/innodb/include/no_checkpoint_end.inc + +--echo # restart +-- source include/start_mysqld.inc +# +# In above server does start but InnoDB refuses to start +# thus we need to restart server with correct key file +# +--echo # Restart mysqld --file-key-management-filename=keys2.txt +-- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt +-- source include/restart_mysqld.inc + +drop table t1, t2,t3,t4; diff --git a/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test b/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test index 606662e44e6..ea6bd65d605 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test +++ b/mysql-test/suite/encryption/t/innodb_encryption-page-compression.test @@ -1,15 +1,20 @@ -- source include/have_innodb.inc -- source include/have_example_key_management_plugin.inc -# embedded does not support restart --- source include/not_embedded.inc - +--disable_query_log let $innodb_compression_algorithm_orig=`SELECT @@innodb_compression_algorithm`; +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; let $innodb_encrypt_tables_orig = `SELECT @@innodb_encrypt_tables`; let $innodb_encryption_threads_orig = `SELECT @@innodb_encryption_threads`; +--enable_query_log +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_encryption_threads = 4; SET GLOBAL innodb_encrypt_tables = on; +--enable_warnings # zlib set global innodb_compression_algorithm = 1; @@ -56,11 +61,8 @@ end// delimiter ;// commit; -set autocommit=0; -call innodb_insert_proc(5000); -commit; -set autocommit=1; -select count(*) from innodb_normal; +begin; +call innodb_insert_proc(2000); insert into innodb_page_compressed1 select * from innodb_normal; insert into innodb_page_compressed2 select * from innodb_normal; insert into innodb_page_compressed3 select * from innodb_normal; @@ -71,23 +73,6 @@ insert into innodb_page_compressed7 select * from innodb_normal; insert into innodb_page_compressed8 select * from innodb_normal; insert into innodb_page_compressed9 select * from innodb_normal; commit; -select count(*) from innodb_page_compressed1 where c1 < 500000; -select count(*) from innodb_page_compressed2 where c1 < 500000; -select count(*) from innodb_page_compressed3 where c1 < 500000; -select count(*) from innodb_page_compressed4 where c1 < 500000; -select count(*) from innodb_page_compressed5 where c1 < 500000; -select count(*) from innodb_page_compressed6 where c1 < 500000; -select count(*) from innodb_page_compressed7 where c1 < 500000; -select count(*) from innodb_page_compressed8 where c1 < 500000; -select count(*) from innodb_page_compressed9 where c1 < 500000; - -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed'; - - ---source include/restart_mysqld.inc select count(*) from innodb_page_compressed1 where c1 < 500000; select count(*) from innodb_page_compressed2 where c1 < 500000; @@ -98,33 +83,18 @@ select count(*) from innodb_page_compressed6 where c1 < 500000; select count(*) from innodb_page_compressed7 where c1 < 500000; select count(*) from innodb_page_compressed8 where c1 < 500000; select count(*) from innodb_page_compressed9 where c1 < 500000; -update innodb_page_compressed1 set c1 = c1 + 1; -update innodb_page_compressed2 set c1 = c1 + 1; -update innodb_page_compressed3 set c1 = c1 + 1; -update innodb_page_compressed4 set c1 = c1 + 1; -update innodb_page_compressed5 set c1 = c1 + 1; -update innodb_page_compressed6 set c1 = c1 + 1; -update innodb_page_compressed7 set c1 = c1 + 1; -update innodb_page_compressed8 set c1 = c1 + 1; -update innodb_page_compressed9 set c1 = c1 + 1; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; +let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_COMPRESSED'; +--source include/wait_condition.inc + +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed'; SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed'; SET GLOBAL innodb_encryption_threads = 4; SET GLOBAL innodb_encrypt_tables = off; -select count(*) from innodb_page_compressed1 where c1 < 500000; -select count(*) from innodb_page_compressed2 where c1 < 500000; -select count(*) from innodb_page_compressed3 where c1 < 500000; -select count(*) from innodb_page_compressed4 where c1 < 500000; -select count(*) from innodb_page_compressed5 where c1 < 500000; -select count(*) from innodb_page_compressed6 where c1 < 500000; -select count(*) from innodb_page_compressed7 where c1 < 500000; -select count(*) from innodb_page_compressed8 where c1 < 500000; -select count(*) from innodb_page_compressed9 where c1 < 500000; update innodb_page_compressed1 set c1 = c1 + 1; update innodb_page_compressed2 set c1 = c1 + 1; update innodb_page_compressed3 set c1 = c1 + 1; @@ -135,9 +105,12 @@ update innodb_page_compressed7 set c1 = c1 + 1; update innodb_page_compressed8 set c1 = c1 + 1; update innodb_page_compressed9 set c1 = c1 + 1; +let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_DECRYPTED'; +--source include/wait_condition.inc + SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; -SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed'; SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed'; drop procedure innodb_insert_proc; @@ -155,6 +128,9 @@ drop table innodb_page_compressed9; # reset system --disable_query_log EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig; +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; EVAL SET GLOBAL innodb_encrypt_tables = $innodb_encrypt_tables_orig; EVAL SET GLOBAL innodb_encryption_threads = $innodb_encryption_threads_orig; --enable_query_log + diff --git a/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test b/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test index 075cb1cf77e..192233a535f 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test +++ b/mysql-test/suite/encryption/t/innodb_encryption_discard_import.test @@ -3,8 +3,7 @@ -- source include/not_valgrind.inc -- source include/not_embedded.inc -call mtr.add_suppression("InnoDB: Tablespace for table .* is set as discarded."); -call mtr.add_suppression("InnoDB: Cannot calculate statistics for table .* because the .ibd file is missing. Please refer to .* for how to resolve the issue."); +call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded."); let $MYSQLD_TMPDIR = `SELECT @@tmpdir`; let $MYSQLD_DATADIR = `SELECT @@datadir`; @@ -15,6 +14,16 @@ let $MYSQLD_DATADIR = `SELECT @@datadir`; --let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd --let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; +--enable_query_log + +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +--enable_warnings + CREATE TABLE t1 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes; CREATE TABLE t2 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB; CREATE TABLE t3 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB row_format=compressed encrypted=yes; @@ -144,3 +153,11 @@ SELECT COUNT(1) FROM t3; DROP PROCEDURE innodb_insert_proc; DROP TABLE t1, t2, t3; + +# reset system +--disable_warnings +--disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; +--enable_query_log +--enable_warnings diff --git a/mysql-test/suite/encryption/t/innodb_encryption_filekeys.test b/mysql-test/suite/encryption/t/innodb_encryption_filekeys.test index 47d546ae698..8f0986071f1 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption_filekeys.test +++ b/mysql-test/suite/encryption/t/innodb_encryption_filekeys.test @@ -5,9 +5,15 @@ call mtr.add_suppression("trying to do an operation on a dropped tablespace .*"); +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; let $encrypt_tables = `SELECT @@innodb_encrypt_tables`; let $threads = `SELECT @@innodb_encryption_threads`; +--enable_query_log +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_encrypt_tables = OFF; SET GLOBAL innodb_encryption_threads = 4; @@ -127,6 +133,8 @@ SELECT COUNT(1) FROM t5; # reset system --disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; EVAL SET GLOBAL innodb_encrypt_tables = $encrypt_tables; EVAL SET GLOBAL innodb_encryption_threads = $threads; --enable_query_log @@ -134,3 +142,4 @@ EVAL SET GLOBAL innodb_encryption_threads = $threads; drop table t1,t2,t3,t4, t5; set GLOBAL innodb_default_encryption_key_id=1; + diff --git a/mysql-test/suite/encryption/t/innodb_encryption_is.test b/mysql-test/suite/encryption/t/innodb_encryption_is.test index 52574aa2b9d..333bb84c634 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption_is.test +++ b/mysql-test/suite/encryption/t/innodb_encryption_is.test @@ -1,6 +1,14 @@ -- source include/have_innodb.inc -- source include/have_file_key_management_plugin.inc +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; +--enable_query_log + +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + CREATE TABLE t1 (c VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=1; CREATE TABLE t2 (c VARCHAR(8)) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=2; INSERT INTO t1 VALUES ('foobar'); @@ -15,3 +23,9 @@ FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE NAME LIKE '%t1' OR NAME LIKE '%t2'; DROP TABLE t1, t2; + +# reset system +--disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; +--enable_query_log diff --git a/mysql-test/suite/encryption/t/innodb_encryption_row_compressed.test b/mysql-test/suite/encryption/t/innodb_encryption_row_compressed.test index d6319164c35..0a28c1690a2 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption_row_compressed.test +++ b/mysql-test/suite/encryption/t/innodb_encryption_row_compressed.test @@ -2,6 +2,14 @@ -- source include/have_file_key_management_plugin.inc -- source include/not_embedded.inc +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; +--enable_query_log + +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + create table innodb_compressed1(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed encrypted=yes; create table innodb_compressed2(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed key_block_size=1 encrypted=yes; create table innodb_compressed3(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed key_block_size=2 encrypted=yes; @@ -44,6 +52,9 @@ insert into innodb_compressed4 select * from innodb_compressed1; -- let SEARCH_FILE=$t4_IBD -- source include/search_pattern_in_file.inc +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + select * from innodb_compressed1 where d = 20; select * from innodb_compressed1 where d = 30; select * from innodb_compressed2 where d = 20; @@ -106,3 +117,9 @@ drop table innodb_compressed1; drop table innodb_compressed2; drop table innodb_compressed3; drop table innodb_compressed4; + +# reset system +--disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; +--enable_query_log diff --git a/mysql-test/suite/encryption/t/innodb_encryption_tables.test b/mysql-test/suite/encryption/t/innodb_encryption_tables.test index bc762faf12e..f806b776800 100644 --- a/mysql-test/suite/encryption/t/innodb_encryption_tables.test +++ b/mysql-test/suite/encryption/t/innodb_encryption_tables.test @@ -2,6 +2,14 @@ -- source include/have_example_key_management_plugin.inc -- source include/not_embedded.inc +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; +--enable_query_log + +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact; create table innodb_dynamic(c1 bigint not null, b char(200)) engine=innodb row_format=dynamic; @@ -65,6 +73,9 @@ SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_ --source include/restart_mysqld.inc +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + update innodb_normal set c1 = c1 + 1; update innodb_compact set c1 = c1 + 1; update innodb_dynamic set c1 = c1 + 1; @@ -95,3 +106,9 @@ drop table innodb_compact; drop table innodb_dynamic; drop table innodb_compressed; drop table innodb_redundant; + +# reset system +--disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; +--enable_query_log diff --git a/mysql-test/suite/encryption/t/innodb_first_page.test b/mysql-test/suite/encryption/t/innodb_first_page.test index 58095e5e110..a0c563b1854 100644 --- a/mysql-test/suite/encryption/t/innodb_first_page.test +++ b/mysql-test/suite/encryption/t/innodb_first_page.test @@ -5,14 +5,25 @@ --source include/have_innodb.inc --source include/have_file_key_management_plugin.inc -let $datadir=`select @@datadir`; ---source include/shutdown_mysqld.inc +call mtr.add_suppression("InnoDB: New log files created, LSN"); +call mtr.add_suppression("InnoDB: Creating foreign key constraint system tables"); + +let datadir=`select @@datadir`; + +--exec echo wait > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect + +shutdown; + +--source include/wait_until_disconnected.inc --remove_file $datadir/ib_logfile0 --remove_file $datadir/ib_logfile1 --remove_file $datadir/ibdata1 ---source include/start_mysqld.inc +--exec echo restart > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect + +--enable_reconnect +--source include/wait_until_connected_again.inc create table t1 (a int); diff --git a/mysql-test/suite/encryption/t/innodb_lotoftables.test b/mysql-test/suite/encryption/t/innodb_lotoftables.test index 6e87c93df4b..6073a5025e4 100644 --- a/mysql-test/suite/encryption/t/innodb_lotoftables.test +++ b/mysql-test/suite/encryption/t/innodb_lotoftables.test @@ -6,6 +6,7 @@ -- source include/not_embedded.inc --disable_query_log +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; let $innodb_encryption_threads_orig = `SELECT @@global.innodb_encryption_threads`; --enable_query_log @@ -13,6 +14,8 @@ let $innodb_encryption_threads_orig = `SELECT @@global.innodb_encryption_threads SET GLOBAL innodb_fast_shutdown=0; --source include/restart_mysqld.inc +SET GLOBAL innodb_file_per_table = ON; + SHOW VARIABLES LIKE 'innodb_encrypt%'; # @@ -270,5 +273,6 @@ drop database innodb_encrypted_2; drop database innodb_encrypted_3; --disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; EVAL SET GLOBAL innodb_encryption_threads = $innodb_encryption_threads_orig; --enable_query_log diff --git a/mysql-test/suite/encryption/t/innodb_onlinealter_encryption.opt b/mysql-test/suite/encryption/t/innodb_onlinealter_encryption.opt index bcff011eb82..38d69691ed6 100644 --- a/mysql-test/suite/encryption/t/innodb_onlinealter_encryption.opt +++ b/mysql-test/suite/encryption/t/innodb_onlinealter_encryption.opt @@ -3,6 +3,3 @@ --innodb-encryption-rotate-key-age=15 --innodb-encryption-threads=4 --innodb-tablespaces-encryption ---innodb-max-dirty-pages-pct=0.001 - - diff --git a/mysql-test/suite/encryption/t/innodb_onlinealter_encryption.test b/mysql-test/suite/encryption/t/innodb_onlinealter_encryption.test index e7e8405e839..3e308bec247 100644 --- a/mysql-test/suite/encryption/t/innodb_onlinealter_encryption.test +++ b/mysql-test/suite/encryption/t/innodb_onlinealter_encryption.test @@ -3,6 +3,11 @@ # test uses restart -- source include/not_embedded.inc +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +--enable_warnings + CREATE TABLE t1 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes; CREATE TABLE t2 (id INT NOT NULL PRIMARY KEY, a VARCHAR(255)) ENGINE=InnoDB; CREATE TABLE t3 (id INT, a VARCHAR(255)) ENGINE=InnoDB encrypted=yes; @@ -17,8 +22,8 @@ begin declare current_num int; set current_num = 0; while current_num < repeat_count do - insert into t1 values (current_num,repeat('foobar',42)); - insert into t2 values (current_num,repeat('temp', 42)); + insert into t1 values (current_num,repeat('foobar',12)); + insert into t2 values (current_num,repeat('tempsecret', 12)); insert into t3 values (current_num,repeat('barfoo',42)); insert into t4 values (current_num,repeat('repeat',42)); insert into t5 values (current_num,substring('A BC DEF GHIJ KLM NOPQRS TUV WXYZ 012 3456789', rand()*36+1, 100), repeat('author new',22)); @@ -42,14 +47,13 @@ set autocommit=1; --let $MYSQLD_DATADIR=`select @@datadir` --source include/shutdown_mysqld.inc ---source include/wait_until_disconnected.inc --let SEARCH_RANGE = 10000000 --let SEARCH_PATTERN=foobar --echo # t1 yes on expecting NOT FOUND -- let SEARCH_FILE=$MYSQLD_DATADIR/test/t1.ibd -- source include/search_pattern_in_file.inc ---let SEARCH_PATTERN=temp +--let SEARCH_PATTERN=tempsecret --echo # t2 ... on expecting NOT FOUND -- let SEARCH_FILE=$MYSQLD_DATADIR/test/t2.ibd -- source include/search_pattern_in_file.inc @@ -103,13 +107,12 @@ SHOW CREATE TABLE t6; SHOW CREATE TABLE t7; --source include/shutdown_mysqld.inc ---source include/wait_until_disconnected.inc --let SEARCH_PATTERN=foobar --echo # t1 yes on expecting NOT FOUND -- let SEARCH_FILE=$MYSQLD_DATADIR/test/t1.ibd -- source include/search_pattern_in_file.inc ---let SEARCH_PATTERN=temp +--let SEARCH_PATTERN=tempsecret --echo # t2 ... on expecting NOT FOUND -- let SEARCH_FILE=$MYSQLD_DATADIR/test/t2.ibd -- source include/search_pattern_in_file.inc diff --git a/mysql-test/suite/encryption/t/innodb_page_encryption_key_change.test b/mysql-test/suite/encryption/t/innodb_page_encryption_key_change.test index 1babf577473..7cbf005ed32 100644 --- a/mysql-test/suite/encryption/t/innodb_page_encryption_key_change.test +++ b/mysql-test/suite/encryption/t/innodb_page_encryption_key_change.test @@ -7,6 +7,14 @@ -- let $restart_parameters=--loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt -- source include/restart_mysqld.inc +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; +--enable_query_log + +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + create table innodb_normal(c1 bigint not null, b char(200)) engine=innodb; show warnings; create table innodb_compact(c1 bigint not null, b char(200)) engine=innodb row_format=compact encrypted=yes encryption_key_id=2; @@ -57,6 +65,9 @@ select * from innodb_redundant; SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted'; +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + alter table innodb_compact engine=innodb encryption_key_id = 2; alter table innodb_compressed engine=innodb encryption_key_id = 3; alter table innodb_dynamic engine=innodb encryption_key_id = 4; @@ -77,3 +88,9 @@ drop table innodb_compact; drop table innodb_compressed; drop table innodb_dynamic; drop table innodb_redundant; + +# reset system +--disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; +--enable_query_log diff --git a/mysql-test/suite/encryption/t/innodb_scrub.opt b/mysql-test/suite/encryption/t/innodb_scrub.opt index ef43b2a20a8..3adb645c589 100644 --- a/mysql-test/suite/encryption/t/innodb_scrub.opt +++ b/mysql-test/suite/encryption/t/innodb_scrub.opt @@ -2,6 +2,8 @@ --innodb-background-scrub-data-uncompressed=OFF --innodb-encrypt-tables=OFF --innodb-encryption-threads=0 +--innodb-file-format=Barracuda +--innodb-file-per-table=1 --innodb-immediate-scrub-data-uncompressed=ON --loose-innodb-debug-force-scrubbing=ON --innodb-tablespaces-scrubbing diff --git a/mysql-test/suite/encryption/t/innodb_scrub.test b/mysql-test/suite/encryption/t/innodb_scrub.test index 3288ae9ce1c..6cb48530f86 100644 --- a/mysql-test/suite/encryption/t/innodb_scrub.test +++ b/mysql-test/suite/encryption/t/innodb_scrub.test @@ -7,7 +7,7 @@ let INNODB_PAGE_SIZE= `select @@innodb_page_size`; create table snapshot_status engine = myisam select * from information_schema.global_status -where variable_name like 'innodb_scrub_background%'; +where variable_name like 'innodb_scrub%'; let $rowcount=500; let $maxformatno= 4; diff --git a/mysql-test/suite/encryption/t/innodb_scrub_background.opt b/mysql-test/suite/encryption/t/innodb_scrub_background.opt index 04987f4a3ad..1cc525b2402 100644 --- a/mysql-test/suite/encryption/t/innodb_scrub_background.opt +++ b/mysql-test/suite/encryption/t/innodb_scrub_background.opt @@ -1,3 +1,5 @@ +--innodb-file-per-table=1 +--innodb-file-format=Barracuda --innodb-immediate-scrub-data-uncompressed=OFF --innodb-background-scrub-data-uncompressed=ON --innodb-background-scrub-data-compressed=ON diff --git a/mysql-test/suite/encryption/t/tempfiles.test b/mysql-test/suite/encryption/t/tempfiles.test index 065d775c182..34dcbdf5963 100644 --- a/mysql-test/suite/encryption/t/tempfiles.test +++ b/mysql-test/suite/encryption/t/tempfiles.test @@ -7,7 +7,9 @@ source include/have_sequence.inc; # Row binlog format to fill binlog cache faster source include/have_binlog_format_row.inc; -source include/have_innodb.inc; +# Nothing XtraDB specific in this test, it just needs *some* transactional +# engine. But there's no need to run it twice for InnoDB and XtraDB. +source include/have_xtradb.inc; # # MyISAM messing around with IO_CACHE::file diff --git a/mysql-test/suite/innodb/disabled.def b/mysql-test/suite/innodb/disabled.def index 631f01a8e65..e92e2051994 100644 --- a/mysql-test/suite/innodb/disabled.def +++ b/mysql-test/suite/innodb/disabled.def @@ -11,4 +11,3 @@ ############################################################################## innodb_defragment_fill_factor : MDEV-10771 -innodb_bug14147491 : MDEV-12253 ut_ad(buf_pool->n_pend_reads > 0) diff --git a/mysql-test/suite/innodb/r/innodb_bug14147491.result b/mysql-test/suite/innodb/r/innodb_bug14147491.result index cf960e3a6ee..7441bc7c60a 100644 --- a/mysql-test/suite/innodb/r/innodb_bug14147491.result +++ b/mysql-test/suite/innodb/r/innodb_bug14147491.result @@ -1,17 +1,23 @@ -CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ROW_FORMAT=COMPACT ENGINE=InnoDB; +# Create and populate the table to be corrupted +set global innodb_file_per_table=ON; +CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB; INSERT INTO t1 (b) VALUES ('corrupt me'); INSERT INTO t1 (b) VALUES ('corrupt me'); # Backup the t1.ibd before corrupting # Corrupt the table Munged a string. Munged a string. -SET DEBUG_DBUG = '+d,innodb_page_corruption_retries'; -# Write file to make mysql-test-run.pl expect the "crash", but don't -# start it until it's told to -# The below SELECT query will crash the server because some pages -# on the disk are corrupted +# The below SELECT query will return that table is not +# in engine because table is corrupted SELECT * FROM t1; -ERROR HY000: Lost connection to MySQL server during query -# Restore the original t1.ibd +ERROR 42S02: Table 'test.t1' doesn't exist in engine +INSERT INTO t1(b) VALUES('abcdef'); +ERROR 42S02: Table 'test.t1' doesn't exist in engine +UPDATE t1 set b = 'deadbeef' where a = 1; +ERROR 42S02: Table 'test.t1' doesn't exist in engine +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check Error Table 'test.t1' doesn't exist in engine +test.t1 check status Operation failed # Cleanup DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/innodb_bug14147491.test b/mysql-test/suite/innodb/t/innodb_bug14147491.test index c848d24294d..c1592537d46 100644 --- a/mysql-test/suite/innodb/t/innodb_bug14147491.test +++ b/mysql-test/suite/innodb/t/innodb_bug14147491.test @@ -1,28 +1,23 @@ # # Test opening a corrupted table. # -# Valgrind can hang or return spurious messages on DBUG_SUICIDE -source include/not_valgrind.inc; -# Avoid CrashReporter popup on Mac -source include/not_crashrep.inc; # Restarting is not supported under embedded -source include/not_embedded.inc; +-- source include/not_embedded.inc # Require InnoDB -source include/have_innodb.inc; -# Require Debug for SET DEBUG -source include/have_debug.inc; -# Not encrypted tables -source include/not_encrypted.inc; -# Test could open crash reporter on Windows -# if compiler set up -source include/not_windows.inc; +-- source include/have_innodb.inc +-- source include/not_encrypted.inc --disable_query_log -CALL mtr.add_suppression("\\[ERROR\\] \\[FATAL\\] InnoDB: Unable to read page \\[page id: space=.*, page number=.*\\] into the buffer pool after 100 attempts"); -CALL mtr.add_suppression("\\[ERROR\\] InnoDB: Database page corruption on disk or a failed"); +call mtr.add_suppression("InnoDB: Table `test`.`t1` is corrupted. Please drop the table and recreate."); +call mtr.add_suppression("InnoDB: Cannot open table test/t1 from the internal data dictionary of InnoDB though the .frm file for the table exists. Please refer to http://dev.mysql.com/doc/refman/5.7/en/innodb-troubleshooting.html for how to resolve the issue."); +call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed file read of tablespace test/t1 page \[page id: space=[0-9]+, page number=[0-9]+\]. You may have to recover from a backup."); --enable_query_log -CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ROW_FORMAT=COMPACT ENGINE=InnoDB; +--echo # Create and populate the table to be corrupted + +set global innodb_file_per_table=ON; + +CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB; INSERT INTO t1 (b) VALUES ('corrupt me'); --disable_query_log --let $i = 10 @@ -71,27 +66,18 @@ EOF --source include/start_mysqld.inc -SET DEBUG_DBUG = '+d,innodb_page_corruption_retries'; - ---echo # Write file to make mysql-test-run.pl expect the "crash", but don't ---echo # start it until it's told to ---exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect - ---echo # The below SELECT query will crash the server because some pages ---echo # on the disk are corrupted ---error 2013 +--echo # The below SELECT query will return that table is not +--echo # in engine because table is corrupted +--error ER_NO_SUCH_TABLE_IN_ENGINE SELECT * FROM t1; -# The below mtr command --remove_file fails randomly on windows with -# error number 13 which is permission denied on nix systems. We sleep -# 1 second hoping that any process holding lock on t1.ibd is released. -SLEEP 1; +--error ER_NO_SUCH_TABLE_IN_ENGINE +INSERT INTO t1(b) VALUES('abcdef'); ---echo # Restore the original t1.ibd ---remove_file $MYSQLD_DATADIR/test/t1.ibd ---move_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd +--error ER_NO_SUCH_TABLE_IN_ENGINE +UPDATE t1 set b = 'deadbeef' where a = 1; ---source include/start_mysqld.inc +CHECK TABLE t1; --echo # Cleanup DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/log_file_name.test b/mysql-test/suite/innodb/t/log_file_name.test index 0a8dc3e1fc0..f2b8dd06fd4 100644 --- a/mysql-test/suite/innodb/t/log_file_name.test +++ b/mysql-test/suite/innodb/t/log_file_name.test @@ -170,6 +170,7 @@ call mtr.add_suppression("InnoDB: Ignoring tablespace for `test`.`u[1-3]` becaus call mtr.add_suppression("InnoDB: Failed to find tablespace for table .* in the cache. Attempting to load the tablespace with space id"); call mtr.add_suppression("InnoDB: Plugin initialization aborted"); call mtr.add_suppression("Plugin 'InnoDB' \(init function returned error\|registration as a STORAGE ENGINE failed\)"); +call mtr.add_suppression("InnoDB: Table test/u[123] in the InnoDB data dictionary has tablespace id [1-9][0-9]*, but tablespace with that id or name does not exist\\. Have you deleted or moved \\.ibd files\\?"); FLUSH TABLES; --enable_query_log @@ -265,7 +266,6 @@ let SEARCH_PATTERN= \[Warning\] InnoDB: Tablespace \d+ was not found at .*u[1-5] DROP TABLE u1,u2,u3,u6; ---remove_file $MYSQLD_DATADIR/test/u1.ibd --echo # List of files: --list_files $MYSQLD_DATADIR/test diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index 489b612a17a..79bbc841a3c 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -169,8 +169,7 @@ btr_root_block_get( if (!block) { if (index && index->table) { - index->table->is_encrypted = TRUE; - index->table->corrupted = FALSE; + index->table->file_unreadable = true; ib_push_warning(index->table->thd, DB_DECRYPTION_FAILED, "Table %s in tablespace %lu is encrypted but encryption service or" @@ -183,6 +182,7 @@ btr_root_block_get( } btr_assert_not_corrupted(block, index); + #ifdef UNIV_BTR_DEBUG if (!dict_index_is_ibuf(index)) { const page_t* root = buf_block_get_frame(block); @@ -5418,7 +5418,7 @@ btr_validate_index( page_t* root = btr_root_get(index, &mtr); - if (root == NULL && index->table->is_encrypted) { + if (root == NULL && index->table->file_unreadable) { err = DB_DECRYPTION_FAILED; mtr_commit(&mtr); return err; diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 773b03775be..85b06412558 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -1109,7 +1109,11 @@ retry_page_get: buf_mode, file, line, mtr, &err); tree_blocks[n_blocks] = block; + /* Note that block==NULL signifies either an error or change + buffering. */ + if (err != DB_SUCCESS) { + ut_ad(block == NULL); if (err == DB_DECRYPTION_FAILED) { ib_push_warning((void *)NULL, DB_DECRYPTION_FAILED, @@ -1117,7 +1121,7 @@ retry_page_get: " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } goto func_exit; @@ -1230,7 +1234,7 @@ retry_page_get: " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } goto func_exit; @@ -1259,7 +1263,7 @@ retry_page_get: " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } goto func_exit; @@ -2057,7 +2061,7 @@ btr_cur_open_at_index_side_func( ulint offsets_[REC_OFFS_NORMAL_SIZE]; ulint* offsets = offsets_; dberr_t err = DB_SUCCESS; - + rec_offs_init(offsets_); estimate = latch_mode & BTR_ESTIMATE; @@ -2139,8 +2143,7 @@ btr_cur_open_at_index_side_func( height = ULINT_UNDEFINED; for (;;) { - buf_block_t* block=NULL; - page_t* page=NULL; + buf_block_t* block; ulint rw_latch; ut_ad(n_blocks < BTR_MAX_LEVELS); @@ -2156,6 +2159,7 @@ btr_cur_open_at_index_side_func( tree_savepoints[n_blocks] = mtr_set_savepoint(mtr); block = buf_page_get_gen(page_id, page_size, rw_latch, NULL, BUF_GET, file, line, mtr, &err); + ut_ad((block != NULL) == (err == DB_SUCCESS)); tree_blocks[n_blocks] = block; if (err != DB_SUCCESS) { @@ -2166,13 +2170,13 @@ btr_cur_open_at_index_side_func( " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } goto exit_loop; } - page = buf_block_get_frame(block); + const page_t* page = buf_block_get_frame(block); if (height == ULINT_UNDEFINED && page_is_leaf(page) @@ -2516,6 +2520,8 @@ btr_cur_open_at_rnd_pos_func( BUF_GET, file, line, mtr, &err); tree_blocks[n_blocks] = block; + ut_ad((block != NULL) == (err == DB_SUCCESS)); + if (err != DB_SUCCESS) { if (err == DB_DECRYPTION_FAILED) { ib_push_warning((void *)NULL, @@ -2524,8 +2530,9 @@ btr_cur_open_at_rnd_pos_func( " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } + goto exit_loop; } @@ -5323,6 +5330,8 @@ btr_estimate_n_rows_in_range_on_level( NULL, BUF_GET_POSSIBLY_FREED, __FILE__, __LINE__, &mtr, &err); + ut_ad((block != NULL) == (err == DB_SUCCESS)); + if (err != DB_SUCCESS) { if (err == DB_DECRYPTION_FAILED) { ib_push_warning((void *)NULL, @@ -5331,7 +5340,7 @@ btr_estimate_n_rows_in_range_on_level( " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } mtr_commit(&mtr); @@ -5518,6 +5527,10 @@ btr_estimate_n_rows_in_range_low( mtr_commit(&mtr); + if (!index->is_readable()) { + return 0; + } + mtr_start(&mtr); cursor.path_arr = path2; @@ -5997,6 +6010,11 @@ btr_estimate_number_of_different_key_vals( because otherwise our algorithm would give a wrong estimate for an index where there is just one key value. */ + if (index->table->file_unreadable) { + mtr_commit(&mtr); + goto exit_loop; + } + page = btr_cur_get_page(&cursor); rec = page_rec_get_next(page_get_infimum_rec(page)); @@ -6080,6 +6098,7 @@ btr_estimate_number_of_different_key_vals( mtr_commit(&mtr); } +exit_loop: /* If we saw k borders between different key values on n_sample_pages leaf pages, we can estimate how many there will be in index->stat_n_leaf_pages */ diff --git a/storage/innobase/btr/btr0defragment.cc b/storage/innobase/btr/btr0defragment.cc index 06cd1315590..d21d306cd91 100644 --- a/storage/innobase/btr/btr0defragment.cc +++ b/storage/innobase/btr/btr0defragment.cc @@ -227,7 +227,7 @@ btr_defragment_add_index( page = buf_block_get_frame(block); } - if (page == NULL && index->table->is_encrypted) { + if (page == NULL && index->table->file_unreadable) { mtr_commit(&mtr); *err = DB_DECRYPTION_FAILED; return NULL; diff --git a/storage/innobase/btr/btr0pcur.cc b/storage/innobase/btr/btr0pcur.cc index 445aa3504b7..fdde6f5d3e7 100644 --- a/storage/innobase/btr/btr0pcur.cc +++ b/storage/innobase/btr/btr0pcur.cc @@ -418,6 +418,11 @@ btr_pcur_move_to_next_page( cursor->old_stored = false; page = btr_pcur_get_page(cursor); + + if (UNIV_UNLIKELY(!page)) { + return; + } + next_page_no = btr_page_get_next(page, mtr); ut_ad(next_page_no != FIL_NULL); @@ -438,6 +443,10 @@ btr_pcur_move_to_next_page( block->page.size, mode, btr_pcur_get_btr_cur(cursor)->index, mtr); + if (UNIV_UNLIKELY(!next_block)) { + return; + } + next_page = buf_block_get_frame(next_block); #ifdef UNIV_BTR_DEBUG ut_a(page_is_comp(next_page) == page_is_comp(page)); diff --git a/storage/innobase/btr/btr0scrub.cc b/storage/innobase/btr/btr0scrub.cc index 307d6d0ec6c..f427a137c4d 100644 --- a/storage/innobase/btr/btr0scrub.cc +++ b/storage/innobase/btr/btr0scrub.cc @@ -140,6 +140,7 @@ btr_scrub_lock_dict_func(ulint space_id, bool lock_to_close_table, } else { return false; } + os_thread_sleep(250000); time_t now = time(0); @@ -571,7 +572,7 @@ btr_scrub_table_needs_scrubbing( return false; } - if (table->corrupted) { + if (!table->is_readable()) { return false; } diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 754b2cd1c25..a5e9c9e8a36 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -412,16 +412,6 @@ bool buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) MY_ATTRIBUTE((nonnull)); -/** Check if page is maybe compressed, encrypted or both when we encounter -corrupted page. Note that we can't be 100% sure if page is corrupted -or decrypt/decompress just failed. -@param[in,out] bpage Page -@return true if page corrupted, false if not */ -static -bool -buf_page_check_corrupt(buf_page_t* bpage) - MY_ATTRIBUTE((nonnull, warn_unused_result)); - /* prototypes for new functions added to ha_innodb.cc */ trx_t* innobase_get_trx(); @@ -970,25 +960,23 @@ buf_page_is_corrupted( SRV_CHECKSUM_ALGORITHM_NONE, page_id); } -#endif /* !UNIV_INNOCHECKSUM */ - -#ifdef UNIV_INNOCHECKSUM +#else /* !UNIV_INNOCHECKSUM */ if (log_file) { - fprintf(log_file, "page::%lu;" - " old style: calculated = " ULINTPF ";" - " recorded = " ULINTPF "\n", cur_page_num, + " old style: calculated = %u;" + " recorded = " ULINTPF "\n", + cur_page_num, buf_calc_page_old_checksum(read_buf), checksum_field2); fprintf(log_file, "page::%lu;" - " new style: calculated = " ULINTPF ";" + " new style: calculated = %u;" " crc32 = %u; recorded = " ULINTPF "\n", cur_page_num, buf_calc_page_new_checksum(read_buf), buf_calc_page_crc32(read_buf), checksum_field1); } -#endif /* UNIV_INNOCHECKSUM */ +#endif /* !UNIV_INNOCHECKSUM */ return(false); } @@ -1055,24 +1043,22 @@ buf_page_is_corrupted( SRV_CHECKSUM_ALGORITHM_NONE, page_id); } -#endif /* !UNIV_INNOCHECKSUM */ - -#ifdef UNIV_INNOCHECKSUM +#else /* !UNIV_INNOCHECKSUM */ if (log_file) { fprintf(log_file, "page::%lu;" - " old style: calculated = %lu;" + " old style: calculated = %u;" " recorded = %lu\n", cur_page_num, buf_calc_page_old_checksum(read_buf), checksum_field2); fprintf(log_file, "page::%lu;" - " new style: calculated = %lu;" + " new style: calculated = %u;" " crc32 = %u; recorded = %lu\n", cur_page_num, buf_calc_page_new_checksum(read_buf), buf_calc_page_crc32(read_buf), checksum_field1); } -#endif /* UNIV_INNOCHECKSUM */ +#endif /* !UNIV_INNOCHECKSUM */ return(false); } @@ -1423,7 +1409,6 @@ buf_block_init( block->page.buf_fix_count = 0; block->page.io_fix = BUF_IO_NONE; block->page.flush_observer = NULL; - block->page.key_version = 0; block->page.encrypted = false; block->page.real_size = 0; block->page.write_size = 0; @@ -3709,7 +3694,6 @@ buf_page_get_zip( ibool discard_attempted = FALSE; ibool must_read; buf_pool_t* buf_pool = buf_pool_get(page_id); - buf_page_t* rpage = NULL; buf_pool->stat.n_page_gets++; @@ -3728,7 +3712,14 @@ lookup: /* Page not in buf_pool: needs to be read from file */ ut_ad(!hash_lock); - buf_read_page(page_id, page_size, &rpage); + dberr_t err = buf_read_page(page_id, page_size); + + if (err != DB_SUCCESS) { + ib::error() << "Reading compressed page " << page_id + << " failed with error: " << ut_strerr(err); + + goto err_exit; + } #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG ut_a(++buf_dbg_counter % 5771 || buf_validate()); @@ -4205,7 +4196,6 @@ loop: } if (block == NULL) { - buf_page_t* bpage=NULL; /* Page not in buf_pool: needs to be read from file */ @@ -4263,7 +4253,20 @@ loop: return(NULL); } - if (buf_read_page(page_id, page_size, &bpage)) { + /* The call path is buf_read_page() -> + buf_read_page_low() (fil_io()) -> + buf_page_io_complete() -> + buf_decrypt_after_read(). Here fil_space_t* is used + and we decrypt -> buf_page_check_corrupt() where page + checksums are compared. Decryption, decompression as + well as error handling takes place at a lower level. + Here we only need to know whether the page really is + corrupted, or if an encrypted page with a valid + checksum cannot be decypted. */ + + dberr_t local_err = buf_read_page(page_id, page_size); + + if (local_err == DB_SUCCESS) { buf_read_ahead_random(page_id, page_size, ibuf_inside(mtr)); @@ -4271,82 +4274,44 @@ loop: } else if (retries < BUF_PAGE_READ_MAX_RETRIES) { ++retries; - bool corrupted = false; - - if (bpage) { - corrupted = buf_page_check_corrupt(bpage); - } - - /* Do not try again for encrypted pages */ - if (corrupted && bpage->encrypted) { - BPageMutex* pmutex = buf_page_get_mutex(bpage); - - buf_pool = buf_pool_from_bpage(bpage); - buf_pool_mutex_enter(buf_pool); - mutex_enter(pmutex); - - ut_ad(buf_pool->n_pend_reads > 0); - my_atomic_addlint(&buf_pool->n_pend_reads, -1); - buf_page_set_io_fix(bpage, BUF_IO_NONE); - mutex_exit(pmutex); - buf_LRU_free_page(bpage, true); - buf_pool_mutex_exit(buf_pool); - rw_lock_x_unlock_gen(&((buf_block_t*) bpage)->lock, - BUF_IO_READ); - - if (err) { - *err = DB_DECRYPTION_FAILED; - } - - return (NULL); - } - DBUG_EXECUTE_IF( "innodb_page_corruption_retries", retries = BUF_PAGE_READ_MAX_RETRIES; ); } else { - bool corrupted = false; - - if (bpage) { - corrupted = buf_page_check_corrupt(bpage); + if (err) { + *err = local_err; } - if (corrupted && !bpage->encrypted) { - ib::fatal() << "Unable to read page " << page_id - << " into the buffer pool after " - << BUF_PAGE_READ_MAX_RETRIES << " attempts." - " The most probable cause of this error may" - " be that the table has been corrupted. Or," - " the table was compressed with with an" - " algorithm that is not supported by this" - " instance. If it is not a decompress failure," - " you can try to fix this problem by using" - " innodb_force_recovery." - " Please see " REFMAN " for more" - " details. Aborting..."; - } else { - BPageMutex* pmutex = buf_page_get_mutex(bpage); - - buf_pool = buf_pool_from_bpage(bpage); - buf_pool_mutex_enter(buf_pool); - mutex_enter(pmutex); - - ut_ad(buf_pool->n_pend_reads > 0); - my_atomic_addlint(&buf_pool->n_pend_reads, -1); - buf_page_set_io_fix(bpage, BUF_IO_NONE); - mutex_exit(pmutex); - buf_LRU_free_page(bpage, true); - buf_pool_mutex_exit(buf_pool); - rw_lock_x_unlock_gen(&((buf_block_t*) bpage)->lock, - BUF_IO_READ); - - if (err) { - *err = DB_DECRYPTION_FAILED; - } + /* Pages whose encryption key is unavailable or used + key, encryption algorithm or encryption method is + incorrect are marked as encrypted in + buf_page_check_corrupt(). Unencrypted page could be + corrupted in a way where the key_id field is + nonzero. There is no checksum on field + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION. */ + if (local_err == DB_DECRYPTION_FAILED) { + return (NULL); + } + /* Try to set table as corrupted instead of + asserting. */ + if (page_id.space() != TRX_SYS_SPACE && + dict_set_corrupted_by_space(page_id.space())) { return (NULL); } + + ib::fatal() << "Unable to read page " << page_id + << " into the buffer pool after " + << BUF_PAGE_READ_MAX_RETRIES + << ". The most probable cause" + " of this error may be that the" + " table has been corrupted." + " You can try to fix this" + " problem by using" + " innodb_force_recovery." + " Please see " REFMAN " for more" + " details. Aborting..."; } #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG @@ -5113,7 +5078,6 @@ buf_page_init_low( bpage->newest_modification = 0; bpage->oldest_modification = 0; bpage->write_size = 0; - bpage->key_version = 0; bpage->encrypted = false; bpage->real_size = 0; bpage->slot = NULL; @@ -5708,72 +5672,73 @@ buf_page_monitor( /********************************************************************//** Mark a table with the specified space pointed by bpage->id.space() corrupted. Also remove the bpage from LRU list. -@return TRUE if successful */ +@param[in,out] bpage Block */ static -ibool -buf_mark_space_corrupt( -/*===================*/ - buf_page_t* bpage) /*!< in: pointer to the block in question */ +void +buf_mark_space_corrupt(buf_page_t* bpage) { buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); const ibool uncompressed = (buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); - ib_uint32_t space = bpage->id.space(); - ibool ret = TRUE; + uint32_t space = bpage->id.space(); - if (!bpage->encrypted) { - /* First unfix and release lock on the bpage */ - buf_pool_mutex_enter(buf_pool); - mutex_enter(buf_page_get_mutex(bpage)); - ut_ad(buf_page_get_io_fix(bpage) == BUF_IO_READ); - ut_ad(bpage->buf_fix_count == 0); - - /* Set BUF_IO_NONE before we remove the block from LRU list */ - buf_page_set_io_fix(bpage, BUF_IO_NONE); + /* First unfix and release lock on the bpage */ + buf_pool_mutex_enter(buf_pool); + mutex_enter(buf_page_get_mutex(bpage)); + ut_ad(buf_page_get_io_fix(bpage) == BUF_IO_READ); + ut_ad(bpage->buf_fix_count == 0); - if (uncompressed) { - rw_lock_x_unlock_gen( - &((buf_block_t*) bpage)->lock, - BUF_IO_READ); - } + /* Set BUF_IO_NONE before we remove the block from LRU list */ + buf_page_set_io_fix(bpage, BUF_IO_NONE); - mutex_exit(buf_page_get_mutex(bpage)); + if (uncompressed) { + rw_lock_x_unlock_gen( + &((buf_block_t*) bpage)->lock, + BUF_IO_READ); } - /* Find the table with specified space id, and mark it corrupted */ - if (dict_set_corrupted_by_space(space)) { - if (!bpage->encrypted) { - buf_LRU_free_one_page(bpage); - } + mutex_exit(buf_page_get_mutex(bpage)); + + /* If block is not encrypted find the table with specified + space id, and mark it corrupted. Encrypted tables + are marked unusable later e.g. in ::open(). */ + if (!bpage->encrypted) { + dict_set_corrupted_by_space(space); } else { - ret = FALSE; + dict_set_encrypted_by_space(space); } - if (!bpage->encrypted) { - ut_ad(buf_pool->n_pend_reads > 0); - buf_pool->n_pend_reads--; + /* After this point bpage can't be referenced. */ + buf_LRU_free_one_page(bpage); - buf_pool_mutex_exit(buf_pool); - } + ut_ad(buf_pool->n_pend_reads > 0); + buf_pool->n_pend_reads--; - return(ret); + buf_pool_mutex_exit(buf_pool); } /** Check if page is maybe compressed, encrypted or both when we encounter corrupted page. Note that we can't be 100% sure if page is corrupted or decrypt/decompress just failed. @param[in,out] bpage Page -@return true if page corrupted, false if not */ +@param[in,out] space tablespace +@retval DB_SUCCESS if page has been read and is not corrupted, +@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted, +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but +after decryption normal page checksum does not match. */ static -bool -buf_page_check_corrupt(buf_page_t* bpage) +dberr_t +buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space) { + ut_ad(space->n_pending_ios > 0); + byte* dst_frame = (bpage->zip.data) ? bpage->zip.data : ((buf_block_t*) bpage)->frame; - fil_space_t* space = fil_space_acquire_silent(bpage->id.space()); bool still_encrypted = false; + dberr_t err = DB_SUCCESS; bool corrupted = false; - fil_space_crypt_t* crypt_data = space ? space->crypt_data : NULL; + + fil_space_crypt_t* crypt_data = space->crypt_data; /* In buf_decrypt_after_read we have either decrypted the page if page post encryption checksum matches and used key_id is found @@ -5784,9 +5749,9 @@ buf_page_check_corrupt(buf_page_t* bpage) still_encrypted = crypt_data && crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && !bpage->encrypted - && fil_space_verify_crypt_checksum(dst_frame, bpage->size, - bpage->id.space(), - bpage->id.page_no()); + && fil_space_verify_crypt_checksum( + dst_frame, bpage->size, + bpage->id.space(), bpage->id.page_no()); if (!still_encrypted) { /* If traditional checksums match, we assume that page is @@ -5796,6 +5761,8 @@ buf_page_check_corrupt(buf_page_t* bpage) if (!corrupted) { bpage->encrypted = false; + } else { + err = DB_PAGE_CORRUPTED; } } @@ -5806,16 +5773,18 @@ buf_page_check_corrupt(buf_page_t* bpage) buf_page_io_complete(). */ } else if (still_encrypted || (bpage->encrypted && corrupted)) { bpage->encrypted = true; - corrupted = true; + err = DB_DECRYPTION_FAILED; ib::error() - << "The page " << bpage->id << " in file " - << (space && space->name ? space->name : "NULL") - << " cannot be decrypted."; + << "The page " << bpage->id << " in file '" + << space->chain.start->name + << "' cannot be decrypted."; ib::info() << "However key management plugin or used key_version " - << bpage->key_version << " is not found or" + << mach_read_from_4(dst_frame + + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) + << " is not found or" " used encryption algorithm or method does not match."; if (bpage->id.space() != TRX_SYS_SPACE) { @@ -5827,32 +5796,28 @@ buf_page_check_corrupt(buf_page_t* bpage) } } - if (space) { - fil_space_release(space); - } - - return corrupted; + return (err); } -/********************************************************************//** -Completes an asynchronous read or write request of a file page to or from -the buffer pool. -@return true if successful */ -bool -buf_page_io_complete( -/*=================*/ - buf_page_t* bpage, /*!< in: pointer to the block in question */ - bool evict) /*!< in: whether or not to evict the page - from LRU list. */ - +/** Complete a read or write request of a file page to or from the buffer pool. +@param[in,out] bpage Page to complete +@param[in] evict whether or not to evict the page + from LRU list. +@return whether the operation succeeded +@retval DB_SUCCESS always when writing, or if a read page was OK +@retval DB_TABLESPACE_DELETED if the tablespace does not exist +@retval DB_PAGE_CORRUPTED if the checksum fails on a page read +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but + after decryption normal page checksum does + not match */ +UNIV_INTERN +dberr_t +buf_page_io_complete(buf_page_t* bpage, bool evict) { enum buf_io_fix io_type; buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); - const ibool uncompressed = (buf_page_get_state(bpage) + const bool uncompressed = (buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); - byte* frame = NULL; - bool corrupted = false; - ut_a(buf_page_in_file(bpage)); /* We do not need protect io_fix here by mutex to read @@ -5863,46 +5828,42 @@ buf_page_io_complete( io_type = buf_page_get_io_fix(bpage); ut_ad(io_type == BUF_IO_READ || io_type == BUF_IO_WRITE); + ut_ad(bpage->size.is_compressed() == (bpage->zip.data != NULL)); + ut_ad(uncompressed || bpage->zip.data); if (io_type == BUF_IO_READ) { - ulint read_page_no; - ulint read_space_id; + ulint read_page_no = 0; + ulint read_space_id = 0; + uint key_version = 0; ut_ad(bpage->zip.data != NULL || ((buf_block_t*)bpage)->frame != NULL); fil_space_t* space = fil_space_acquire_for_io( bpage->id.space()); if (!space) { - return false; + return DB_TABLESPACE_DELETED; } buf_page_decrypt_after_read(bpage, space); - if (bpage->size.is_compressed()) { - frame = bpage->zip.data; - } else { - frame = ((buf_block_t*) bpage)->frame; - } - - if (bpage->size.is_compressed()) { - frame = bpage->zip.data; - buf_pool->n_pend_unzip++; + byte* frame = bpage->zip.data + ? bpage->zip.data + : reinterpret_cast<buf_block_t*>(bpage)->frame; + dberr_t err; - if (uncompressed - && !buf_zip_decompress((buf_block_t*) bpage, - FALSE)) { - - buf_pool->n_pend_unzip--; + if (bpage->zip.data && uncompressed) { + my_atomic_addlint(&buf_pool->n_pend_unzip, 1); + ibool ok = buf_zip_decompress((buf_block_t*) bpage, + FALSE); + my_atomic_addlint(&buf_pool->n_pend_unzip, -1); + if (!ok) { ib::info() << "Page " << bpage->id << " zip_decompress failure."; + err = DB_PAGE_CORRUPTED; goto database_corrupted; } - buf_pool->n_pend_unzip--; - } else { - ut_a(uncompressed); - frame = ((buf_block_t*) bpage)->frame; } /* If this page is not uninitialized and not in the @@ -5911,6 +5872,8 @@ buf_page_io_complete( read_page_no = mach_read_from_4(frame + FIL_PAGE_OFFSET); read_space_id = mach_read_from_4( frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + key_version = mach_read_from_4( + frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); if (bpage->id.space() == TRX_SYS_SPACE && buf_dblwr_page_inside(bpage->id.page_no())) { @@ -5934,28 +5897,29 @@ buf_page_io_complete( << ", should be " << bpage->id; } - corrupted = buf_page_check_corrupt(bpage); + err = buf_page_check_corrupt(bpage, space); database_corrupted: - if (corrupted) { + if (err != DB_SUCCESS) { /* Not a real corruption if it was triggered by error injection */ DBUG_EXECUTE_IF( "buf_page_import_corrupt_failure", if (bpage->id.space() > srv_undo_tablespaces_open - && bpage->id.space() != SRV_TMP_SPACE_ID - && buf_mark_space_corrupt(bpage)) { + && bpage->id.space() != SRV_TMP_SPACE_ID) { + buf_mark_space_corrupt(bpage); ib::info() << "Simulated IMPORT " "corruption"; fil_space_release_for_io(space); - return(true); + return(err); } + err = DB_SUCCESS; goto page_not_corrupt; ); - if (!bpage->encrypted) { + if (err == DB_PAGE_CORRUPTED) { ib::error() << "Database page corruption on disk" " or a failed file read of tablespace " @@ -5985,38 +5949,14 @@ database_corrupted: /* If page space id is larger than TRX_SYS_SPACE (0), we will attempt to mark the corresponding table as corrupted instead of crashing server */ - - if (bpage->id.space() > TRX_SYS_SPACE - && buf_mark_space_corrupt(bpage)) { - fil_space_release_for_io(space); - return(false); - } else { - if (!bpage->encrypted) { - ib::fatal() - << "Aborting because of a" - " corrupt database page in" - " the system tablespace. Or, " - " there was a failure in" - " tagging the tablespace " - " as corrupt."; - } - - ib_push_warning(innobase_get_trx(), DB_DECRYPTION_FAILED, - "Table in tablespace %u encrypted." - "However key management plugin or used key_id %lu is not found or" - " used encryption algorithm or method does not match." - " Can't continue opening the table.", - bpage->id.space(), bpage->key_version); - - if (bpage->encrypted && bpage->id.space() > TRX_SYS_SPACE) { - buf_mark_space_corrupt(bpage); - } else { - ut_error; - } - - fil_space_release_for_io(space); - return(false); + if (bpage->id.space() == TRX_SYS_SPACE) { + ib::fatal() << "Aborting because of" + " a corrupt database page."; } + + buf_mark_space_corrupt(bpage); + fil_space_release_for_io(space); + return(err); } } @@ -6046,7 +5986,7 @@ database_corrupted: << bpage->id.space() << " encrypted. However key " "management plugin or used " - << "key_version " << bpage->key_version + << "key_version " << key_version << "is not found or" " used encryption algorithm or method does not match." " Can't continue opening the table."; @@ -6150,7 +6090,7 @@ database_corrupted: buf_pool_mutex_exit(buf_pool); - return(true); + return DB_SUCCESS; } /*********************************************************************//** @@ -6173,9 +6113,7 @@ buf_all_freed_instance( for (i = buf_pool->n_chunks; i--; chunk++) { - const buf_block_t* block = buf_chunk_not_freed(chunk); - - if (UNIV_LIKELY_NULL(block) && block->page.key_version == 0) { + if (const buf_block_t* block = buf_chunk_not_freed(chunk)) { ib::fatal() << "Page " << block->page.id << " still fixed or dirty"; } @@ -7324,33 +7262,29 @@ buf_page_encrypt_before_write( switch (bpage->id.page_no()) { case 0: /* Page 0 of a tablespace is not encrypted/compressed */ - ut_ad(bpage->key_version == 0); return src_frame; case TRX_SYS_PAGE_NO: if (bpage->id.space() == TRX_SYS_SPACE) { /* don't encrypt/compress page as it contains address to dblwr buffer */ - bpage->key_version = 0; return src_frame; } } - const page_size_t page_size(space->flags); fil_space_crypt_t* crypt_data = space->crypt_data; + const bool encrypted = crypt_data && !crypt_data->not_encrypted() && crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && (!crypt_data->is_default_encryption() || srv_encrypt_tables); - if (!encrypted) { - bpage->key_version = 0; - } - bool page_compressed = FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags); if (!encrypted && !page_compressed) { - /* No need to encrypt or page compress the page */ + /* No need to encrypt or page compress the page. + Clear key-version & crypt-checksum. */ + memset(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); return src_frame; } @@ -7370,11 +7304,6 @@ buf_page_encrypt_before_write( src_frame, dst_frame); - uint32_t key_version = mach_read_from_4( - dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - ut_ad(key_version == 0 || key_version >= bpage->key_version); - bpage->key_version = key_version; - bpage->real_size = page_size.physical(); slot->out_buf = dst_frame = tmp; ut_d(fil_page_type_validate(tmp)); @@ -7386,7 +7315,7 @@ buf_page_encrypt_before_write( space, (byte *)src_frame, slot->comp_buf, - page_size.logical(), + srv_page_size, fsp_flags_get_page_compression_level(space->flags), fil_space_get_block_size(space, bpage->id.page_no()), encrypted, @@ -7440,8 +7369,6 @@ buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); bool success = true; - bpage->key_version = key_version; - if (bpage->id.page_no() == 0) { /* File header pages are not encrypted/compressed */ return (true); @@ -7481,6 +7408,10 @@ buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) if (!fil_space_verify_crypt_checksum( dst_frame, size, bpage->id.space(), bpage->id.page_no())) { + if (space->crypt_data->type + != CRYPT_SCHEME_UNENCRYPTED) { + bpage->encrypted=true; + } return (false); } diff --git a/storage/innobase/buf/buf0checksum.cc b/storage/innobase/buf/buf0checksum.cc index 94eafec0584..4b56cc81e98 100644 --- a/storage/innobase/buf/buf0checksum.cc +++ b/storage/innobase/buf/buf0checksum.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -80,15 +81,13 @@ buf_calc_page_crc32( return(c1 ^ c2); } -/********************************************************************//** -Calculates a page checksum which is stored to the page when it is written +/** Calculate a checksum which is stored to the page when it is written to a file. Note that we must be careful to calculate the same value on 32-bit and 64-bit architectures. +@param[in] page file page (srv_page_size bytes) @return checksum */ -ulint -buf_calc_page_new_checksum( -/*=======================*/ - const byte* page) /*!< in: buffer page */ +uint32_t +buf_calc_page_new_checksum(const byte* page) { ulint checksum; @@ -106,40 +105,29 @@ buf_calc_page_new_checksum( + ut_fold_binary(page + FIL_PAGE_DATA, UNIV_PAGE_SIZE - FIL_PAGE_DATA - FIL_PAGE_END_LSN_OLD_CHKSUM); - checksum = checksum & 0xFFFFFFFFUL; - - return(checksum); + return(static_cast<uint32_t>(checksum)); } -/********************************************************************//** -In versions < 4.0.14 and < 4.1.1 there was a bug that the checksum only -looked at the first few bytes of the page. This calculates that old -checksum. +/** In MySQL before 4.0.14 or 4.1.1 there was an InnoDB bug that +the checksum only looked at the first few bytes of the page. +This calculates that old checksum. NOTE: we must first store the new formula checksum to FIL_PAGE_SPACE_OR_CHKSUM before calculating and storing this old checksum because this takes that field as an input! +@param[in] page file page (srv_page_size bytes) @return checksum */ -ulint -buf_calc_page_old_checksum( -/*=======================*/ - const byte* page) /*!< in: buffer page */ +uint32_t +buf_calc_page_old_checksum(const byte* page) { - ulint checksum; - - checksum = ut_fold_binary(page, FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - - checksum = checksum & 0xFFFFFFFFUL; - - return(checksum); + return(static_cast<uint32_t> + (ut_fold_binary(page, FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION))); } -/********************************************************************//** -Return a printable string describing the checksum algorithm. +/** Return a printable string describing the checksum algorithm. +@param[in] algo algorithm @return algorithm name */ const char* -buf_checksum_algorithm_name( -/*========================*/ - srv_checksum_algorithm_t algo) /*!< in: algorithm */ +buf_checksum_algorithm_name(srv_checksum_algorithm_t algo) { switch (algo) { case SRV_CHECKSUM_ALGORITHM_CRC32: diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index d0c0316bf13..e3302fdba59 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1119,15 +1119,20 @@ buf_flush_write_block_low( fil_flush(space); } - /* true means we want to evict this page from the - LRU list as well. */ /* The tablespace could already have been dropped, because fil_io(request, sync) would already have decremented the node->n_pending. However, buf_page_io_complete() only needs to look up the tablespace during read requests, not during writes. */ ut_ad(buf_page_get_io_fix(bpage) == BUF_IO_WRITE); +#ifdef UNIV_DEBUG + dberr_t err = +#endif + /* true means we want to evict this page from the + LRU list as well. */ buf_page_io_complete(bpage, true); + + ut_ad(err == DB_SUCCESS); } fil_space_release_for_io(space); diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index f5bd088ce5c..53946dc1905 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2016 MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -118,8 +118,7 @@ buf_read_page_low( ulint mode, const page_id_t& page_id, const page_size_t& page_size, - bool unzip, - buf_page_t** rbpage) /*!< out: page */ + bool unzip) { buf_page_t* bpage; @@ -155,12 +154,9 @@ buf_read_page_low( return(0); } - DBUG_PRINT("ib_buf", ("read page %u:%u size=%u unzip=%u,%s", - (unsigned) page_id.space(), - (unsigned) page_id.page_no(), - (unsigned) page_size.physical(), - (unsigned) unzip, - sync ? "sync" : "async")); + DBUG_LOG("ib_buf", + "read page " << page_id << " size=" << page_size.physical() + << " unzip=" << unzip << ',' << (sync ? "sync" : "async")); ut_ad(buf_page_in_file(bpage)); @@ -213,19 +209,13 @@ buf_read_page_low( if (sync) { /* The i/o is already completed when we arrive from fil_read */ + *err = buf_page_io_complete(bpage); - if (!buf_page_io_complete(bpage)) { - if (rbpage) { - *rbpage = bpage; - } + if (*err != DB_SUCCESS) { return(0); } } - if (rbpage) { - *rbpage = bpage; - } - return(1); } @@ -256,7 +246,7 @@ buf_read_ahead_random( ulint ibuf_mode; ulint count; ulint low, high; - dberr_t err; + dberr_t err = DB_SUCCESS; ulint i; const ulint buf_read_ahead_random_area = BUF_READ_AHEAD_AREA(buf_pool); @@ -369,19 +359,24 @@ read_ahead: const page_id_t cur_page_id(page_id.space(), i); if (!ibuf_bitmap_page(cur_page_id, page_size)) { - buf_page_t* rpage = NULL; count += buf_read_page_low( &err, false, IORequest::DO_NOT_WAKE, ibuf_mode, - cur_page_id, page_size, false, &rpage); + cur_page_id, page_size, false); - if (err == DB_TABLESPACE_DELETED) { - ib::warn() << "Random readahead trying to" + switch (err) { + case DB_SUCCESS: + case DB_TABLESPACE_TRUNCATED: + break; + case DB_TABLESPACE_DELETED: + ib::info() << "Random readahead trying to" " access page " << cur_page_id << " in nonexisting or" " being-dropped tablespace"; break; + default: + ut_error; } } } @@ -414,15 +409,18 @@ an exclusive lock on the buffer frame. The flag is cleared and the x-lock released by the i/o-handler thread. @param[in] page_id page id @param[in] page_size page size -@return TRUE if page has been read in, FALSE in case of failure */ -ibool +@retval DB_SUCCESS if the page was read and is not corrupted, +@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted, +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but +after decryption normal page checksum does not match. +@retval DB_TABLESPACE_DELETED if tablespace .ibd file is missing */ +dberr_t buf_read_page( const page_id_t& page_id, - const page_size_t& page_size, - buf_page_t** bpage) /*!< out: page */ + const page_size_t& page_size) { ulint count; - dberr_t err; + dberr_t err = DB_SUCCESS; /* We do synchronous IO because our AIO completion code is sub-optimal. See buf_page_io_complete(), we have to @@ -432,19 +430,19 @@ buf_read_page( count = buf_read_page_low( &err, true, - 0, BUF_READ_ANY_PAGE, page_id, page_size, false, bpage); + 0, BUF_READ_ANY_PAGE, page_id, page_size, false); srv_stats.buf_pool_reads.add(count); if (err == DB_TABLESPACE_DELETED) { - ib::error() << "trying to read page " << page_id + ib::info() << "trying to read page " << page_id << " in nonexisting or being-dropped tablespace"; } /* Increment number of I/O operations used for LRU policy. */ buf_LRU_stat_inc_io(); - return(count > 0); + return(err); } /** High-level function which reads a page asynchronously from a file to the @@ -453,9 +451,8 @@ an exclusive lock on the buffer frame. The flag is cleared and the x-lock released by the i/o-handler thread. @param[in] page_id page id @param[in] page_size page size -@param[in] sync true if synchronous aio is desired -@return TRUE if page has been read in, FALSE in case of failure */ -ibool +@param[in] sync true if synchronous aio is desired */ +void buf_read_page_background( const page_id_t& page_id, const page_size_t& page_size, @@ -463,13 +460,30 @@ buf_read_page_background( { ulint count; dberr_t err; - buf_page_t* rbpage = NULL; count = buf_read_page_low( &err, sync, IORequest::DO_NOT_WAKE | IORequest::IGNORE_MISSING, BUF_READ_ANY_PAGE, - page_id, page_size, false, &rbpage); + page_id, page_size, false); + + switch (err) { + case DB_SUCCESS: + case DB_TABLESPACE_TRUNCATED: + break; + case DB_TABLESPACE_DELETED: + ib::info() << "trying to read page " << page_id + << " in the background" + " in a non-existing or being-dropped tablespace"; + break; + case DB_DECRYPTION_FAILED: + ib::error() + << "Background Page read failed to decrypt page " + << page_id; + break; + default: + ut_error; + } srv_stats.buf_pool_reads.add(count); @@ -479,8 +493,6 @@ buf_read_page_background( buffer pool. Since this function is called from buffer pool load these IOs are deliberate and are not part of normal workload we can ignore these in our heuristics. */ - - return(count > 0); } /** Applies linear read-ahead if in the buf_pool the page is a border page of @@ -525,7 +537,7 @@ buf_read_ahead_linear( ulint new_offset; ulint fail_count; ulint low, high; - dberr_t err; + dberr_t err = DB_SUCCESS; ulint i; const ulint buf_read_ahead_linear_area = BUF_READ_AHEAD_AREA(buf_pool); @@ -730,19 +742,28 @@ buf_read_ahead_linear( const page_id_t cur_page_id(page_id.space(), i); if (!ibuf_bitmap_page(cur_page_id, page_size)) { - buf_page_t* rpage = NULL; - count += buf_read_page_low( &err, false, IORequest::DO_NOT_WAKE, - ibuf_mode, cur_page_id, page_size, false, &rpage); + ibuf_mode, cur_page_id, page_size, false); - if (err == DB_TABLESPACE_DELETED) { - ib::warn() << "linear readahead trying to" + switch (err) { + case DB_SUCCESS: + case DB_TABLESPACE_TRUNCATED: + break; + case DB_TABLESPACE_DELETED: + ib::info() << "linear readahead trying to" " access page " << page_id_t(page_id.space(), i) << " in nonexisting or being-dropped" " tablespace"; + case DB_DECRYPTION_FAILED: + ib::error() << "linear readahead failed to" + " decrypt page " + << page_id_t(page_id.space(), i); + break; + default: + ut_error; } } } @@ -797,13 +818,13 @@ buf_read_ibuf_merge_pages( const page_id_t page_id(space_ids[i], page_nos[i]); buf_pool_t* buf_pool = buf_pool_get(page_id); - buf_page_t* rpage = NULL; bool found; const page_size_t page_size(fil_space_get_page_size( space_ids[i], &found)); if (!found) { +tablespace_deleted: /* The tablespace was not found, remove the entries for that page */ ibuf_merge_or_delete_for_page(NULL, page_id, @@ -822,13 +843,20 @@ buf_read_ibuf_merge_pages( sync && (i + 1 == n_stored), 0, BUF_READ_ANY_PAGE, page_id, page_size, - true, &rpage); - - if (err == DB_TABLESPACE_DELETED) { - /* We have deleted or are deleting the single-table - tablespace: remove the entries for that page */ - ibuf_merge_or_delete_for_page(NULL, page_id, - &page_size, FALSE); + true); + + switch(err) { + case DB_SUCCESS: + case DB_TABLESPACE_TRUNCATED: + break; + case DB_TABLESPACE_DELETED: + goto tablespace_deleted; + case DB_DECRYPTION_FAILED: + ib::error() << "Failed to decrypt page " << page_id + << " for change buffer merge"; + break; + default: + ut_error; } } @@ -855,9 +883,6 @@ buf_read_recv_pages( const ulint* page_nos, ulint n_stored) { - ulint count; - dberr_t err; - ulint i; fil_space_t* space = fil_space_get(space_id); if (space == NULL) { @@ -869,12 +894,11 @@ buf_read_recv_pages( const page_size_t page_size(space->flags); - for (i = 0; i < n_stored; i++) { + for (ulint i = 0; i < n_stored; i++) { buf_pool_t* buf_pool; const page_id_t cur_page_id(space_id, page_nos[i]); - buf_page_t* rpage = NULL; - count = 0; + ulint count = 0; buf_pool = buf_pool_get(cur_page_id); while (buf_pool->n_pend_reads >= recv_n_pool_free_frames / 2) { @@ -894,18 +918,25 @@ buf_read_recv_pages( } } - if ((i + 1 == n_stored) && sync) { + dberr_t err; + + if (sync && i + 1 == n_stored) { buf_read_page_low( &err, true, 0, BUF_READ_ANY_PAGE, - cur_page_id, page_size, true, &rpage); + cur_page_id, page_size, true); } else { buf_read_page_low( &err, false, IORequest::DO_NOT_WAKE, BUF_READ_ANY_PAGE, - cur_page_id, page_size, true, &rpage); + cur_page_id, page_size, true); + } + + if (err == DB_DECRYPTION_FAILED) { + ib::error() << "Recovery failed to decrypt page " + << cur_page_id; } } diff --git a/storage/innobase/dict/dict0crea.cc b/storage/innobase/dict/dict0crea.cc index c1bd5c2d368..4add551a8d6 100644 --- a/storage/innobase/dict/dict0crea.cc +++ b/storage/innobase/dict/dict0crea.cc @@ -890,7 +890,7 @@ dict_create_index_tree_step( mtr_start(&mtr); - const bool missing = index->table->ibd_file_missing + const bool missing = index->table->file_unreadable || dict_table_is_discarded(index->table); if (!missing) { @@ -963,8 +963,8 @@ dict_create_index_tree_in_mem( /* Currently this function is being used by temp-tables only. Import/Discard of temp-table is blocked and so this assert. */ - ut_ad(index->table->ibd_file_missing == 0 - && !dict_table_is_discarded(index->table)); + ut_ad(!index->table->file_unreadable); + ut_ad(!dict_table_is_discarded(index->table)); page_no = btr_create( index->type, index->space, diff --git a/storage/innobase/dict/dict0defrag_bg.cc b/storage/innobase/dict/dict0defrag_bg.cc index 016b774217f..ccb73e02f43 100644 --- a/storage/innobase/dict/dict0defrag_bg.cc +++ b/storage/innobase/dict/dict0defrag_bg.cc @@ -320,25 +320,14 @@ dict_stats_save_defrag_stats( { dberr_t ret; - if (index->table->ibd_file_missing) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Cannot save defragment stats because " - ".ibd file is missing.\n"); - return (DB_TABLESPACE_DELETED); - } - if (dict_index_is_corrupted(index)) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Cannot save defragment stats because " - "index is corrupted.\n"); - return(DB_CORRUPTION); - } - if (dict_index_is_univ(index)) { return DB_SUCCESS; } + if (!index->is_readable()) { + return dict_stats_report_error(index->table, true); + } + lint now = (lint) ut_time(); mtr_t mtr; ulint n_leaf_pages; diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index c6420129164..fc4aa761ee6 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -1178,12 +1178,24 @@ dict_table_open_on_name( if (table != NULL) { - /* If table is encrypted return table */ + /* If table is encrypted or corrupted */ if (ignore_err == DICT_ERR_IGNORE_NONE - && table->is_encrypted) { + && !table->is_readable()) { /* Make life easy for drop table. */ dict_table_prevent_eviction(table); + if (table->corrupted) { + + ib::error() << "Table " << table->name + << " is corrupted. Please " + "drop the table and recreate."; + if (!dict_locked) { + mutex_exit(&dict_sys->mutex); + } + + DBUG_RETURN(NULL); + } + if (table->can_be_evicted) { dict_move_to_mru(table); } @@ -1196,22 +1208,6 @@ dict_table_open_on_name( DBUG_RETURN(table); } - /* If table is corrupted, return NULL */ - else if (ignore_err == DICT_ERR_IGNORE_NONE - && table->corrupted) { - /* Make life easy for drop table. */ - dict_table_prevent_eviction(table); - if (!dict_locked) { - mutex_exit(&dict_sys->mutex); - } - - ib::info() << "Table " - << table->name - << " is corrupted. Please drop the table" - " and recreate it"; - - DBUG_RETURN(NULL); - } if (table->can_be_evicted) { dict_move_to_mru(table); @@ -6067,11 +6063,29 @@ dict_set_corrupted_by_space( /* mark the table->corrupted bit only, since the caller could be too deep in the stack for SYS_INDEXES update */ - table->corrupted = TRUE; + table->corrupted = true; + table->file_unreadable = true; return(TRUE); } + +/** Flag a table with specified space_id encrypted in the data dictionary +cache +@param[in] space_id Tablespace id */ +UNIV_INTERN +void +dict_set_encrypted_by_space(ulint space_id) +{ + dict_table_t* table; + + table = dict_find_single_table_by_space(space_id); + + if (table) { + table->file_unreadable = true; + } +} + /**********************************************************************//** Flags an index corrupted both in the data dictionary cache and in the SYS_INDEXES */ @@ -6581,7 +6595,8 @@ dict_table_schema_check( } } - if (table->ibd_file_missing) { + if (!table->is_readable() && + fil_space_get(table->space) == NULL) { /* missing tablespace */ ut_snprintf(errstr, errstr_sz, diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index da72126793f..153c0a49cb9 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -80,7 +80,7 @@ key constraints are loaded into memory. constraints are loaded. @return table, NULL if does not exist; if the table is stored in an .ibd file, but the file does not exist, then we set the -ibd_file_missing flag TRUE in the table object we return */ +file_unreadable flag in the table object we return */ static dict_table_t* dict_load_table_one( @@ -2430,7 +2430,7 @@ dict_load_indexes( dict_mem_index_free(index); goto func_exit; } else if (index->page == FIL_NULL - && !table->ibd_file_missing + && !table->file_unreadable && (!(index->type & DICT_FTS))) { ib::error() << "Trying to load index " << index->name @@ -2551,7 +2551,7 @@ dict_load_table_low(table_name_t& name, const rec_t* rec, dict_table_t** table) *table = dict_mem_table_create( name.m_name, space_id, n_cols + n_v_col, n_v_col, flags, flags2); (*table)->id = table_id; - (*table)->ibd_file_missing = FALSE; + (*table)->file_unreadable = false; return(NULL); } @@ -2644,7 +2644,7 @@ a foreign key references columns in this table. @param[in] ignore_err Error to be ignored when loading table and its index definition @return table, NULL if does not exist; if the table is stored in an -.ibd file, but the file does not exist, then we set the ibd_file_missing +.ibd file, but the file does not exist, then we set the file_unreadable flag in the table object we return. */ dict_table_t* dict_load_table( @@ -2709,7 +2709,7 @@ dict_load_tablespace( if (table->flags2 & DICT_TF2_DISCARDED) { ib::warn() << "Tablespace for table " << table->name << " is set as discarded."; - table->ibd_file_missing = TRUE; + table->file_unreadable = true; return; } @@ -2754,7 +2754,7 @@ dict_load_tablespace( if (err != DB_SUCCESS) { /* We failed to find a sensible tablespace file */ - table->ibd_file_missing = TRUE; + table->file_unreadable = true; } ut_free(filepath); @@ -2777,7 +2777,7 @@ key constraints are loaded into memory. constraints are loaded. @return table, NULL if does not exist; if the table is stored in an .ibd file, but the file does not exist, then we set the -ibd_file_missing flag TRUE in the table object we return */ +file_unreadable flag in the table object we return */ static dict_table_t* dict_load_table_one( @@ -2886,9 +2886,10 @@ err_exit: were not allowed while the table is being locked by a transaction. */ dict_err_ignore_t index_load_err = !(ignore_err & DICT_ERR_IGNORE_RECOVER_LOCK) - && table->ibd_file_missing + && table->file_unreadable ? DICT_ERR_IGNORE_ALL : ignore_err; + err = dict_load_indexes(table, heap, index_load_err); if (err == DB_INDEX_CORRUPT) { @@ -2908,7 +2909,7 @@ err_exit: clust_index = dict_table_get_first_index(table); if (dict_index_is_corrupted(clust_index)) { - table->corrupted = TRUE; + table->corrupted = true; } } } @@ -2947,7 +2948,7 @@ err_exit: of the error condition, since the user may want to dump data from the clustered index. However we load the foreign key information only if all indexes were loaded. */ - if (!cached || table->ibd_file_missing) { + if (!cached || table->file_unreadable) { /* Don't attempt to load the indexes from disk. */ } else if (err == DB_SUCCESS) { err = dict_load_foreigns(table->name.m_name, NULL, @@ -2981,12 +2982,12 @@ err_exit: table = NULL; } else if (dict_index_is_corrupted(index) - && !table->ibd_file_missing) { + && !table->file_unreadable) { /* It is possible we force to load a corrupted clustered index if srv_load_corrupted is set. Mark the table as corrupted in this case */ - table->corrupted = TRUE; + table->corrupted = true; } } @@ -2995,7 +2996,7 @@ func_exit: ut_ad(!table || ignore_err != DICT_ERR_IGNORE_NONE - || table->ibd_file_missing + || table->file_unreadable || !table->corrupted); if (table && table->fts) { diff --git a/storage/innobase/dict/dict0mem.cc b/storage/innobase/dict/dict0mem.cc index 7a6f09569a6..230706976dc 100644 --- a/storage/innobase/dict/dict0mem.cc +++ b/storage/innobase/dict/dict0mem.cc @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index 537a70c2069..54988b910d8 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2009, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -913,10 +913,15 @@ dict_stats_update_transient_for_index( index->stat_n_leaf_pages = size; - /* We don't handle the return value since it will be false - only when some thread is dropping the table and we don't - have to empty the statistics of the to be dropped index */ - btr_estimate_number_of_different_key_vals(index); + /* Do not continue if table decryption has failed or + table is already marked as corrupted. */ + if (index->is_readable()) { + /* We don't handle the return value since it + will be false only when some thread is + dropping the table and we don't have to empty + the statistics of the to be dropped index */ + btr_estimate_number_of_different_key_vals(index); + } } } @@ -967,8 +972,9 @@ dict_stats_update_transient( continue; } - /* Do not continue if table decryption has failed. */ - if (index->table->is_encrypted) { + /* Do not continue if table decryption has failed or + table is already marked as corrupted. */ + if (!index->is_readable()) { break; } @@ -2419,6 +2425,41 @@ dict_stats_save_index_stat( return(ret); } +/** Report an error if updating table statistics failed because +.ibd file is missing, table decryption failed or table is corrupted. +@param[in,out] table Table +@param[in] defragment true if statistics is for defragment +@retval DB_DECRYPTION_FAILED if decryption of the table failed +@retval DB_TABLESPACE_DELETED if .ibd file is missing +@retval DB_CORRUPTION if table is marked as corrupted */ +dberr_t +dict_stats_report_error(dict_table_t* table, bool defragment) +{ + dberr_t err; + + FilSpace space(table->space); + const char* df = defragment ? " defragment" : ""; + + if (!space()) { + ib::warn() << "Cannot save" << df << " statistics for table " + << table->name + << " because the .ibd file is missing. " + << TROUBLESHOOTING_MSG; + err = DB_TABLESPACE_DELETED; + } else { + ib::warn() << "Cannot save" << df << " statistics for table " + << table->name + << " because file " << space()->chain.start->name + << (table->corrupted + ? " is corrupted." + : " cannot be decrypted."); + err = table->corrupted ? DB_CORRUPTION : DB_DECRYPTION_FAILED; + } + + dict_stats_empty_table(table, defragment); + return err; +} + /** Save the table's statistics into the persistent statistics storage. @param[in] table_orig table whose stats to save @param[in] only_for_index if this is non-NULL, then stats for indexes @@ -2438,6 +2479,11 @@ dict_stats_save( char db_utf8[MAX_DB_UTF8_LEN]; char table_utf8[MAX_TABLE_UTF8_LEN]; + if (table_orig->is_readable()) { + } else { + return (dict_stats_report_error(table_orig)); + } + table = dict_stats_snapshot_create(table_orig); dict_fs2utf8(table->name.m_name, db_utf8, sizeof(db_utf8), @@ -3160,15 +3206,8 @@ dict_stats_update( { ut_ad(!mutex_own(&dict_sys->mutex)); - if (table->ibd_file_missing) { - - ib::warn() << "Cannot calculate statistics for table " - << table->name - << " because the .ibd file is missing. " - << TROUBLESHOOTING_MSG; - - dict_stats_empty_table(table, true); - return(DB_TABLESPACE_DELETED); + if (!table->is_readable()) { + return (dict_stats_report_error(table)); } else if (srv_force_recovery >= SRV_FORCE_NO_IBUF_MERGE) { /* If we have set a high innodb_force_recovery level, do not calculate statistics, as a badly corrupted index can diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index b031a77baa8..7f35e86bbc8 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -444,11 +444,12 @@ Parse a MLOG_FILE_WRITE_CRYPT_DATA log entry @param[in] block buffer block @return position on log buffer */ UNIV_INTERN -const byte* +byte* fil_parse_write_crypt_data( - const byte* ptr, + byte* ptr, const byte* end_ptr, - const buf_block_t* block) + const buf_block_t* block, + dberr_t* err) { /* check that redo log entry is complete */ uint entry_size = @@ -460,6 +461,8 @@ fil_parse_write_crypt_data( 4 + // size of key_id 1; // fil_encryption_t + *err = DB_SUCCESS; + if (ptr + entry_size > end_ptr) { return NULL; } @@ -506,6 +509,11 @@ fil_parse_write_crypt_data( fil_space_release(space); } + /* Check is used key found from encryption plugin */ + if (crypt_data->should_encrypt() && !crypt_data->is_key_found()) { + *err = DB_DECRYPTION_FAILED; + } + return ptr; } @@ -663,10 +671,12 @@ fil_space_encrypt( } bool corrupted = buf_page_is_corrupted(true, tmp_mem, page_size, space); + memcpy(tmp_mem+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, src+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 8); bool different = memcmp(src, tmp_mem, page_size.physical()); if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) { - fprintf(stderr, "ok %d corrupted %d corrupted1 %d err %d different %d\n", ok , corrupted, corrupted1, err, different); + fprintf(stderr, "ok %d corrupted %d corrupted1 %d err %d different %d\n", + ok , corrupted, corrupted1, err, different); fprintf(stderr, "src_frame\n"); buf_page_print(src_frame, page_size, BUF_PAGE_PRINT_NO_CRASH); fprintf(stderr, "encrypted_frame\n"); @@ -719,27 +729,6 @@ fil_space_decrypt( return false; } - if (crypt_data == NULL) { - if (!(space == 0 && offset == 0) && key_version != 0) { - /* FIL_PAGE_FILE_FLUSH_LSN field i.e. - FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION - should be only defined for the - first page in a system tablespace - data file (ibdata*, not *.ibd), if not - clear it. */ - - DBUG_LOG("crypt", - "Page " << page_id_t(space, offset) - << " carries key_version " << key_version - << " (should be undefined)"); - - memset(src_frame - + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 4); - } - - return false; - } - ut_a(crypt_data != NULL && crypt_data->is_encrypted()); /* read space & lsn */ @@ -789,9 +778,6 @@ fil_space_decrypt( memcpy(tmp_frame + page_size.physical() - FIL_PAGE_DATA_END, src_frame + page_size.physical() - FIL_PAGE_DATA_END, FIL_PAGE_DATA_END); - - // clear key-version & crypt-checksum from dst - memset(tmp_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); } srv_stats.pages_decrypted.inc(); @@ -1782,11 +1768,11 @@ fil_crypt_rotate_page( bool modified = false; int needs_scrubbing = BTR_SCRUB_SKIP_PAGE; lsn_t block_lsn = block->page.newest_modification; - uint kv = block->page.key_version; + byte* frame = buf_block_get_frame(block); + uint kv = mach_read_from_4(frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); /* check if tablespace is closing after reading page */ - if (space->is_stopping()) { - byte* frame = buf_block_get_frame(block); + if (!space->is_stopping()) { if (kv == 0 && fil_crypt_is_page_uninitialized(frame, page_size)) { @@ -1808,9 +1794,6 @@ fil_crypt_rotate_page( FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id, MLOG_4BYTES, &mtr); - /* update block */ - block->page.key_version = key_state->key_version; - /* statistics */ state->crypt_stat.pages_modified++; } else { @@ -1902,6 +1885,13 @@ fil_crypt_rotate_page( state->end_lsn = block_lsn; } } + } else { + /* If block read failed mtr memo and log should be empty. */ + ut_ad(!mtr.has_modifications()); + ut_ad(!mtr.is_dirty()); + ut_ad(mtr.get_memo()->size() == 0); + ut_ad(mtr.get_log()->size() == 0); + mtr.commit(); } if (sleeptime_ms) { @@ -2527,29 +2517,29 @@ fil_space_verify_crypt_checksum( return (true); } - /* Compressed pages use different checksum method. We first store - the post encryption checksum on checksum location and after function - restore the original. */ - if (page_size.is_compressed()) { - ib_uint32_t old = static_cast<ib_uint32_t>(mach_read_from_4( - page + FIL_PAGE_SPACE_OR_CHKSUM)); - - mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); - - bool valid = page_zip_verify_checksum(page, - page_size.physical()); + uint32 cchecksum1, cchecksum2; - mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, old); - - return (valid); + /* Calculate checksums */ + if (page_size.is_compressed()) { + cchecksum1 = page_zip_calc_checksum( + page, page_size.physical(), + SRV_CHECKSUM_ALGORITHM_CRC32); + + cchecksum2 = (cchecksum1 == checksum) + ? 0 + : page_zip_calc_checksum( + page, page_size.physical(), + SRV_CHECKSUM_ALGORITHM_INNODB); + } else { + cchecksum1 = buf_calc_page_crc32(page); + cchecksum2 = (cchecksum1 == checksum) + ? 0 + : buf_calc_page_new_checksum(page); } /* If stored checksum matches one of the calculated checksums page is not corrupted. */ - ib_uint32_t cchecksum1 = buf_calc_page_crc32(page); - ib_uint32_t cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum( - page); bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2 || checksum == BUF_NO_CHECKSUM_MAGIC); @@ -2577,17 +2567,24 @@ fil_space_verify_crypt_checksum( when FIL_PAGE_FILE_FLUSH_LSN does not contain 0. */ - ulint checksum1 = mach_read_from_4( - page + FIL_PAGE_SPACE_OR_CHKSUM); + uint32_t checksum1 = mach_read_from_4(page + FIL_PAGE_SPACE_OR_CHKSUM); + uint32_t checksum2; - ulint checksum2 = mach_read_from_4( - page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); + bool valid; - bool valid = buf_page_is_checksum_valid_crc32( - page, checksum1, checksum2, false - /* FIXME: also try the original crc32 that was - buggy on big-endian architectures? */) - || buf_page_is_checksum_valid_innodb(page, checksum1, checksum2); + if (page_size.is_compressed()) { + valid = checksum1 == cchecksum1; + checksum2 = checksum1; + } else { + checksum2 = mach_read_from_4( + page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); + valid = buf_page_is_checksum_valid_crc32( + page, checksum1, checksum2, false + /* FIXME: also try the original crc32 that was + buggy on big-endian architectures? */) + || buf_page_is_checksum_valid_innodb( + page, checksum1, checksum2); + } if (encrypted && valid) { /* If page is encrypted and traditional checksums match, @@ -2597,7 +2594,7 @@ fil_space_verify_crypt_checksum( fprintf(log_file ? log_file : stderr, "Page " ULINTPF ":" ULINTPF " may be corrupted." " Post encryption checksum %u" - " stored [" ULINTPF ":" ULINTPF "] key_version %u\n", + " stored [%u:%u] key_version %u\n", space, offset, checksum, checksum1, checksum2, key_version); #else /* UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 7a169c0bf2c..93279667466 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -306,7 +306,9 @@ fil_write( } /*******************************************************************//** -Returns the table space by a given id, NULL if not found. */ +Returns the table space by a given id, NULL if not found. +It is unsafe to dereference the returned pointer. It is fine to check +for NULL. */ fil_space_t* fil_space_get_by_id( /*================*/ @@ -2276,7 +2278,7 @@ for concurrency control. @param[in] silent whether to silently ignore missing tablespaces @return the tablespace @retval NULL if missing or being deleted or truncated */ -inline +UNIV_INTERN fil_space_t* fil_space_acquire_low(ulint id, bool silent) { @@ -2302,30 +2304,6 @@ fil_space_acquire_low(ulint id, bool silent) return(space); } -/** Acquire a tablespace when it could be dropped concurrently. -Used by background threads that do not necessarily hold proper locks -for concurrency control. -@param[in] id tablespace ID -@return the tablespace -@retval NULL if missing or being deleted or truncated */ -fil_space_t* -fil_space_acquire(ulint id) -{ - return(fil_space_acquire_low(id, false)); -} - -/** Acquire a tablespace that may not exist. -Used by background threads that do not necessarily hold proper locks -for concurrency control. -@param[in] id tablespace ID -@return the tablespace -@retval NULL if missing or being deleted */ -fil_space_t* -fil_space_acquire_silent(ulint id) -{ - return(fil_space_acquire_low(id, true)); -} - /** Release a tablespace acquired with fil_space_acquire(). @param[in,out] space tablespace to release */ void @@ -3102,7 +3080,7 @@ checked @return whether the table is accessible */ bool fil_table_accessible(const dict_table_t* table) { - if (UNIV_UNLIKELY(table->ibd_file_missing || table->corrupted)) { + if (UNIV_UNLIKELY(!table->is_readable() || table->corrupted)) { return(false); } @@ -4921,7 +4899,7 @@ fil_space_for_table_exists_in_mem( " deleted or moved .ibd files?"; } error_exit: - ib::warn() << TROUBLESHOOT_DATADICT_MSG; + ib::info() << TROUBLESHOOT_DATADICT_MSG; valid = false; goto func_exit; } @@ -5509,6 +5487,8 @@ fil_aio_wait( mutex_enter(&fil_system->mutex); fil_node_complete_io(node, type); + const fil_type_t purpose = node->space->purpose; + const ulint space_id = node->space->id; mutex_exit(&fil_system->mutex); @@ -5520,7 +5500,11 @@ fil_aio_wait( deadlocks in the i/o system. We keep tablespace 0 data files always open, and use a special i/o thread to serve insert buffer requests. */ - switch (node->space->purpose) { + switch (purpose) { + case FIL_TYPE_LOG: + srv_set_io_thread_op_info(segment, "complete io for log"); + log_io_complete(static_cast<log_group_t*>(message)); + return; case FIL_TYPE_TABLESPACE: case FIL_TYPE_TEMPORARY: case FIL_TYPE_IMPORT: @@ -5528,13 +5512,32 @@ fil_aio_wait( /* async single page writes from the dblwr buffer don't have access to the page */ - if (message != NULL) { - buf_page_io_complete(static_cast<buf_page_t*>(message)); + buf_page_t* bpage = static_cast<buf_page_t*>(message); + if (!bpage) { + return; + } + + ulint offset = bpage->id.page_no(); + dberr_t err = buf_page_io_complete(bpage); + if (err == DB_SUCCESS) { + return; + } + + ut_ad(type.is_read()); + if (recv_recovery_is_on() && !srv_force_recovery) { + recv_sys->found_corrupt_fs = true; + } + + if (fil_space_t* space = fil_space_acquire_for_io(space_id)) { + if (space == node->space) { + ib::error() << "Failed to read file '" + << node->name + << "' at offset " << offset + << ": " << ut_strerr(err); + } + + fil_space_release_for_io(space); } - return; - case FIL_TYPE_LOG: - srv_set_io_thread_op_info(segment, "complete io for log"); - log_io_complete(static_cast<log_group_t*>(message)); return; } @@ -6920,8 +6923,8 @@ fil_space_keyrotate_next( or dropped or truncated. Note that rotation_list contains only space->purpose == FIL_TYPE_TABLESPACE. */ while (space != NULL - && (UT_LIST_GET_LEN(space->chain) == 0 - || space->is_stopping())) { + && (UT_LIST_GET_LEN(space->chain) == 0 + || space->is_stopping())) { old = space; space = UT_LIST_GET_NEXT(rotation_list, space); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index c486b0d4c4d..75d34537deb 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -6601,6 +6601,7 @@ ha_innobase::open( /* Mark this table as corrupted, so the drop table or force recovery can still use it, but not others. */ + ib_table->file_unreadable = true; ib_table->corrupted = true; dict_table_close(ib_table, FALSE, FALSE); ib_table = NULL; @@ -6629,7 +6630,7 @@ ha_innobase::open( ib_table->thd = (void*)thd; /* No point to init any statistics if tablespace is still encrypted. */ - if (!ib_table->is_encrypted) { + if (ib_table->is_readable()) { dict_stats_init(ib_table); } else { ib_table->stat_initialized = 1; @@ -6637,7 +6638,9 @@ ha_innobase::open( MONITOR_INC(MONITOR_TABLE_OPEN); - bool no_tablespace; + bool no_tablespace = false; + bool encrypted = false; + FilSpace space; if (dict_table_is_discarded(ib_table)) { @@ -6652,21 +6655,28 @@ ha_innobase::open( no_tablespace = false; - } else if (ib_table->ibd_file_missing) { + } else if (!ib_table->is_readable()) { + space = fil_space_acquire_silent(ib_table->space); - ib_senderrf( - thd, IB_LOG_LEVEL_WARN, - ER_TABLESPACE_MISSING, norm_name); - - /* This means we have no idea what happened to the tablespace - file, best to play it safe. */ - - no_tablespace = true; - } else if (ib_table->is_encrypted) { - /* This means that tablespace was found but we could not - decrypt encrypted page. */ - no_tablespace = true; - ib_table->ibd_file_missing = true; + if (space()) { + if (space()->crypt_data && space()->crypt_data->is_encrypted()) { + /* This means that tablespace was found but we could not + decrypt encrypted page. */ + no_tablespace = true; + encrypted = true; + } else { + no_tablespace = true; + } + } else { + ib_senderrf( + thd, IB_LOG_LEVEL_WARN, + ER_TABLESPACE_MISSING, norm_name); + + /* This means we have no idea what happened to the tablespace + file, best to play it safe. */ + + no_tablespace = true; + } } else { no_tablespace = false; } @@ -6679,34 +6689,35 @@ ha_innobase::open( /* If table has no talespace but it has crypt data, check is tablespace made unaccessible because encryption service or used key_id is not available. */ - if (ib_table) { + if (encrypted) { bool warning_pushed = false; - fil_space_crypt_t* crypt_data = ib_table->crypt_data; - - if (crypt_data && crypt_data->should_encrypt()) { - if (!encryption_key_id_exists(crypt_data->key_id)) { - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - HA_ERR_DECRYPTION_FAILED, - "Table %s is encrypted but encryption service or" - " used key_id %u is not available. " - " Can't continue reading table.", - ib_table->name, crypt_data->key_id); - ret_err = HA_ERR_DECRYPTION_FAILED; - warning_pushed = true; - } + if (!encryption_key_id_exists(space()->crypt_data->key_id)) { + push_warning_printf( + thd, Sql_condition::WARN_LEVEL_WARN, + HA_ERR_DECRYPTION_FAILED, + "Table %s in file %s is encrypted but encryption service or" + " used key_id %u is not available. " + " Can't continue reading table.", + table_share->table_name.str, + space()->chain.start->name, + space()->crypt_data->key_id); + ret_err = HA_ERR_DECRYPTION_FAILED; + warning_pushed = true; } /* If table is marked as encrypted then we push warning if it has not been already done as used key_id might be found but it is incorrect. */ - if (ib_table->is_encrypted && !warning_pushed) { - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + if (!warning_pushed) { + push_warning_printf( + thd, Sql_condition::WARN_LEVEL_WARN, HA_ERR_DECRYPTION_FAILED, - "Table %s is encrypted but encryption service or" + "Table %s in file %s is encrypted but encryption service or" " used key_id is not available. " " Can't continue reading table.", - ib_table->name); + table_share->table_name.str, + space()->chain.start->name); ret_err = HA_ERR_DECRYPTION_FAILED; } } @@ -6865,7 +6876,7 @@ ha_innobase::open( if (m_prebuilt->table == NULL || dict_table_is_temporary(m_prebuilt->table) || m_prebuilt->table->persistent_autoinc - || m_prebuilt->table->ibd_file_missing) { + || m_prebuilt->table->file_unreadable) { } else if (const Field* ai = table->found_next_number_field) { initialize_auto_increment(m_prebuilt->table, ai); } @@ -10303,7 +10314,6 @@ ha_innobase::general_fetch( DBUG_ENTER("general_fetch"); const trx_t* trx = m_prebuilt->trx; - dberr_t ret; ut_ad(trx == thd_to_trx(m_user_thd)); @@ -10317,7 +10327,20 @@ ha_innobase::general_fetch( innobase_srv_conc_enter_innodb(m_prebuilt); - ret = row_search_mvcc( + if (m_prebuilt->table->is_readable()) { + } else if (m_prebuilt->table->corrupted) { + DBUG_RETURN(HA_ERR_CRASHED); + } else { + FilSpace space(m_prebuilt->table->space, true); + + DBUG_RETURN(space() + ? HA_ERR_DECRYPTION_FAILED + : HA_ERR_NO_SUCH_TABLE); + } + + innobase_srv_conc_enter_innodb(m_prebuilt); + + dberr_t ret = row_search_mvcc( buf, PAGE_CUR_UNSUPP, m_prebuilt, match_mode, direction); innobase_srv_conc_exit_innodb(m_prebuilt); @@ -13646,7 +13669,7 @@ ha_innobase::discard_or_import_tablespace( user may want to set the DISCARD flag in order to IMPORT a new tablespace. */ - if (dict_table->ibd_file_missing) { + if (!dict_table->is_readable()) { ib_senderrf( m_prebuilt->trx->mysql_thd, IB_LOG_LEVEL_WARN, ER_TABLESPACE_MISSING, @@ -13656,7 +13679,7 @@ ha_innobase::discard_or_import_tablespace( err = row_discard_tablespace_for_mysql( dict_table->name.m_name, m_prebuilt->trx); - } else if (!dict_table->ibd_file_missing) { + } else if (dict_table->is_readable()) { /* Commit the transaction in order to release the table lock. */ trx_commit_for_mysql(m_prebuilt->trx); @@ -15325,7 +15348,8 @@ ha_innobase::check( DBUG_RETURN(HA_ADMIN_CORRUPT); - } else if (m_prebuilt->table->ibd_file_missing) { + } else if (m_prebuilt->table->file_unreadable && + !fil_space_get(m_prebuilt->table->space)) { ib_senderrf( thd, IB_LOG_LEVEL_ERROR, diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 6f931d8d5a6..9194d6f182f 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -4863,7 +4863,7 @@ new_clustered_failed: clustered index of the old table, later. */ if (new_clustered || !ctx->online - || user_table->ibd_file_missing + || !user_table->is_readable() || dict_table_is_discarded(user_table)) { /* No need to allocate a modification log. */ ut_ad(!ctx->add_index[a]->online_log); @@ -5545,30 +5545,6 @@ ha_innobase::prepare_inplace_alter_table( indexed_table = m_prebuilt->table; - if (indexed_table->is_encrypted) { - String str; - const char* engine= table_type(); - push_warning_printf(m_user_thd, Sql_condition::WARN_LEVEL_WARN, - HA_ERR_DECRYPTION_FAILED, - "Table %s is encrypted but encryption service or" - " used key_id is not available. " - " Can't continue reading table.", - indexed_table->name); - get_error_message(HA_ERR_DECRYPTION_FAILED, &str); - my_error(ER_GET_ERRMSG, MYF(0), HA_ERR_DECRYPTION_FAILED, str.c_ptr(), engine); - - DBUG_RETURN(true); - } - - if (indexed_table->corrupted - || dict_table_get_first_index(indexed_table) == NULL - || dict_index_is_corrupted( - dict_table_get_first_index(indexed_table))) { - /* The clustered index is corrupted. */ - my_error(ER_CHECK_NO_SUCH_TABLE, MYF(0)); - DBUG_RETURN(true); - } - /* ALTER TABLE will not implicitly move a table from a single-table tablespace to the system tablespace when innodb_file_per_table=OFF. But it will implicitly move a table from the system tablespace to a @@ -5588,6 +5564,42 @@ ha_innobase::prepare_inplace_alter_table( } } + if (indexed_table->is_readable()) { + } else { + if (indexed_table->corrupted) { + /* Handled below */ + } else { + FilSpace space(indexed_table->space, true); + + if (space()) { + String str; + const char* engine= table_type(); + + push_warning_printf( + m_user_thd, + Sql_condition::WARN_LEVEL_WARN, + HA_ERR_DECRYPTION_FAILED, + "Table %s in file %s is encrypted but encryption service or" + " used key_id is not available. " + " Can't continue reading table.", + table_share->table_name.str, + space()->chain.start->name); + + my_error(ER_GET_ERRMSG, MYF(0), HA_ERR_DECRYPTION_FAILED, str.c_ptr(), engine); + DBUG_RETURN(true); + } + } + } + + if (indexed_table->corrupted + || dict_table_get_first_index(indexed_table) == NULL + || dict_index_is_corrupted( + dict_table_get_first_index(indexed_table))) { + /* The clustered index is corrupted. */ + my_error(ER_CHECK_NO_SUCH_TABLE, MYF(0)); + DBUG_RETURN(true); + } + if (ha_alter_info->handler_flags & Alter_inplace_info::CHANGE_CREATE_OPTION) { const char* invalid_opt = info.create_options_are_invalid(); @@ -6381,7 +6393,7 @@ ok_exit: ctx->m_stage = UT_NEW_NOKEY(ut_stage_alter_t(pk)); - if (m_prebuilt->table->ibd_file_missing + if (m_prebuilt->table->file_unreadable || dict_table_is_discarded(m_prebuilt->table)) { goto all_done; } @@ -7804,7 +7816,7 @@ commit_try_rebuild( /* The new table must inherit the flag from the "parent" table. */ if (dict_table_is_discarded(user_table)) { - rebuilt_table->ibd_file_missing = true; + rebuilt_table->file_unreadable = true; rebuilt_table->flags2 |= DICT_TF2_DISCARDED; } @@ -8311,20 +8323,20 @@ alter_stats_rebuild( } #ifndef DBUG_OFF - bool ibd_file_missing_orig = false; + bool file_unreadable_orig = false; #endif /* DBUG_OFF */ DBUG_EXECUTE_IF( "ib_rename_index_fail2", - ibd_file_missing_orig = table->ibd_file_missing; - table->ibd_file_missing = TRUE; + file_unreadable_orig = table->file_unreadable; + table->file_unreadable = true; ); dberr_t ret = dict_stats_update(table, DICT_STATS_RECALC_PERSISTENT); DBUG_EXECUTE_IF( "ib_rename_index_fail2", - table->ibd_file_missing = ibd_file_missing_orig; + table->file_unreadable = file_unreadable_orig; ); if (ret != DB_SUCCESS) { @@ -8455,6 +8467,19 @@ ha_innobase::commit_inplace_alter_table( = static_cast<ha_innobase_inplace_ctx*>(*pctx); DBUG_ASSERT(ctx->prebuilt->trx == m_prebuilt->trx); + /* If decryption failed for old table or new table + fail here. */ + if ((ctx->old_table->file_unreadable && + fil_space_get(ctx->old_table->space) != NULL)|| + (ctx->new_table->file_unreadable && + fil_space_get(ctx->new_table->space) != NULL)) { + String str; + const char* engine= table_type(); + get_error_message(HA_ERR_DECRYPTION_FAILED, &str); + my_error(ER_GET_ERRMSG, MYF(0), HA_ERR_DECRYPTION_FAILED, str.c_ptr(), engine); + DBUG_RETURN(true); + } + /* Exclusively lock the table, to ensure that no other transaction is holding locks on the table while we change the table definition. The MySQL meta-data lock diff --git a/storage/innobase/include/btr0btr.ic b/storage/innobase/include/btr0btr.ic index 308fcfe9b03..bd4f2a40267 100644 --- a/storage/innobase/include/btr0btr.ic +++ b/storage/innobase/include/btr0btr.ic @@ -63,7 +63,7 @@ btr_block_get_func( if (err == DB_DECRYPTION_FAILED) { if (index && index->table) { - index->table->is_encrypted = true; + index->table->file_unreadable = true; } } diff --git a/storage/innobase/include/btr0pcur.h b/storage/innobase/include/btr0pcur.h index e00af130479..fab934ca0ee 100644 --- a/storage/innobase/include/btr0pcur.h +++ b/storage/innobase/include/btr0pcur.h @@ -99,7 +99,7 @@ btr_pcur_free( Initializes and opens a persistent cursor to an index tree. It should be closed with btr_pcur_close. */ UNIV_INLINE -void +dberr_t btr_pcur_open_low( /*==============*/ dict_index_t* index, /*!< in: index */ diff --git a/storage/innobase/include/btr0pcur.ic b/storage/innobase/include/btr0pcur.ic index 425593631d3..4490942a2bb 100644 --- a/storage/innobase/include/btr0pcur.ic +++ b/storage/innobase/include/btr0pcur.ic @@ -418,7 +418,7 @@ btr_pcur_free( Initializes and opens a persistent cursor to an index tree. It should be closed with btr_pcur_close. */ UNIV_INLINE -void +dberr_t btr_pcur_open_low( /*==============*/ dict_index_t* index, /*!< in: index */ @@ -471,6 +471,8 @@ btr_pcur_open_low( cursor->pos_state = BTR_PCUR_IS_POSITIONED; cursor->trx_if_known = NULL; + + return(err); } /**************************************************************//** diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 7dee52d8f74..ea5cd5a3f4c 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1270,16 +1270,21 @@ buf_page_init_for_read( const page_size_t& page_size, ibool unzip); -/********************************************************************//** -Completes an asynchronous read or write request of a file page to or from -the buffer pool. -@return true if successful */ -bool -buf_page_io_complete( -/*=================*/ - buf_page_t* bpage, /*!< in: pointer to the block in question */ - bool evict = false);/*!< in: whether or not to evict - the page from LRU list. */ +/** Complete a read or write request of a file page to or from the buffer pool. +@param[in,out] bpage Page to complete +@param[in] evict whether or not to evict the page + from LRU list. +@return whether the operation succeeded +@retval DB_SUCCESS always when writing, or if a read page was OK +@retval DB_PAGE_CORRUPTED if the checksum fails on a page read +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but + after decryption normal page checksum does + not match */ +UNIV_INTERN +dberr_t +buf_page_io_complete(buf_page_t* bpage, bool evict = false) + MY_ATTRIBUTE((nonnull)); + /********************************************************************//** Calculates the index of a buffer pool to the buf_pool[] array. @return the position of the buffer pool in buf_pool[] */ @@ -1586,7 +1591,6 @@ public: if written again we check is TRIM operation needed. */ - unsigned key_version; /*!< key version for this block */ bool encrypted; /*!< page is still encrypted */ ulint real_size; /*!< Real size of the page diff --git a/storage/innobase/include/buf0checksum.h b/storage/innobase/include/buf0checksum.h index 9405251dc74..20955a5b2e6 100644 --- a/storage/innobase/include/buf0checksum.h +++ b/storage/innobase/include/buf0checksum.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -30,7 +31,7 @@ Created Aug 11, 2011 Vasil Dimov #include "buf0types.h" -/** Calculates the CRC32 checksum of a page. The value is stored to the page +/** Calculate the CRC32 checksum of a page. The value is stored to the page when it is written to a file and also checked for a match when reading from the file. When reading we allow both normal CRC32 and CRC-legacy-big-endian variants. Note that we must be careful to calculate the same value on 32-bit @@ -44,37 +45,30 @@ buf_calc_page_crc32( const byte* page, bool use_legacy_big_endian = false); -/********************************************************************//** -Calculates a page checksum which is stored to the page when it is written +/** Calculate a checksum which is stored to the page when it is written to a file. Note that we must be careful to calculate the same value on 32-bit and 64-bit architectures. +@param[in] page file page (srv_page_size bytes) @return checksum */ -ulint -buf_calc_page_new_checksum( -/*=======================*/ - const byte* page); /*!< in: buffer page */ - -/********************************************************************//** -In versions < 4.0.14 and < 4.1.1 there was a bug that the checksum only -looked at the first few bytes of the page. This calculates that old -checksum. +uint32_t +buf_calc_page_new_checksum(const byte* page); + +/** In MySQL before 4.0.14 or 4.1.1 there was an InnoDB bug that +the checksum only looked at the first few bytes of the page. +This calculates that old checksum. NOTE: we must first store the new formula checksum to FIL_PAGE_SPACE_OR_CHKSUM before calculating and storing this old checksum because this takes that field as an input! +@param[in] page file page (srv_page_size bytes) @return checksum */ -ulint -buf_calc_page_old_checksum( -/*=======================*/ - const byte* page); /*!< in: buffer page */ - +uint32_t +buf_calc_page_old_checksum(const byte* page); -/********************************************************************//** -Return a printable string describing the checksum algorithm. +/** Return a printable string describing the checksum algorithm. +@param[in] algo algorithm @return algorithm name */ const char* -buf_checksum_algorithm_name( -/*========================*/ - srv_checksum_algorithm_t algo); /*!< in: algorithm */ +buf_checksum_algorithm_name(srv_checksum_algorithm_t algo); extern ulong srv_checksum_algorithm; extern bool legacy_big_endian_checksum; diff --git a/storage/innobase/include/buf0rea.h b/storage/innobase/include/buf0rea.h index 9c97a5147c1..5c766b9412a 100644 --- a/storage/innobase/include/buf0rea.h +++ b/storage/innobase/include/buf0rea.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -37,12 +37,15 @@ an exclusive lock on the buffer frame. The flag is cleared and the x-lock released by the i/o-handler thread. @param[in] page_id page id @param[in] page_size page size -@return TRUE if page has been read in, FALSE in case of failure */ -ibool +@retval DB_SUCCESS if the page was read and is not corrupted, +@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted, +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but +after decryption normal page checksum does not match. +@retval DB_TABLESPACE_DELETED if tablespace .ibd file is missing */ +dberr_t buf_read_page( const page_id_t& page_id, - const page_size_t& page_size, - buf_page_t** bpage); + const page_size_t& page_size); /********************************************************************//** High-level function which reads a page asynchronously from a file to the @@ -51,9 +54,8 @@ an exclusive lock on the buffer frame. The flag is cleared and the x-lock released by the i/o-handler thread. @param[in] page_id page id @param[in] page_size page size -@param[in] sync true if synchronous aio is desired -@return TRUE if page has been read in, FALSE in case of failure */ -ibool +@param[in] sync true if synchronous aio is desired */ +void buf_read_page_background( const page_id_t& page_id, const page_size_t& page_size, diff --git a/storage/innobase/include/db0err.h b/storage/innobase/include/db0err.h index b0609991f61..ef6f8b39abb 100644 --- a/storage/innobase/include/db0err.h +++ b/storage/innobase/include/db0err.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -163,6 +163,8 @@ enum dberr_t { DB_IO_NO_PUNCH_HOLE, /*!< Punch hole not supported by file system. */ + DB_PAGE_CORRUPTED, /* Page read from tablespace is + corrupted. */ /* The following are partial failure codes */ DB_FAIL = 1000, DB_OVERFLOW, diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index 18578388723..f80047f29a9 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -1889,6 +1889,14 @@ dict_set_corrupted_by_space( /*========================*/ ulint space_id); /*!< in: space ID */ +/** Flag a table with specified space_id encrypted in the data dictionary +cache +@param[in] space_id Tablespace id */ +UNIV_INTERN +void +dict_set_encrypted_by_space( + ulint space_id); + /** Sets merge_threshold in the SYS_INDEXES @param[in,out] index index @param[in] merge_threshold value to set */ diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index 8849be1206d..e9507302e66 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -1680,7 +1680,7 @@ dict_index_is_corrupted( ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return((index->type & DICT_CORRUPT) - || (index->table && index->table->corrupted)); + || (index->table && index->table->corrupted)); } /********************************************************************//** diff --git a/storage/innobase/include/dict0load.h b/storage/innobase/include/dict0load.h index fb443caf770..22fd27c2484 100644 --- a/storage/innobase/include/dict0load.h +++ b/storage/innobase/include/dict0load.h @@ -119,7 +119,7 @@ a foreign key references columns in this table. @param[in] ignore_err Error to be ignored when loading table and its index definition @return table, NULL if does not exist; if the table is stored in an -.ibd file, but the file does not exist, then we set the ibd_file_missing +.ibd file, but the file does not exist, then we set the file_unreadable flag in the table object we return. */ dict_table_t* dict_load_table( diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index 0630137bb4f..f94d5f2b1ca 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -1012,6 +1012,13 @@ struct dict_index_t{ ut_ad(committed || !(type & DICT_CLUSTERED)); uncommitted = !committed; } + + /** @return whether this index is readable + @retval true normally + @retval false if this is a single-table tablespace + and the .ibd file is missing, or a + page cannot be read or decrypted */ + inline bool is_readable() const; }; /** The status of online index creation */ @@ -1357,6 +1364,16 @@ struct dict_table_t { /** Release the table handle. */ inline void release(); + /** @return whether this table is readable + @retval true normally + @retval false if this is a single-table tablespace + and the .ibd file is missing, or a + page cannot be read or decrypted */ + bool is_readable() const + { + return(UNIV_LIKELY(!file_unreadable)); + } + /** Id of the table. */ table_id_t id; @@ -1403,10 +1420,9 @@ struct dict_table_t { Use DICT_TF2_FLAG_IS_SET() to parse this flag. */ unsigned flags2:DICT_TF2_BITS; - /** TRUE if this is in a single-table tablespace and the .ibd file is - missing. Then we must return in ha_innodb.cc an error if the user - tries to query such an orphaned table. */ - unsigned ibd_file_missing:1; + /*!< whether this is in a single-table tablespace and the .ibd + file is missing or page decryption failed and page is corrupted */ + unsigned file_unreadable:1; /** TRUE if the table object has been added to the dictionary cache. */ unsigned cached:1; @@ -1727,8 +1743,6 @@ public: /** Timestamp of the last modification of this table. */ time_t update_time; - bool is_encrypted; - #ifdef UNIV_DEBUG /** Value of 'magic_n'. */ #define DICT_TABLE_MAGIC_N 76333786 @@ -1741,6 +1755,11 @@ public: dict_vcol_templ_t* vc_templ; }; +inline bool dict_index_t::is_readable() const +{ + return(UNIV_LIKELY(!table->file_unreadable)); +} + /*******************************************************************//** Initialise the table lock list. */ void diff --git a/storage/innobase/include/dict0stats.h b/storage/innobase/include/dict0stats.h index f02b8eb8eed..752c197f8c3 100644 --- a/storage/innobase/include/dict0stats.h +++ b/storage/innobase/include/dict0stats.h @@ -252,6 +252,17 @@ dict_stats_save_index_stat( const char* stat_description, trx_t* trx); +/** Report an error if updating table statistics failed because +.ibd file is missing, table decryption failed or table is corrupted. +@param[in,out] table Table +@param[in] defragment true if statistics is for defragment +@retval DB_DECRYPTION_FAILED if decryption of the table failed +@retval DB_TABLESPACE_DELETED if .ibd file is missing +@retval DB_CORRUPTION if table is marked as corrupted */ +dberr_t +dict_stats_report_error(dict_table_t* table, bool defragment = false) + MY_ATTRIBUTE((nonnull, warn_unused_result)); + #include "dict0stats.ic" #ifdef UNIV_ENABLE_UNIT_TEST_DICT_STATS diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index ca65df21402..f8d0e8c4d27 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -116,8 +116,7 @@ struct fil_space_crypt_t : st_encryption_scheme min_key_version(new_min_key_version), page0_offset(0), encryption(new_encryption), - mutex(), - key_found(new_min_key_version), + key_found(0), rotate_state() { key_id = new_key_id; @@ -134,6 +133,8 @@ struct fil_space_crypt_t : st_encryption_scheme type = CRYPT_SCHEME_1; min_key_version = key_get_latest_version(); } + + key_found = min_key_version; } /** Destructor */ @@ -293,13 +294,15 @@ Parse a MLOG_FILE_WRITE_CRYPT_DATA log entry @param[in] ptr Log entry start @param[in] end_ptr Log entry end @param[in] block buffer block +@param[out] err DB_SUCCESS or DB_DECRYPTION_FAILED @return position on log buffer */ UNIV_INTERN -const byte* +byte* fil_parse_write_crypt_data( - const byte* ptr, + byte* ptr, const byte* end_ptr, - const buf_block_t* block) + const buf_block_t* block, + dberr_t* err) MY_ATTRIBUTE((warn_unused_result)); /** Encrypt a buffer. diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index cef5b578a4a..d6a42dafaae 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -733,21 +733,39 @@ MY_ATTRIBUTE((warn_unused_result)); Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID +@param[in] silent whether to silently ignore missing tablespaces @return the tablespace @retval NULL if missing or being deleted or truncated */ +UNIV_INTERN fil_space_t* -fil_space_acquire(ulint id) +fil_space_acquire_low(ulint id, bool silent) MY_ATTRIBUTE((warn_unused_result)); +/** Acquire a tablespace when it could be dropped concurrently. +Used by background threads that do not necessarily hold proper locks +for concurrency control. +@param[in] id tablespace ID +@return the tablespace +@retval NULL if missing or being deleted or truncated */ +inline +fil_space_t* +fil_space_acquire(ulint id) +{ + return (fil_space_acquire_low(id, false)); +} + /** Acquire a tablespace that may not exist. Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID @return the tablespace @retval NULL if missing or being deleted */ +inline fil_space_t* fil_space_acquire_silent(ulint id) - MY_ATTRIBUTE((warn_unused_result)); +{ + return (fil_space_acquire_low(id, true)); +} /** Release a tablespace acquired with fil_space_acquire(). @param[in,out] space tablespace to release */ @@ -803,9 +821,10 @@ public: /** Constructor: Look up the tablespace and increment the reference count if found. - @param[in] space_id tablespace ID */ - explicit FilSpace(ulint space_id) - : m_space(fil_space_acquire(space_id)) {} + @param[in] space_id tablespace ID + @param[in] silent whether not to display errors */ + explicit FilSpace(ulint space_id, bool silent = false) + : m_space(fil_space_acquire_low(space_id, silent)) {} /** Assignment operator: This assumes that fil_space_acquire() has already been done for the fil_space_t. The caller must @@ -1510,13 +1529,6 @@ fil_mtr_rename_log( /*******************************************************************//** Returns the table space by a given id, NULL if not found. */ fil_space_t* -fil_space_found_by_id( -/*==================*/ - ulint id); /*!< in: space id */ - -/*******************************************************************//** -Returns the table space by a given id, NULL if not found. */ -fil_space_t* fil_space_get_by_id( /*================*/ ulint id); /*!< in: space id */ diff --git a/storage/innobase/include/fil0fil.ic b/storage/innobase/include/fil0fil.ic index e3d4fd4d88b..9505cc0bd69 100644 --- a/storage/innobase/include/fil0fil.ic +++ b/storage/innobase/include/fil0fil.ic @@ -75,19 +75,6 @@ fil_get_page_type_name( } /****************************************************************//** -Get block size from fil node -@return block size*/ -UNIV_INLINE -ulint -fil_node_get_block_size( -/*====================*/ - fil_node_t* node) /*!< in: Node where to get block - size */ -{ - return (node->block_size); -} - -/****************************************************************//** Validate page type. @return true if valid, false if not */ UNIV_INLINE diff --git a/storage/innobase/include/fil0pagecompress.h b/storage/innobase/include/fil0pagecompress.h index cd754151d80..67ff7895b02 100644 --- a/storage/innobase/include/fil0pagecompress.h +++ b/storage/innobase/include/fil0pagecompress.h @@ -67,23 +67,4 @@ fil_decompress_page( /*!< in: true if only an error should be produced when decompression fails. By default this parameter is false. */ - -/****************************************************************//** -Get space id from fil node -@return space id*/ -UNIV_INTERN -ulint -fil_node_get_space_id( -/*==================*/ - fil_node_t* node); /*!< in: Node where to get space id*/ - -/****************************************************************//** -Get block size from fil node -@return block size*/ -UNIV_INLINE -ulint -fil_node_get_block_size( -/*====================*/ - fil_node_t* node); /*!< in: Node where to get block - size */ #endif diff --git a/storage/innobase/include/row0mysql.h b/storage/innobase/include/row0mysql.h index 0557b585ae1..d1f90560d8d 100644 --- a/storage/innobase/include/row0mysql.h +++ b/storage/innobase/include/row0mysql.h @@ -474,7 +474,7 @@ row_drop_table_for_mysql( /*********************************************************************//** Discards the tablespace of a table which stored in an .ibd file. Discarding means that this function deletes the .ibd file and assigns a new table id for -the table. Also the flag table->ibd_file_missing is set TRUE. +the table. Also the file_unreadable flag is set. @return error code or DB_SUCCESS */ dberr_t row_discard_tablespace_for_mysql( diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 0b46c92f00a..f3a00e7b5e6 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1497,7 +1497,12 @@ parse_log: } break; case MLOG_FILE_WRITE_CRYPT_DATA: - ptr = const_cast<byte*>(fil_parse_write_crypt_data(ptr, end_ptr, block)); + dberr_t err; + ptr = const_cast<byte*>(fil_parse_write_crypt_data(ptr, end_ptr, block, &err)); + + if (err != DB_SUCCESS) { + recv_sys->found_corrupt_log = TRUE; + } break; default: ptr = NULL; @@ -1993,6 +1998,11 @@ recv_apply_hashed_log_recs(bool last_batch) break; } + if (recv_sys->found_corrupt_log) { + mutex_exit(&recv_sys->mutex); + return; + } + mutex_exit(&recv_sys->mutex); os_thread_sleep(500000); } @@ -2076,6 +2086,10 @@ recv_apply_hashed_log_recs(bool last_batch) mutex_exit(&(recv_sys->mutex)); + if (recv_sys->found_corrupt_log) { + return; + } + os_thread_sleep(500000); mutex_enter(&(recv_sys->mutex)); diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index df7d26c63e8..e5c903935f1 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -2060,14 +2060,10 @@ page_copy_rec_list_end_to_created_page( /* Individual inserts are logged in a shorter form */ - mtr_log_t log_mode; - - if (dict_table_is_temporary(index->table) - || index->table->ibd_file_missing /* IMPORT TABLESPACE */) { - log_mode = mtr_get_log_mode(mtr); - } else { - log_mode = mtr_set_log_mode(mtr, MTR_LOG_SHORT_INSERTS); - } + const mtr_log_t log_mode = dict_table_is_temporary(index->table) + || !index->is_readable() /* IMPORT TABLESPACE */ + ? mtr_get_log_mode(mtr) + : mtr_set_log_mode(mtr, MTR_LOG_SHORT_INSERTS); prev_rec = page_get_infimum_rec(new_page); if (page_is_comp(new_page)) { diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index 01a55d0dc61..6cbdb78314d 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -1959,16 +1959,13 @@ PageConverter::validate( buf_block_t* block) UNIV_NOTHROW { buf_frame_t* page = get_frame(block); - ulint space_id = mach_read_from_4( - page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - fil_space_t* space = fil_space_found_by_id(space_id); /* Check that the page number corresponds to the offset in the file. Flag as corrupt if it doesn't. Disable the check for LSN in buf_page_is_corrupted() */ if (buf_page_is_corrupted( - false, page, get_page_size(), space) + false, page, get_page_size(), NULL) || (page_get_page_no(page) != offset / m_page_size.physical() && page_get_page_no(page) != 0)) { @@ -2101,7 +2098,7 @@ row_import_discard_changes( index->space = FIL_NULL; } - table->ibd_file_missing = TRUE; + table->file_unreadable = true; fil_close_tablespace(trx, table->space); } @@ -3357,7 +3354,7 @@ row_import_for_mysql( ut_a(table->space); ut_ad(prebuilt->trx); - ut_a(table->ibd_file_missing); + ut_a(table->file_unreadable); ibuf_delete_for_discarded_space(table->space); @@ -3691,7 +3688,7 @@ row_import_for_mysql( return(row_import_error(prebuilt, trx, err)); } - table->ibd_file_missing = false; + table->file_unreadable = false; table->flags2 &= ~DICT_TF2_DISCARDED; /* Set autoinc value read from .cfg file, if one was specified. diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index 5803bc226cd..dc5efb93e74 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -1656,7 +1656,7 @@ row_ins_check_foreign_constraint( } if (check_table == NULL - || check_table->ibd_file_missing + || check_table->file_unreadable || check_index == NULL) { if (!srv_read_only_mode && check_ref) { @@ -2613,8 +2613,14 @@ row_ins_clust_index_entry_low( /* Note that we use PAGE_CUR_LE as the search mode, because then the function will return in both low_match and up_match of the cursor sensible values */ - btr_pcur_open_low(index, 0, entry, PAGE_CUR_LE, mode, &pcur, + err = btr_pcur_open_low(index, 0, entry, PAGE_CUR_LE, mode, &pcur, __FILE__, __LINE__, auto_inc, &mtr); + if (err != DB_SUCCESS) { + index->table->file_unreadable = true; + mtr.commit(); + goto func_exit; + } + cursor = btr_pcur_get_btr_cur(&pcur); cursor->thr = thr; @@ -2950,7 +2956,7 @@ row_ins_sec_index_entry_low( " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } goto func_exit; } diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index aeff27c31eb..9891258e3cc 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -1996,7 +1996,8 @@ row_merge_read_clustered_index( page_cur_t* cur = btr_pcur_get_page_cur(&pcur); /* Do not continue if table pages are still encrypted */ - if (old_table->is_encrypted || new_table->is_encrypted) { + if (!old_table->is_readable() || + !new_table->is_readable()) { err = DB_DECRYPTION_FAILED; trx->error_key_num = 0; goto func_exit; @@ -4300,7 +4301,7 @@ row_merge_rename_tables_dict( renamed along with the table. */ if (err == DB_SUCCESS && dict_table_is_file_per_table(old_table) - && !old_table->ibd_file_missing) { + && fil_space_get(old_table->space) != NULL) { /* Make pathname to update SYS_DATAFILES. */ char* tmp_path = row_make_new_pathname(old_table, tmp_name); @@ -4779,13 +4780,15 @@ row_merge_build_indexes( pct_cost = COST_READ_CLUSTERED_INDEX * 100 / (total_static_cost + total_dynamic_cost); /* Do not continue if we can't encrypt table pages */ - if (old_table->is_encrypted || new_table->is_encrypted) { + if (!old_table->is_readable() || + !new_table->is_readable()) { error = DB_DECRYPTION_FAILED; ib_push_warning(trx->mysql_thd, DB_DECRYPTION_FAILED, "Table %s is encrypted but encryption service or" " used key_id is not available. " " Can't continue reading table.", - old_table->is_encrypted ? old_table->name : new_table->name); + !old_table->is_readable() ? old_table->name : + new_table->name); goto func_exit; } diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 20a8777b71e..3078dc56cf1 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -819,6 +819,7 @@ handle_new_error: break; case DB_CORRUPTION: + case DB_PAGE_CORRUPTED: ib::error() << "We detected index corruption in an InnoDB type" " table. You have to dump + drop + reimport the" " table or, in a case of widespread corruption," @@ -1399,6 +1400,55 @@ run_again: return(err); } +/** Determine is tablespace encrypted but decryption failed, is table corrupted +or is tablespace .ibd file missing. +@param[in] table Table +@param[in] trx Transaction +@param[in] push_warning true if we should push warning to user +@retval DB_DECRYPTION_FAILED table is encrypted but decryption failed +@retval DB_CORRUPTION table is corrupted +@retval DB_TABLESPACE_NOT_FOUND tablespace .ibd file not found */ +static +dberr_t +row_mysql_get_table_status( + const dict_table_t* table, + trx_t* trx, + bool push_warning = true) +{ + dberr_t err; + if (fil_space_t* space = fil_space_acquire_silent(table->space)) { + if (space->crypt_data && space->crypt_data->is_encrypted()) { + // maybe we cannot access the table due to failing + // to decrypt + if (push_warning) { + ib_push_warning(trx, DB_DECRYPTION_FAILED, + "Table %s in tablespace %lu encrypted." + "However key management plugin or used key_id is not found or" + " used encryption algorithm or method does not match.", + table->name, table->space); + } + + err = DB_DECRYPTION_FAILED; + } else { + if (push_warning) { + ib_push_warning(trx, DB_CORRUPTION, + "Table %s in tablespace %lu corrupted.", + table->name, table->space); + } + + err = DB_CORRUPTION; + } + + fil_space_release(space); + } else { + ib::error() << ".ibd file is missing for table " + << table->name; + err = DB_TABLESPACE_NOT_FOUND; + } + + return(err); +} + /** Does an insert for MySQL. @param[in] mysql_rec row in the MySQL format @param[in,out] prebuilt prebuilt struct in MySQL handle @@ -1432,19 +1482,8 @@ row_insert_for_mysql( return(DB_TABLESPACE_DELETED); - } else if (prebuilt->table->ibd_file_missing) { - - ib::error() << ".ibd file is missing for table " - << prebuilt->table->name; - - return(DB_TABLESPACE_NOT_FOUND); - } else if (prebuilt->table->is_encrypted) { - ib_push_warning(trx, DB_DECRYPTION_FAILED, - "Table %s in tablespace " ULINTPF " encrypted." - "However key management plugin or used key_id is not found or" - " used encryption algorithm or method does not match.", - prebuilt->table->name, prebuilt->table->space); - return(DB_DECRYPTION_FAILED); + } else if (!prebuilt->table->is_readable()) { + return(row_mysql_get_table_status(prebuilt->table, trx, true)); } else if (srv_force_recovery) { ib::error() << MODIFICATIONS_NOT_ALLOWED_MSG_FORCE_RECOVERY; @@ -1858,21 +1897,8 @@ row_update_for_mysql_using_upd_graph( ut_a(prebuilt->magic_n2 == ROW_PREBUILT_ALLOCATED); UT_NOT_USED(mysql_rec); - if (prebuilt->table->ibd_file_missing) { - ib::error() << "MySQL is trying to use a table handle but the" - " .ibd file for table " << prebuilt->table->name - << " does not exist. Have you deleted" - " the .ibd file from the database directory under" - " the MySQL datadir, or have you used DISCARD" - " TABLESPACE? " << TROUBLESHOOTING_MSG; - DBUG_RETURN(DB_ERROR); - } else if (prebuilt->table->is_encrypted) { - ib_push_warning(trx, DB_DECRYPTION_FAILED, - "Table %s in tablespace " ULINTPF " encrypted." - "However key management plugin or used key_id is not found or" - " used encryption algorithm or method does not match.", - prebuilt->table->name, prebuilt->table->space); - return (DB_TABLE_NOT_FOUND); + if (!table->is_readable()) { + return(row_mysql_get_table_status(table, trx, true)); } if(srv_force_recovery) { @@ -3259,7 +3285,7 @@ row_discard_tablespace( /* All persistent operations successful, update the data dictionary memory cache. */ - table->ibd_file_missing = TRUE; + table->file_unreadable = true; table->flags2 |= DICT_TF2_DISCARDED; @@ -3298,7 +3324,7 @@ row_discard_tablespace( /*********************************************************************//** Discards the tablespace of a table which stored in an .ibd file. Discarding means that this function renames the .ibd file and assigns a new table id for -the table. Also the flag table->ibd_file_missing is set to TRUE. +the table. Also the file_unreadable flag is set. @return error code or DB_SUCCESS */ dberr_t row_discard_tablespace_for_mysql( @@ -3315,8 +3341,6 @@ row_discard_tablespace_for_mysql( if (table == 0) { err = DB_TABLE_NOT_FOUND; - } else if (table->is_encrypted) { - err = DB_DECRYPTION_FAILED; } else if (dict_table_is_temporary(table)) { ib_senderrf(trx->mysql_thd, IB_LOG_LEVEL_ERROR, @@ -3324,7 +3348,7 @@ row_discard_tablespace_for_mysql( err = DB_ERROR; - } else if (table->space == srv_sys_space.space_id()) { + } else if (table->space == TRX_SYS_SPACE) { char table_name[MAX_FULL_NAME_LEN + 1]; innobase_format_name( @@ -3655,18 +3679,6 @@ row_drop_table_for_mysql( err = DB_TABLE_NOT_FOUND; goto funct_exit; } - /* If table is encrypted and table page encryption failed - return error. */ - if (table->is_encrypted) { - - if (table->can_be_evicted) { - dict_table_move_from_lru_to_non_lru(table); - } - - dict_table_close(table, TRUE, FALSE); - err = DB_DECRYPTION_FAILED; - goto funct_exit; - } /* This function is called recursively via fts_drop_tables(). */ if (!trx_is_started(trx)) { @@ -4065,13 +4077,13 @@ row_drop_table_for_mysql( switch (err) { ulint space_id; - bool ibd_file_missing; + bool file_unreadable; bool is_discarded; ulint table_flags; case DB_SUCCESS: space_id = table->space; - ibd_file_missing = table->ibd_file_missing; + file_unreadable = table->file_unreadable; is_discarded = dict_table_is_discarded(table); table_flags = table->flags; ut_ad(!dict_table_is_temporary(table)); @@ -4102,8 +4114,7 @@ row_drop_table_for_mysql( /* Do not attempt to drop known-to-be-missing tablespaces, nor the system tablespace. */ - if (is_discarded || ibd_file_missing - || is_system_tablespace(space_id)) { + if (is_discarded || is_system_tablespace(space_id)) { break; } @@ -4336,7 +4347,8 @@ loop: << table->name << ".frm' was lost."; } - if (table->ibd_file_missing) { + if (!table->is_readable() + && !fil_space_get(table->space)) { ib::warn() << "Missing .ibd file for table " << table->name << "."; } @@ -4592,7 +4604,8 @@ row_rename_table_for_mysql( err = DB_TABLE_NOT_FOUND; goto funct_exit; - } else if (table->ibd_file_missing + } else if (!table->is_readable() + && fil_space_get(table->space) == NULL && !dict_table_is_discarded(table)) { err = DB_TABLE_NOT_FOUND; @@ -4658,8 +4671,7 @@ row_rename_table_for_mysql( /* SYS_TABLESPACES and SYS_DATAFILES need to be updated if the table is in a single-table tablespace. */ if (err == DB_SUCCESS - && dict_table_is_file_per_table(table) - && !table->ibd_file_missing) { + && dict_table_is_file_per_table(table)) { /* Make a new pathname to update SYS_DATAFILES. */ char* new_path = row_make_new_pathname(table, new_name); diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc index 492d864ec96..e49fd7f0f8c 100644 --- a/storage/innobase/row/row0purge.cc +++ b/storage/innobase/row/row0purge.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 229bd567c48..c4a87bde2cb 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -4191,13 +4191,10 @@ row_search_mvcc( DBUG_RETURN(DB_TABLESPACE_DELETED); - } else if (prebuilt->table->ibd_file_missing) { - - DBUG_RETURN(DB_TABLESPACE_NOT_FOUND); - - } else if (prebuilt->table->is_encrypted) { - - return(DB_DECRYPTION_FAILED); + } else if (!prebuilt->table->is_readable()) { + DBUG_RETURN(fil_space_get(prebuilt->table->space) + ? DB_DECRYPTION_FAILED + : DB_TABLESPACE_NOT_FOUND); } else if (!prebuilt->index_usable) { DBUG_RETURN(DB_MISSING_HISTORY); @@ -4673,7 +4670,7 @@ wait_table_again: " used key_id is not available. " " Can't continue reading table.", prebuilt->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } rec = NULL; goto lock_wait_or_error; @@ -4695,7 +4692,7 @@ rec_loop: rec = btr_pcur_get_rec(pcur); - if (!rec) { + if (!index->table->is_readable()) { err = DB_DECRYPTION_FAILED; goto lock_wait_or_error; } @@ -4821,7 +4818,7 @@ wrong_offs: if (!rec_validate(rec, offsets) || !btr_index_rec_validate(rec, index, FALSE)) { - ib::info() << "Index corruption: rec offs " + ib::error() << "Index corruption: rec offs " << page_offset(rec) << " next offs " << next_offs << ", page no " << page_get_page_no(page_align(rec)) diff --git a/storage/innobase/row/row0trunc.cc b/storage/innobase/row/row0trunc.cc index 1fc30e714f4..5724fad801f 100644 --- a/storage/innobase/row/row0trunc.cc +++ b/storage/innobase/row/row0trunc.cc @@ -1681,10 +1681,13 @@ row_truncate_sanity_checks( return(DB_TABLESPACE_DELETED); - } else if (table->ibd_file_missing) { - - return(DB_TABLESPACE_NOT_FOUND); + } else if (!table->is_readable()) { + if (fil_space_get(table->space) == NULL) { + return(DB_TABLESPACE_NOT_FOUND); + } else { + return(DB_DECRYPTION_FAILED); + } } else if (dict_table_is_corrupted(table)) { return(DB_TABLE_CORRUPT); diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 6d153ff1bc6..6be9c34ac5a 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -2210,6 +2210,11 @@ files_checked: recv_group_scan_log_recs(). */ recv_apply_hashed_log_recs(true); + + if (recv_sys->found_corrupt_log) { + return (DB_CORRUPTION); + } + DBUG_PRINT("ib_log", ("apply completed")); if (recv_needed_recovery) { diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index a6cfcf0910b..c8e286b3533 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -773,7 +773,7 @@ trx_resurrect_table_locks( i != tables.end(); i++) { if (dict_table_t* table = dict_table_open_on_id( *i, FALSE, DICT_TABLE_OP_LOAD_TABLESPACE)) { - if (table->ibd_file_missing + if (table->file_unreadable || dict_table_is_temporary(table)) { mutex_enter(&dict_sys->mutex); dict_table_close(table, TRUE, FALSE); diff --git a/storage/innobase/ut/ut0ut.cc b/storage/innobase/ut/ut0ut.cc index 2cae865cff2..74a5123790c 100644 --- a/storage/innobase/ut/ut0ut.cc +++ b/storage/innobase/ut/ut0ut.cc @@ -757,6 +757,8 @@ ut_strerr( "of stored column"); case DB_IO_NO_PUNCH_HOLE: return ("File system does not support punch hole (trim) operation."); + case DB_PAGE_CORRUPTED: + return("Page read from tablespace is corrupted."); /* do not add default: in order to produce a warning if new code is added to the enum but not added here */ diff --git a/storage/xtradb/api/api0api.cc b/storage/xtradb/api/api0api.cc index a6dfad40150..2a46dd4b4c1 100644 --- a/storage/xtradb/api/api0api.cc +++ b/storage/xtradb/api/api0api.cc @@ -247,7 +247,7 @@ ib_open_table_by_id( table = dict_table_open_on_id(table_id, TRUE, DICT_TABLE_OP_NORMAL); - if (table != NULL && table->ibd_file_missing) { + if (table != NULL && table->file_unreadable) { table = NULL; } @@ -272,7 +272,7 @@ ib_open_table_by_name( table = dict_table_open_on_name(name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); - if (table != NULL && table->ibd_file_missing) { + if (table != NULL && table->file_unreadable) { table = NULL; } @@ -292,7 +292,7 @@ ib_lookup_table_by_name( table = dict_table_get_low(name); - if (table != NULL && table->ibd_file_missing) { + if (table != NULL && table->file_unreadable) { table = NULL; } diff --git a/storage/xtradb/btr/btr0btr.cc b/storage/xtradb/btr/btr0btr.cc index 48411b6ff6a..c94b539c2c7 100644 --- a/storage/xtradb/btr/btr0btr.cc +++ b/storage/xtradb/btr/btr0btr.cc @@ -2,7 +2,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2014, 2016, MariaDB Corporation +Copyright (c) 2014, 2017, MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -744,8 +744,7 @@ btr_root_block_get( if (!block) { if (index && index->table) { - index->table->is_encrypted = TRUE; - index->table->corrupted = FALSE; + index->table->file_unreadable = true; ib_push_warning(index->table->thd, DB_DECRYPTION_FAILED, "Table %s in tablespace %lu is encrypted but encryption service or" @@ -760,6 +759,7 @@ btr_root_block_get( SRV_CORRUPT_TABLE_CHECK(block, return(0);); btr_assert_not_corrupted(block, index); + #ifdef UNIV_BTR_DEBUG if (!dict_index_is_ibuf(index)) { const page_t* root = buf_block_get_frame(block); @@ -5211,7 +5211,7 @@ btr_validate_index( page_t* root = btr_root_get(index, &mtr); - if (root == NULL && index->table->is_encrypted) { + if (root == NULL && index->table->file_unreadable) { err = DB_DECRYPTION_FAILED; mtr_commit(&mtr); return err; diff --git a/storage/xtradb/btr/btr0cur.cc b/storage/xtradb/btr/btr0cur.cc index 78683f35028..1ea1ec0696b 100644 --- a/storage/xtradb/btr/btr0cur.cc +++ b/storage/xtradb/btr/btr0cur.cc @@ -3,7 +3,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2015, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -658,7 +658,10 @@ retry_page_get: space, zip_size, page_no, rw_latch, guess, buf_mode, file, line, mtr, &err); + /* Note that block==NULL signifies either an error or change + buffering. */ if (err != DB_SUCCESS) { + ut_ad(block == NULL); if (err == DB_DECRYPTION_FAILED) { ib_push_warning((void *)NULL, DB_DECRYPTION_FAILED, @@ -666,7 +669,7 @@ retry_page_get: " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } goto func_exit; @@ -944,7 +947,7 @@ btr_cur_open_at_index_side_func( ulint offsets_[REC_OFFS_NORMAL_SIZE]; ulint* offsets = offsets_; dberr_t err = DB_SUCCESS; - + rec_offs_init(offsets_); estimate = latch_mode & BTR_ESTIMATE; @@ -988,6 +991,9 @@ btr_cur_open_at_index_side_func( block = buf_page_get_gen(space, zip_size, page_no, RW_NO_LATCH, NULL, BUF_GET, file, line, mtr, &err); + + ut_ad((block != NULL) == (err == DB_SUCCESS)); + if (err != DB_SUCCESS) { if (err == DB_DECRYPTION_FAILED) { ib_push_warning((void *)NULL, @@ -996,7 +1002,7 @@ btr_cur_open_at_index_side_func( " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } goto exit_loop; @@ -1161,6 +1167,8 @@ btr_cur_open_at_rnd_pos_func( RW_NO_LATCH, NULL, BUF_GET, file, line, mtr, &err); + ut_ad((block != NULL) == (err == DB_SUCCESS)); + if (err != DB_SUCCESS) { if (err == DB_DECRYPTION_FAILED) { ib_push_warning((void *)NULL, @@ -1169,8 +1177,9 @@ btr_cur_open_at_rnd_pos_func( " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } + goto exit_loop; } @@ -3853,6 +3862,8 @@ btr_estimate_n_rows_in_range_on_level( NULL, BUF_GET_POSSIBLY_FREED, __FILE__, __LINE__, &mtr, &err); + ut_ad((block != NULL) == (err == DB_SUCCESS)); + if (err != DB_SUCCESS) { if (err == DB_DECRYPTION_FAILED) { ib_push_warning((void *)NULL, @@ -3861,14 +3872,13 @@ btr_estimate_n_rows_in_range_on_level( " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } mtr_commit(&mtr); goto inexact; } - page = buf_block_get_frame(block); /* It is possible that the tree has been reorganized in the @@ -4010,6 +4020,10 @@ btr_estimate_n_rows_in_range_low( mtr_commit(&mtr); + if (index->table->file_unreadable) { + return (0); + } + mtr_start_trx(&mtr, trx); #ifdef UNIV_DEBUG @@ -4360,6 +4374,11 @@ btr_estimate_number_of_different_key_vals( because otherwise our algorithm would give a wrong estimate for an index where there is just one key value. */ + if (index->table->file_unreadable) { + mtr_commit(&mtr); + goto exit_loop; + } + page = btr_cur_get_page(&cursor); SRV_CORRUPT_TABLE_CHECK(page, goto exit_loop;); diff --git a/storage/xtradb/btr/btr0defragment.cc b/storage/xtradb/btr/btr0defragment.cc index 3f4185341a6..64dc077d582 100644 --- a/storage/xtradb/btr/btr0defragment.cc +++ b/storage/xtradb/btr/btr0defragment.cc @@ -227,7 +227,7 @@ btr_defragment_add_index( page = buf_block_get_frame(block); } - if (page == NULL && index->table->is_encrypted) { + if (page == NULL && index->table->file_unreadable) { mtr_commit(&mtr); *err = DB_DECRYPTION_FAILED; return NULL; diff --git a/storage/xtradb/btr/btr0pcur.cc b/storage/xtradb/btr/btr0pcur.cc index dd6ef484fc3..0b970e1cf49 100644 --- a/storage/xtradb/btr/btr0pcur.cc +++ b/storage/xtradb/btr/btr0pcur.cc @@ -420,6 +420,11 @@ btr_pcur_move_to_next_page( cursor->old_stored = BTR_PCUR_OLD_NOT_STORED; page = btr_pcur_get_page(cursor); + + if (UNIV_UNLIKELY(!page)) { + return; + } + next_page_no = btr_page_get_next(page, mtr); space = buf_block_get_space(btr_pcur_get_block(cursor)); zip_size = buf_block_get_zip_size(btr_pcur_get_block(cursor)); @@ -429,6 +434,11 @@ btr_pcur_move_to_next_page( next_block = btr_block_get(space, zip_size, next_page_no, cursor->latch_mode, btr_pcur_get_btr_cur(cursor)->index, mtr); + + if (UNIV_UNLIKELY(!next_block)) { + return; + } + next_page = buf_block_get_frame(next_block); SRV_CORRUPT_TABLE_CHECK(next_page, diff --git a/storage/xtradb/btr/btr0scrub.cc b/storage/xtradb/btr/btr0scrub.cc index e9434c9f778..24c84ed301b 100644 --- a/storage/xtradb/btr/btr0scrub.cc +++ b/storage/xtradb/btr/btr0scrub.cc @@ -139,6 +139,7 @@ btr_scrub_lock_dict_func(ulint space_id, bool lock_to_close_table, } else { return false; } + os_thread_sleep(250000); time_t now = time(0); @@ -577,7 +578,7 @@ btr_scrub_table_needs_scrubbing( return false; } - if (table->corrupted) { + if (!table->is_readable()) { return false; } diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 78ebbb7902b..ebf6bb10caa 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -1181,7 +1181,6 @@ buf_block_init( block->page.buf_fix_count = 0; block->page.io_fix = BUF_IO_NONE; block->page.encrypted = false; - block->page.key_version = 0; block->page.real_size = 0; block->page.write_size = 0; block->modify_clock = 0; @@ -2354,7 +2353,17 @@ lookup: /* Page not in buf_pool: needs to be read from file */ ut_ad(!hash_lock); - buf_read_page(space, zip_size, offset, trx, NULL); + dberr_t err = buf_read_page(space, zip_size, offset, trx); + + if (err != DB_SUCCESS) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Reading compressed page " ULINTPF + ":" ULINTPF + " failed with error: %s.", + space, offset, ut_strerr(err)); + + goto err_exit; + } #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG ut_a(++buf_dbg_counter % 5771 || buf_validate()); @@ -2960,7 +2969,6 @@ loop: } if (block == NULL) { - buf_page_t* bpage=NULL; /* Page not in buf_pool: needs to be read from file */ @@ -2996,7 +3004,18 @@ loop: return(NULL); } - if (buf_read_page(space, zip_size, offset, trx, &bpage)) { + /* Call path is buf_read_page() -> buf_read_page_low() + (_fil_io()) -> buf_page_io_complete() -> + buf_decrypt_after_read() here fil_space_t* is used + and we decrypt -> buf_page_check_corrupt() where + page checksums are compared. Decryption/decompression + is handled lower level, error handling is handled on lower + level, here we need only to know is page really corrupted + or encrypted page with correct checksum. */ + + dberr_t local_err = buf_read_page(space, zip_size, offset, trx); + + if (local_err == DB_SUCCESS) { buf_read_ahead_random(space, zip_size, offset, ibuf_inside(mtr), trx); @@ -3004,89 +3023,47 @@ loop: } else if (retries < BUF_PAGE_READ_MAX_RETRIES) { ++retries; - bool corrupted = false; - - if (bpage) { - corrupted = buf_page_check_corrupt(bpage); - } - - /* Do not try again for encrypted pages */ - if (corrupted && bpage->encrypted) { - ib_mutex_t* pmutex = buf_page_get_mutex(bpage); - mutex_enter(&buf_pool->LRU_list_mutex); - mutex_enter(pmutex); - - ut_ad(buf_pool->n_pend_reads > 0); - os_atomic_decrement_ulint(&buf_pool->n_pend_reads, 1); - buf_page_set_io_fix(bpage, BUF_IO_NONE); - - if (!buf_LRU_free_page(bpage, true)) { - mutex_exit(&buf_pool->LRU_list_mutex); - } - - mutex_exit(pmutex); - rw_lock_x_unlock_gen(&((buf_block_t*) bpage)->lock, - BUF_IO_READ); - - if (err) { - *err = DB_DECRYPTION_FAILED; - } - - return (NULL); - } - DBUG_EXECUTE_IF( "innodb_page_corruption_retries", retries = BUF_PAGE_READ_MAX_RETRIES; ); } else { - bool corrupted = false; - - if (bpage) { - corrupted = buf_page_check_corrupt(bpage); + if (err) { + *err = local_err; } - if (corrupted && !bpage->encrypted) { - ib_logf(IB_LOG_LEVEL_ERROR, "Unable" - " to read tablespace %lu page no" - " %lu into the buffer pool after" - " %lu attempts\n" - "InnoDB: The most probable cause" - " of this error may be that the" - " table has been corrupted.\n" - "InnoDB: You can try to fix this" - " problem by using" - " innodb_force_recovery.\n" - "InnoDB: Please see reference manual" - " for more details.\n" - "InnoDB: Aborting...\n", - space, offset, - BUF_PAGE_READ_MAX_RETRIES); - - ut_error; - } else { - ib_mutex_t* pmutex = buf_page_get_mutex(bpage); - mutex_enter(&buf_pool->LRU_list_mutex); - mutex_enter(pmutex); - - ut_ad(buf_pool->n_pend_reads > 0); - os_atomic_decrement_ulint(&buf_pool->n_pend_reads, 1); - buf_page_set_io_fix(bpage, BUF_IO_NONE); - - if (!buf_LRU_free_page(bpage, true)) { - mutex_exit(&buf_pool->LRU_list_mutex); - } - - mutex_exit(pmutex); - rw_lock_x_unlock_gen(&((buf_block_t*) bpage)->lock, - BUF_IO_READ); - - if (err) { - *err = DB_DECRYPTION_FAILED; - } + /* Pages whose encryption key is unavailable or used + key, encryption algorithm or encryption method is + incorrect are marked as encrypted in + buf_page_check_corrupt(). Unencrypted page could be + corrupted in a way where the key_id field is + nonzero. There is no checksum on field + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION. */ + if (local_err == DB_DECRYPTION_FAILED) { + return (NULL); + } + /* Try to set table as corrupted instead of + asserting. */ + if (space > TRX_SYS_SPACE && + dict_set_corrupted_by_space(space)) { return (NULL); } + + ib_logf(IB_LOG_LEVEL_FATAL, "Unable" + " to read tablespace %lu page no" + " %lu into the buffer pool after" + " %lu attempts" + " The most probable cause" + " of this error may be that the" + " table has been corrupted." + " You can try to fix this" + " problem by using" + " innodb_force_recovery." + " Please see " REFMAN " for more" + " details. Aborting...", + space, offset, + BUF_PAGE_READ_MAX_RETRIES); } #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG @@ -3859,7 +3836,6 @@ buf_page_init_low( bpage->oldest_modification = 0; bpage->write_size = 0; bpage->encrypted = false; - bpage->key_version = 0; bpage->real_size = 0; HASH_INVALIDATE(bpage, hash); @@ -4502,61 +4478,55 @@ buf_page_monitor( /********************************************************************//** Mark a table with the specified space pointed by bpage->space corrupted. Also remove the bpage from LRU list. -@return TRUE if successful */ +@param[in,out] bpage Block */ static -ibool +void buf_mark_space_corrupt( -/*===================*/ - buf_page_t* bpage) /*!< in: pointer to the block in question */ + buf_page_t* bpage) { buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); const ibool uncompressed = (buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); ulint space = bpage->space; - ibool ret = TRUE; const ulint fold = buf_page_address_fold(bpage->space, bpage->offset); prio_rw_lock_t* hash_lock = buf_page_hash_lock_get(buf_pool, fold); + ib_mutex_t* block_mutex = buf_page_get_mutex(bpage); /* First unfix and release lock on the bpage */ ut_ad(!mutex_own(&buf_pool->LRU_list_mutex)); - if (!bpage->encrypted) { - mutex_enter(&buf_pool->LRU_list_mutex); - rw_lock_x_lock(hash_lock); - mutex_enter(buf_page_get_mutex(bpage)); - ut_ad(buf_page_get_io_fix(bpage) == BUF_IO_READ); - ut_ad(bpage->buf_fix_count == 0); + mutex_enter(&buf_pool->LRU_list_mutex); + rw_lock_x_lock(hash_lock); + mutex_enter(block_mutex); + ut_ad(buf_page_get_io_fix(bpage) == BUF_IO_READ); + ut_ad(bpage->buf_fix_count == 0); - /* Set BUF_IO_NONE before we remove the block from LRU list */ - buf_page_set_io_fix(bpage, BUF_IO_NONE); + /* Set BUF_IO_NONE before we remove the block from LRU list */ + buf_page_set_io_fix(bpage, BUF_IO_NONE); - if (uncompressed) { - rw_lock_x_unlock_gen( - &((buf_block_t*) bpage)->lock, - BUF_IO_READ); - } + if (uncompressed) { + rw_lock_x_unlock_gen( + &((buf_block_t*) bpage)->lock, + BUF_IO_READ); } - /* Find the table with specified space id, and mark it corrupted */ - if (dict_set_corrupted_by_space(space)) { - if (!bpage->encrypted) { - buf_LRU_free_one_page(bpage); - } + /* If block is not encrypted find the table with specified + space id, and mark it corrupted. Encrypted tables + are marked unusable later e.g. in ::open(). */ + if (!bpage->encrypted) { + dict_set_corrupted_by_space(space); } else { - if (!bpage->encrypted) { - mutex_exit(buf_page_get_mutex(bpage)); - } - ret = FALSE; + dict_set_encrypted_by_space(space); } - if(!bpage->encrypted) { - mutex_exit(&buf_pool->LRU_list_mutex); - ut_ad(buf_pool->n_pend_reads > 0); - os_atomic_decrement_ulint(&buf_pool->n_pend_reads, 1); - } + /* After this point bpage can't be referenced. This + function will release the hash_lock acquired above. */ + buf_LRU_free_one_page(bpage); - return(ret); + ut_ad(buf_pool->n_pend_reads > 0); + os_atomic_decrement_ulint(&buf_pool->n_pend_reads, 1); + mutex_exit(&buf_pool->LRU_list_mutex); } /********************************************************************//** @@ -4564,24 +4534,29 @@ Check if page is maybe compressed, encrypted or both when we encounter corrupted page. Note that we can't be 100% sure if page is corrupted or decrypt/decompress just failed. @param[in,out] bpage Page -@return true if page corrupted, false if not */ -UNIV_INTERN -bool -buf_page_check_corrupt( - buf_page_t* bpage) +@return DB_SUCCESS if page has been read and is not corrupted, +@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted, +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but +after decryption normal page checksum does not match. +@retval DB_TABLESPACE_DELETED if accessed tablespace is not found */ +static +dberr_t +buf_page_check_corrupt(buf_page_t* bpage) { ulint zip_size = buf_page_get_zip_size(bpage); byte* dst_frame = (zip_size) ? bpage->zip.data : ((buf_block_t*) bpage)->frame; - ulint space_id = bpage->space; - fil_space_t* space = fil_space_acquire_silent(space_id); + FilSpace space(bpage->space, true); bool still_encrypted = false; + dberr_t err = DB_SUCCESS; bool corrupted = false; - ulint page_type = mach_read_from_2(dst_frame + FIL_PAGE_TYPE); fil_space_crypt_t* crypt_data = NULL; - ut_ad(space); - crypt_data = space->crypt_data; + if (!space()) { + return(DB_TABLESPACE_DELETED); + } + + crypt_data = space()->crypt_data; /* In buf_decrypt_after_read we have either decrypted the page if page post encryption checksum matches and used key_id is found @@ -4593,62 +4568,65 @@ buf_page_check_corrupt( crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && !bpage->encrypted && fil_space_verify_crypt_checksum(dst_frame, zip_size, - space, bpage->offset)); + space(), bpage->offset)); if (!still_encrypted) { /* If traditional checksums match, we assume that page is not anymore encrypted. */ - corrupted = buf_page_is_corrupted(true, dst_frame, zip_size, space); + corrupted = buf_page_is_corrupted(true, dst_frame, zip_size, space()); if (!corrupted) { bpage->encrypted = false; + } else { + err = DB_PAGE_CORRUPTED; } } /* Pages that we think are unencrypted but do not match the checksum checks could be corrupted or encrypted or both. */ if (corrupted && !bpage->encrypted) { - ib_logf(IB_LOG_LEVEL_ERROR, - "%s: Block in space_id " ULINTPF " in file %s corrupted.", - page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED ? "Maybe corruption" : "Corruption", - space_id, (space && space->name) ? space->name : "NULL"); - ib_logf(IB_LOG_LEVEL_ERROR, - "Based on page type %s (" ULINTPF ")", - fil_get_page_type_name(page_type), page_type); + /* An error will be reported by + buf_page_io_complete(). */ } else if (still_encrypted || (bpage->encrypted && corrupted)) { bpage->encrypted = true; - corrupted = true; + err = DB_DECRYPTION_FAILED; ib_logf(IB_LOG_LEVEL_ERROR, - "Block in space_id " ULINTPF " in file %s encrypted.", - space_id, (space && space->name) ? space->name : "NULL"); - ib_logf(IB_LOG_LEVEL_ERROR, - "However key management plugin or used key_version %u is not found or" - " used encryption algorithm or method does not match.", - bpage->key_version); - if (space_id > TRX_SYS_SPACE) { - ib_logf(IB_LOG_LEVEL_ERROR, + "The page [page id: space=%u" + ", page number=%u]" + " in file %s cannot be decrypted.", + bpage->space, bpage->offset, + space()->name); + + ib_logf(IB_LOG_LEVEL_INFO, + "However key management plugin or used key_version " ULINTPF + " is not found or" + " used encryption algorithm or method does not match.", + mach_read_from_4(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)); + + if (bpage->space > TRX_SYS_SPACE) { + ib_logf(IB_LOG_LEVEL_INFO, "Marking tablespace as missing. You may drop this table or" " install correct key management plugin and key file."); } } - if (space) { - fil_space_release(space); - } - - return corrupted; + return (err); } /********************************************************************//** Completes an asynchronous read or write request of a file page to or from the buffer pool. -@return true if successful */ +@param[in,out] bpage Page to complete +@return DB_SUCCESS if page has been read and is not corrupted, +DB_PAGE_CORRUPTED if page based on checksum check is corrupted, +DB_DECRYPTION_FAILED if page post encryption checksum matches but +after decryption normal page checksum does not match. +in write only DB_SUCCESS is possible. */ UNIV_INTERN -bool +dberr_t buf_page_io_complete( -/*=================*/ - buf_page_t* bpage) /*!< in: pointer to the block in question */ + buf_page_t* bpage) { enum buf_io_fix io_type; buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); @@ -4656,8 +4634,8 @@ buf_page_io_complete( == BUF_BLOCK_FILE_PAGE); bool have_LRU_mutex = false; fil_space_t* space = NULL; - byte* frame = NULL; - bool corrupted = false; + byte* frame = NULL; + dberr_t err = DB_SUCCESS; ut_a(buf_page_in_file(bpage)); @@ -4671,8 +4649,9 @@ buf_page_io_complete( ut_ad(io_type == BUF_IO_READ || io_type == BUF_IO_WRITE); if (io_type == BUF_IO_READ) { - ulint read_page_no; - ulint read_space_id; + ulint read_page_no = 0; + ulint read_space_id = 0; + uint key_version = 0; buf_page_decrypt_after_read(bpage); @@ -4696,10 +4675,11 @@ buf_page_io_complete( "Page %u in tablespace %u zip_decompress failure.", bpage->offset, bpage->space); - corrupted = true; + err = DB_PAGE_CORRUPTED; goto database_corrupted; } + os_atomic_decrement_ulint(&buf_pool->n_pend_unzip, 1); } else { ut_a(uncompressed); @@ -4712,6 +4692,8 @@ buf_page_io_complete( read_page_no = mach_read_from_4(frame + FIL_PAGE_OFFSET); read_space_id = mach_read_from_4( frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + key_version = mach_read_from_4( + frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); if (bpage->space == TRX_SYS_SPACE && buf_dblwr_page_inside(bpage->offset)) { @@ -4732,12 +4714,11 @@ buf_page_io_complete( page may contain garbage in MySQL < 4.1.1, which only supported bpage->space == 0. */ - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: space id and page n:o" - " stored in the page\n" - "InnoDB: read in are " ULINTPF ":" ULINTPF "," - " should be %u:%u!\n", + ib_logf(IB_LOG_LEVEL_ERROR, + "Space id and page n:o" + " stored in the page" + " read in are " ULINTPF ":" ULINTPF "," + " should be %u:%u!", read_space_id, read_page_no, bpage->space, @@ -4746,81 +4727,68 @@ buf_page_io_complete( if (UNIV_LIKELY(!bpage->is_corrupt || !srv_pass_corrupt_table)) { - corrupted = buf_page_check_corrupt(bpage); - + err = buf_page_check_corrupt(bpage); } database_corrupted: - if (corrupted) { + if (err != DB_SUCCESS) { /* Not a real corruption if it was triggered by error injection */ - DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", - if (bpage->space > TRX_SYS_SPACE - && buf_mark_space_corrupt(bpage)) { + if (bpage->space > TRX_SYS_SPACE) { + buf_mark_space_corrupt(bpage); ib_logf(IB_LOG_LEVEL_INFO, "Simulated page corruption"); - return(true); + return(err); } + err = DB_SUCCESS; goto page_not_corrupt; ); - if (!bpage->encrypted) { + if (err == DB_PAGE_CORRUPTED) { fil_system_enter(); space = fil_space_get_by_id(bpage->space); - fil_system_exit(); + ib_logf(IB_LOG_LEVEL_ERROR, "Database page corruption on disk" - " or a failed"); - ib_logf(IB_LOG_LEVEL_ERROR, - "Space %u file %s read of page %u.", - bpage->space, - space->name ? space->name : "NULL", - bpage->offset); - ib_logf(IB_LOG_LEVEL_ERROR, - "You may have to recover" - " from a backup."); + " or a failed file read of tablespace %s" + " page [page id: space=%u" + ", page number=%u]" + ". You may have to recover from " + "a backup.", + space->name, + bpage->space, bpage->offset); + + fil_system_exit(); buf_page_print(frame, buf_page_get_zip_size(bpage), BUF_PAGE_PRINT_NO_CRASH); - ib_logf(IB_LOG_LEVEL_ERROR, - "It is also possible that your operating" - "system has corrupted its own file cache."); - ib_logf(IB_LOG_LEVEL_ERROR, - "and rebooting your computer removes the error."); - ib_logf(IB_LOG_LEVEL_ERROR, - "If the corrupt page is an index page you can also try to"); - ib_logf(IB_LOG_LEVEL_ERROR, - "fix the corruption by dumping, dropping, and reimporting"); - ib_logf(IB_LOG_LEVEL_ERROR, - "the corrupt table. You can use CHECK"); - ib_logf(IB_LOG_LEVEL_ERROR, - "TABLE to scan your table for corruption."); - ib_logf(IB_LOG_LEVEL_ERROR, - "See also " - REFMAN "forcing-innodb-recovery.html" - " about forcing recovery."); + ib_logf(IB_LOG_LEVEL_INFO, + "It is also possible that your" + " operating system has corrupted" + " its own file cache and rebooting" + " your computer removes the error." + " If the corrupt page is an index page." + " You can also try to fix the" + " corruption by dumping, dropping," + " and reimporting the corrupt table." + " You can use CHECK TABLE to scan" + " your table for corruption. " + "Please refer to " REFMAN "forcing-innodb-recovery.html" + " for information about forcing recovery."); } if (srv_pass_corrupt_table && bpage->space != 0 && bpage->space < SRV_LOG_SPACE_FIRST_ID) { - trx_t* trx; - fprintf(stderr, - "InnoDB: space %u will be treated as corrupt.\n", + ib_logf(IB_LOG_LEVEL_INFO, + "InnoDB: space %u will be treated as corrupt.", bpage->space); fil_space_set_corrupt(bpage->space); - trx = innobase_get_trx(); - - if (trx && trx->dict_operation_lock_mode == RW_X_LATCH) { - dict_table_set_corrupt_by_space(bpage->space, FALSE); - } else { - dict_table_set_corrupt_by_space(bpage->space, TRUE); - } - + dict_set_corrupted_by_space(bpage->space); bpage->is_corrupt = TRUE; } @@ -4828,31 +4796,12 @@ database_corrupted: /* If page space id is larger than TRX_SYS_SPACE (0), we will attempt to mark the corresponding table as corrupted instead of crashing server */ - if (bpage->space > TRX_SYS_SPACE - && buf_mark_space_corrupt(bpage)) { - return(false); + if (bpage->space > TRX_SYS_SPACE) { + buf_mark_space_corrupt(bpage); + return(err); } else { - if (!bpage->encrypted) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Ending processing because of a corrupt database page."); - - ut_error; - } - - ib_push_warning(innobase_get_trx(), DB_DECRYPTION_FAILED, - "Table in tablespace %lu encrypted." - "However key management plugin or used key_id %lu is not found or" - " used encryption algorithm or method does not match." - " Can't continue opening the table.", - bpage->space, bpage->key_version); - - if (bpage->encrypted && bpage->space > TRX_SYS_SPACE) { - buf_mark_space_corrupt(bpage); - } else { - ut_error; - } - - return(false); + ib_logf(IB_LOG_LEVEL_FATAL, + "Ending processing because of a corrupt database page."); } } } @@ -4872,11 +4821,12 @@ database_corrupted: if (bpage && bpage->encrypted) { ib_logf(IB_LOG_LEVEL_WARN, - "Table in tablespace %lu encrypted." - "However key management plugin or used key_version %u is not found or" + "Table in tablespace " ULINTPF " encrypted." + "However key management plugin or used " + " key_version %u is not found or" " used encryption algorithm or method does not match." - " Can't continue opening the table.\n", - (ulint)bpage->space, bpage->key_version); + " Can't continue opening the table.", + read_space_id, key_version); } else { ibuf_merge_or_delete_for_page( @@ -4998,7 +4948,7 @@ retry_mutex: mutex_exit(block_mutex); - return(true); + return(err); } /*********************************************************************//** @@ -5026,7 +4976,7 @@ buf_all_freed_instance( mutex_exit(&buf_pool->LRU_list_mutex); - if (UNIV_LIKELY_NULL(block) && block->page.key_version == 0) { + if (UNIV_LIKELY_NULL(block)) { fil_space_t* space = fil_space_get(block->page.space); ib_logf(IB_LOG_LEVEL_ERROR, "Page %u %u still fixed or dirty.", @@ -6266,32 +6216,29 @@ buf_page_encrypt_before_write( switch (bpage->offset) { case 0: /* Page 0 of a tablespace is not encrypted/compressed */ - ut_ad(bpage->key_version == 0); return src_frame; case TRX_SYS_PAGE_NO: if (bpage->space == TRX_SYS_SPACE) { /* don't encrypt/compress page as it contains address to dblwr buffer */ - bpage->key_version = 0; return src_frame; } } fil_space_crypt_t* crypt_data = space->crypt_data; + const bool encrypted = crypt_data && !crypt_data->not_encrypted() && crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && (!crypt_data->is_default_encryption() || srv_encrypt_tables); - if (!encrypted) { - bpage->key_version = 0; - } - bool page_compressed = FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags); if (!encrypted && !page_compressed) { - /* No need to encrypt or page compress the page */ + /* No need to encrypt or page compress the page. + Clear key-version & crypt-checksum. */ + memset(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); return src_frame; } @@ -6313,9 +6260,6 @@ buf_page_encrypt_before_write( src_frame, dst_frame); - ulint key_version = mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - ut_ad(key_version == 0 || key_version >= bpage->key_version); - bpage->key_version = key_version; bpage->real_size = page_size; slot->out_buf = dst_frame = tmp; @@ -6384,19 +6328,17 @@ buf_page_decrypt_after_read( buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); bool success = true; - bpage->key_version = key_version; - if (bpage->offset == 0) { /* File header pages are not encrypted/compressed */ return (true); } - fil_space_t* space = fil_space_acquire(bpage->space, true); + FilSpace space(bpage->space, false, true); /* Page is encrypted if encryption information is found from tablespace and page contains used key_version. This is true also for pages first compressed and then encrypted. */ - if (!space || !space->crypt_data) { + if (!space() || !space()->crypt_data) { key_version = 0; } @@ -6430,6 +6372,14 @@ buf_page_decrypt_after_read( decrypt. */ if (!fil_space_verify_crypt_checksum(dst_frame, zip_size, NULL, bpage->offset)) { + + /* Mark page encrypted in case it should + be. */ + if (key_version && space()->crypt_data && + space()->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED) { + bpage->encrypted = true; + } + return (false); } @@ -6441,7 +6391,7 @@ buf_page_decrypt_after_read( #endif /* decrypt using crypt_buf to dst_frame */ - byte* res = fil_space_decrypt(space, + byte* res = fil_space_decrypt(space(), slot->crypt_buf, dst_frame, &bpage->encrypted); @@ -6477,8 +6427,5 @@ buf_page_decrypt_after_read( } } - if (space != NULL) { - fil_space_release(space); - } return (success); } diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index 55c5e4d543a..1c9646c0bd6 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -493,7 +493,9 @@ buf_dblwr_process() page_no = mach_read_from_4(page + FIL_PAGE_OFFSET); space_id = mach_read_from_4(page + FIL_PAGE_SPACE_ID); - if (!fil_tablespace_exists_in_mem(space_id)) { + FilSpace space(space_id, true); + + if (!space()) { /* Maybe we have dropped the single-table tablespace and this page once belonged to it: do nothing */ continue; @@ -508,8 +510,7 @@ buf_dblwr_process() continue; } - fil_space_t* space = fil_space_found_by_id(space_id); - ulint zip_size = fil_space_get_zip_size(space_id); + ulint zip_size = fsp_flags_get_zip_size(space()->flags); ut_ad(!buf_page_is_zeroes(page, zip_size)); /* Read in the actual page from the file */ @@ -545,7 +546,7 @@ buf_dblwr_process() if (fil_space_verify_crypt_checksum( read_buf, zip_size, NULL, page_no) || !buf_page_is_corrupted( - true, read_buf, zip_size, space)) { + true, read_buf, zip_size, space())) { /* The page is good; there is no need to consult the doublewrite buffer. */ continue; diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index ffd40157c30..62a2468ba66 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -994,7 +994,13 @@ buf_flush_write_block_low( if (sync) { ut_ad(flush_type == BUF_FLUSH_SINGLE_PAGE); fil_flush(space); + +#ifdef UNIV_DEBUG + dberr_t err = +#endif buf_page_io_complete(bpage); + + ut_ad(err == DB_SUCCESS); } fil_space_release(space); diff --git a/storage/xtradb/buf/buf0rea.cc b/storage/xtradb/buf/buf0rea.cc index be2ee56504b..e275eead4cc 100644 --- a/storage/xtradb/buf/buf0rea.cc +++ b/storage/xtradb/buf/buf0rea.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2014, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -103,36 +103,46 @@ flag is cleared and the x-lock released by an i/o-handler thread. in buf_pool, or if the page is in the doublewrite buffer blocks in which case it is never read into the pool, or if the tablespace does not exist or is being dropped -@return 1 if read request is issued. 0 if it is not */ -UNIV_INTERN -ulint -buf_read_page_low( -/*==============*/ - dberr_t* err, /*!< out: DB_SUCCESS or DB_TABLESPACE_DELETED if we are - trying to read from a non-existent tablespace, or a - tablespace which is just now being dropped */ - bool sync, /*!< in: true if synchronous aio is desired */ - ulint mode, /*!< in: BUF_READ_IBUF_PAGES_ONLY, ..., - ORed to OS_AIO_SIMULATED_WAKE_LATER (see below - at read-ahead functions) */ - ulint space, /*!< in: space id */ - ulint zip_size,/*!< in: compressed page size, or 0 */ - ibool unzip, /*!< in: TRUE=request uncompressed page */ - ib_int64_t tablespace_version, /*!< in: if the space memory object has + +@param[out] err DB_SUCCESS, DB_TABLESPACE_DELETED if we are + trying to read from a non-existent tablespace, or a + tablespace which is just now being dropped, + DB_PAGE_CORRUPTED if page based on checksum + check is corrupted, or DB_DECRYPTION_FAILED + if page post encryption checksum matches but + after decryption normal page checksum does not match. +@param[in] sync true if synchronous aio is desired +@param[in] mode BUF_READ_IBUF_PAGES_ONLY, ..., + ORed to OS_AIO_SIMULATED_WAKE_LATER (see below + at read-ahead functions) +@param[in] space space id +@param[in] zip_size compressed page size, or 0 +@param[in] unzip TRUE=request uncompressed page +@param[in] tablespace_version if the space memory object has this timestamp different from what we are giving here, treat the tablespace as dropped; this is a timestamp we use to stop dangling page reads from a tablespace - which we have DISCARDed + IMPORTed back */ - ulint offset, /*!< in: page number */ - trx_t* trx, /*!< in: trx */ - buf_page_t** rbpage) /*!< out: page */ + which we have DISCARDed + IMPORTed back +@param[in] offset page number +@param[in] trx transaction +@return 1 if read request is issued. 0 if it is not */ +static +ulint +buf_read_page_low( + dberr_t* err, + bool sync, + ulint mode, + ulint space, + ulint zip_size, + ibool unzip, + ib_int64_t tablespace_version, + ulint offset, + trx_t* trx = NULL) { buf_page_t* bpage; ulint wake_later; ibool ignore_nonexistent_pages; - *err = DB_SUCCESS; - wake_later = mode & OS_AIO_SIMULATED_WAKE_LATER; mode = mode & ~OS_AIO_SIMULATED_WAKE_LATER; @@ -259,18 +269,13 @@ not_to_recover: if (sync) { /* The i/o is already completed when we arrive from fil_read */ - if (!buf_page_io_complete(bpage)) { - if (rbpage) { - *rbpage = bpage; - } + *err = buf_page_io_complete(bpage); + + if (*err != DB_SUCCESS) { return(0); } } - if (rbpage) { - *rbpage = bpage; - } - return(1); } @@ -307,7 +312,7 @@ buf_read_ahead_random( ulint ibuf_mode; ulint count; ulint low, high; - dberr_t err; + dberr_t err = DB_SUCCESS; ulint i; const ulint buf_read_ahead_random_area = BUF_READ_AHEAD_AREA(buf_pool); @@ -402,20 +407,33 @@ read_ahead: mode: hence FALSE as the first parameter */ if (!ibuf_bitmap_page(zip_size, i)) { + count += buf_read_page_low( &err, false, ibuf_mode | OS_AIO_SIMULATED_WAKE_LATER, space, zip_size, FALSE, - tablespace_version, i, trx, NULL); - if (err == DB_TABLESPACE_DELETED) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Warning: in random" - " readahead trying to access\n" - "InnoDB: tablespace %lu page %lu,\n" - "InnoDB: but the tablespace does not" - " exist or is just being dropped.\n", - (ulong) space, (ulong) i); + tablespace_version, i, trx); + + switch(err) { + case DB_SUCCESS: + break; + case DB_TABLESPACE_DELETED: + ib_logf(IB_LOG_LEVEL_WARN, + "In random" + " readahead trying to access" + " tablespace " ULINTPF ":" ULINTPF + " but the tablespace does not" + " exist or is just being dropped.", + space, i); + break; + case DB_DECRYPTION_FAILED: + ib_logf(IB_LOG_LEVEL_ERROR, + "Random readahead failed to decrypt page " + ULINTPF ":" ULINTPF ".", + i, space); + break; + default: + ut_error; } } } @@ -449,44 +467,60 @@ High-level function which reads a page asynchronously from a file to the buffer buf_pool if it is not already there. Sets the io_fix flag and sets an exclusive lock on the buffer frame. The flag is cleared and the x-lock released by the i/o-handler thread. -@return TRUE if page has been read in, FALSE in case of failure */ + +@param[in] space_id space_id +@param[in] zip_size compressed page size in bytes, or 0 +@param[in] offset page number +@param[in] trx transaction +@param[out] encrypted true if page encrypted +@return DB_SUCCESS if page has been read and is not corrupted, +@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted, +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but +after decryption normal page checksum does not match. +@retval DB_TABLESPACE_DELETED if tablespace .ibd file is missing */ UNIV_INTERN -ibool +dberr_t buf_read_page( -/*==========*/ - ulint space, /*!< in: space id */ - ulint zip_size, /*!< in: compressed page size in bytes, or 0 */ - ulint offset, /*!< in: page number */ - trx_t* trx, /*!< in: trx */ - buf_page_t** bpage) /*!< out: page */ + ulint space_id, + ulint zip_size, + ulint offset, + trx_t* trx) { ib_int64_t tablespace_version; ulint count; - dberr_t err; + dberr_t err = DB_SUCCESS; - tablespace_version = fil_space_get_version(space); + tablespace_version = fil_space_get_version(space_id); - /* We do the i/o in the synchronous aio mode to save thread - switches: hence TRUE */ + FilSpace space(space_id, true); - count = buf_read_page_low(&err, true, BUF_READ_ANY_PAGE, space, + if (space()) { + + /* We do the i/o in the synchronous aio mode to save thread + switches: hence TRUE */ + count = buf_read_page_low(&err, true, BUF_READ_ANY_PAGE, space_id, zip_size, FALSE, - tablespace_version, offset, trx, bpage); - srv_stats.buf_pool_reads.add(count); - if (err == DB_TABLESPACE_DELETED) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: trying to access" - " tablespace %lu page no. %lu,\n" - "InnoDB: but the tablespace does not exist" - " or is just being dropped.\n", - (ulong) space, (ulong) offset); + tablespace_version, offset, trx); + + srv_stats.buf_pool_reads.add(count); + } + + /* Page corruption and decryption failures are already reported + in above function. */ + if (!space() || err == DB_TABLESPACE_DELETED) { + err = DB_TABLESPACE_DELETED; + ib_logf(IB_LOG_LEVEL_ERROR, + "Trying to access" + " tablespace [space=" ULINTPF ": page=" ULINTPF + "] but the tablespace does not exist" + " or is just being dropped.", + space_id, offset); } /* Increment number of I/O operations used for LRU policy. */ buf_LRU_stat_inc_io(); - return(count > 0); + return(err); } /********************************************************************//** @@ -494,23 +528,23 @@ High-level function which reads a page asynchronously from a file to the buffer buf_pool if it is not already there. Sets the io_fix flag and sets an exclusive lock on the buffer frame. The flag is cleared and the x-lock released by the i/o-handler thread. -@return TRUE if page has been read in, FALSE in case of failure */ +@param[in] space Tablespace id +@param[in] offset Page no */ UNIV_INTERN -ibool +void buf_read_page_async( -/*================*/ - ulint space, /*!< in: space id */ - ulint offset) /*!< in: page number */ + ulint space, + ulint offset) { ulint zip_size; ib_int64_t tablespace_version; ulint count; - dberr_t err; + dberr_t err = DB_SUCCESS; zip_size = fil_space_get_zip_size(space); if (zip_size == ULINT_UNDEFINED) { - return(FALSE); + return; } tablespace_version = fil_space_get_version(space); @@ -519,7 +553,30 @@ buf_read_page_async( | OS_AIO_SIMULATED_WAKE_LATER | BUF_READ_IGNORE_NONEXISTENT_PAGES, space, zip_size, FALSE, - tablespace_version, offset, NULL,NULL); + tablespace_version, offset); + + switch(err) { + case DB_SUCCESS: + break; + case DB_TABLESPACE_DELETED: + ib_logf(IB_LOG_LEVEL_ERROR, + "In async page read " + "trying to access " + "page " ULINTPF ":" ULINTPF + " in nonexisting or being-dropped tablespace", + space, offset); + break; + + case DB_DECRYPTION_FAILED: + ib_logf(IB_LOG_LEVEL_ERROR, + "Async page read failed to decrypt page " + ULINTPF ":" ULINTPF ".", + space, offset); + break; + default: + ut_error; + } + srv_stats.buf_pool_reads.add(count); /* We do not increment number of I/O operations used for LRU policy @@ -528,8 +585,6 @@ buf_read_page_async( buffer pool. Since this function is called from buffer pool load these IOs are deliberate and are not part of normal workload we can ignore these in our heuristics. */ - - return(count > 0); } /********************************************************************//** @@ -580,7 +635,7 @@ buf_read_ahead_linear( ulint fail_count; ulint ibuf_mode; ulint low, high; - dberr_t err; + dberr_t err = DB_SUCCESS; ulint i; const ulint buf_read_ahead_linear_area = BUF_READ_AHEAD_AREA(buf_pool); @@ -784,19 +839,34 @@ buf_read_ahead_linear( aio mode: hence FALSE as the first parameter */ if (!ibuf_bitmap_page(zip_size, i)) { + count += buf_read_page_low( &err, false, ibuf_mode, - space, zip_size, FALSE, tablespace_version, i, trx, NULL); - if (err == DB_TABLESPACE_DELETED) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Warning: in" - " linear readahead trying to access\n" - "InnoDB: tablespace %lu page %lu,\n" - "InnoDB: but the tablespace does not" - " exist or is just being dropped.\n", - (ulong) space, (ulong) i); + space, zip_size, FALSE, tablespace_version, + i, trx); + + switch(err) { + case DB_SUCCESS: + break; + case DB_TABLESPACE_DELETED: + ib_logf(IB_LOG_LEVEL_WARN, + "In linear" + " readahead trying to access" + " tablespace " ULINTPF ":" ULINTPF + " but the tablespace does not" + " exist or is just being dropped.", + space, i); + break; + + case DB_DECRYPTION_FAILED: + ib_logf(IB_LOG_LEVEL_ERROR, + "Linear readahead failed to decrypt page " + ULINTPF ":" ULINTPF ".", + i, space); + break; + default: + ut_error; } } } @@ -858,9 +928,9 @@ buf_read_ibuf_merge_pages( #endif for (i = 0; i < n_stored; i++) { - dberr_t err; buf_pool_t* buf_pool; ulint zip_size = fil_space_get_zip_size(space_ids[i]); + dberr_t err = DB_SUCCESS; buf_pool = buf_pool_get(space_ids[i], page_nos[i]); @@ -870,16 +940,19 @@ buf_read_ibuf_merge_pages( } if (UNIV_UNLIKELY(zip_size == ULINT_UNDEFINED)) { - goto tablespace_deleted; } buf_read_page_low(&err, sync && (i + 1 == n_stored), BUF_READ_ANY_PAGE, space_ids[i], zip_size, TRUE, space_versions[i], - page_nos[i], NULL, NULL); + page_nos[i], NULL); + + switch(err) { + case DB_SUCCESS: + break; + case DB_TABLESPACE_DELETED: - if (UNIV_UNLIKELY(err == DB_TABLESPACE_DELETED)) { tablespace_deleted: /* We have deleted or are deleting the single-table tablespace: remove the entries for that page */ @@ -887,6 +960,15 @@ tablespace_deleted: ibuf_merge_or_delete_for_page(NULL, space_ids[i], page_nos[i], zip_size, FALSE); + break; + case DB_DECRYPTION_FAILED: + ib_logf(IB_LOG_LEVEL_ERROR, + "Failed to decrypt insert buffer page " + ULINTPF ":" ULINTPF ".", + space_ids[i], page_nos[i]); + break; + default: + ut_error; } } @@ -924,7 +1006,7 @@ buf_read_recv_pages( { ib_int64_t tablespace_version; ulint count; - dberr_t err; + dberr_t err = DB_SUCCESS; ulint i; zip_size = fil_space_get_zip_size(space); @@ -1013,12 +1095,20 @@ not_to_recover: if ((i + 1 == n_stored) && sync) { buf_read_page_low(&err, true, BUF_READ_ANY_PAGE, space, zip_size, TRUE, tablespace_version, - page_nos[i], NULL, NULL); + page_nos[i], NULL); } else { buf_read_page_low(&err, false, BUF_READ_ANY_PAGE | OS_AIO_SIMULATED_WAKE_LATER, space, zip_size, TRUE, - tablespace_version, page_nos[i], NULL, NULL); + tablespace_version, page_nos[i], + NULL); + } + + if (err == DB_DECRYPTION_FAILED) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Recovery failed to decrypt read page " + ULINTPF ":" ULINTPF ".", + space, page_nos[i]); } } diff --git a/storage/xtradb/dict/dict0crea.cc b/storage/xtradb/dict/dict0crea.cc index 1ada35a89a2..6d5b12474eb 100644 --- a/storage/xtradb/dict/dict0crea.cc +++ b/storage/xtradb/dict/dict0crea.cc @@ -692,7 +692,7 @@ dict_create_index_tree_step( dberr_t err = DB_SUCCESS; ulint zip_size = dict_table_zip_size(index->table); - if (node->index->table->ibd_file_missing + if (node->index->table->file_unreadable || dict_table_is_discarded(node->index->table)) { node->page_no = FIL_NULL; diff --git a/storage/xtradb/dict/dict0dict.cc b/storage/xtradb/dict/dict0dict.cc index 49de1cf7ef8..bc46bcab63b 100644 --- a/storage/xtradb/dict/dict0dict.cc +++ b/storage/xtradb/dict/dict0dict.cc @@ -1185,47 +1185,42 @@ dict_table_open_on_name( if (table != NULL) { - /* If table is encrypted return table */ + /* If table is encrypted or corrupted */ if (ignore_err == DICT_ERR_IGNORE_NONE - && table->is_encrypted) { + && !table->is_readable()) { /* Make life easy for drop table. */ if (table->can_be_evicted) { dict_table_move_from_lru_to_non_lru(table); } - if (table->can_be_evicted) { - dict_move_to_mru(table); - } + if (table->corrupted) { - ++table->n_ref_count; + if (!dict_locked) { + mutex_exit(&dict_sys->mutex); + } - if (!dict_locked) { - mutex_exit(&dict_sys->mutex); - } + char buf[MAX_FULL_NAME_LEN]; + ut_format_name(table->name, TRUE, buf, sizeof(buf)); - return (table); - } - /* If table is corrupted, return NULL */ - else if (ignore_err == DICT_ERR_IGNORE_NONE - && table->corrupted) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Table %s is corrupted. Please " + "drop the table and recreate.", + buf); + + return(NULL); + } - /* Make life easy for drop table. */ if (table->can_be_evicted) { - dict_table_move_from_lru_to_non_lru(table); + dict_move_to_mru(table); } + ++table->n_ref_count; + if (!dict_locked) { mutex_exit(&dict_sys->mutex); } - ut_print_timestamp(stderr); - - fprintf(stderr, " InnoDB: table "); - ut_print_name(stderr, NULL, TRUE, table->name); - fprintf(stderr, "is corrupted. Please drop the table " - "and recreate\n"); - - return(NULL); + return (table); } if (table->can_be_evicted) { @@ -6183,11 +6178,29 @@ dict_set_corrupted_by_space( /* mark the table->corrupted bit only, since the caller could be too deep in the stack for SYS_INDEXES update */ - table->corrupted = TRUE; + table->corrupted = true; + table->file_unreadable = true; return(TRUE); } + +/** Flags a table with specified space_id encrypted in the data dictionary +cache +@param[in] space_id Tablespace id */ +UNIV_INTERN +void +dict_set_encrypted_by_space(ulint space_id) +{ + dict_table_t* table; + + table = dict_find_table_by_space(space_id); + + if (table) { + table->file_unreadable = true; + } +} + /**********************************************************************//** Flags an index corrupted both in the data dictionary cache and in the SYS_INDEXES */ @@ -6325,43 +6338,6 @@ dict_set_corrupted_index_cache_only( index->type |= DICT_CORRUPT; } -/************************************************************************* -set is_corrupt flag by space_id*/ - -void -dict_table_set_corrupt_by_space( -/*============================*/ - ulint space_id, - ibool need_mutex) -{ - dict_table_t* table; - ibool found = FALSE; - - ut_a(space_id != 0 && space_id < SRV_LOG_SPACE_FIRST_ID); - - if (need_mutex) - mutex_enter(&(dict_sys->mutex)); - - table = UT_LIST_GET_FIRST(dict_sys->table_LRU); - - while (table) { - if (table->space == space_id) { - table->is_corrupt = TRUE; - found = TRUE; - } - - table = UT_LIST_GET_NEXT(table_LRU, table); - } - - if (need_mutex) - mutex_exit(&(dict_sys->mutex)); - - if (!found) { - fprintf(stderr, "InnoDB: space to be marked as " - "crashed was not found for id " ULINTPF ".\n", - space_id); - } -} #endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** @@ -6683,7 +6659,8 @@ dict_table_schema_check( } } - if (table->ibd_file_missing) { + if (!table->is_readable() && + fil_space_get(table->space) == NULL) { /* missing tablespace */ ut_snprintf(errstr, errstr_sz, diff --git a/storage/xtradb/dict/dict0load.cc b/storage/xtradb/dict/dict0load.cc index b843891f16c..2dbde465369 100644 --- a/storage/xtradb/dict/dict0load.cc +++ b/storage/xtradb/dict/dict0load.cc @@ -1964,7 +1964,7 @@ dict_load_indexes( dict_mem_index_free(index); goto func_exit; } else if (index->page == FIL_NULL - && !table->ibd_file_missing + && !table->file_unreadable && (!(index->type & DICT_FTS))) { fprintf(stderr, @@ -2193,7 +2193,7 @@ err_len: (*table)->id = mach_read_from_8(field); - (*table)->ibd_file_missing = FALSE; + (*table)->file_unreadable = false; return(NULL); } @@ -2380,7 +2380,7 @@ err_exit: "Table '%s' tablespace is set as discarded.", table_name); - table->ibd_file_missing = TRUE; + table->file_unreadable = true; } else if (!fil_space_for_table_exists_in_mem( table->space, name, false, true, heap, @@ -2388,7 +2388,7 @@ err_exit: if (DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY)) { /* Do not bother to retry opening temporary tables. */ - table->ibd_file_missing = TRUE; + table->file_unreadable = true; } else { if (!(ignore_err & DICT_ERR_IGNORE_RECOVER_LOCK)) { @@ -2423,8 +2423,9 @@ err_exit: /* We failed to find a sensible tablespace file */ - table->ibd_file_missing = TRUE; + table->file_unreadable = true; } + if (filepath) { mem_free(filepath); } @@ -2448,9 +2449,10 @@ err_exit: were not allowed while the table is being locked by a transaction. */ dict_err_ignore_t index_load_err = !(ignore_err & DICT_ERR_IGNORE_RECOVER_LOCK) - && table->ibd_file_missing + && table->file_unreadable ? DICT_ERR_IGNORE_ALL : ignore_err; + err = dict_load_indexes(table, heap, index_load_err); if (err == DB_INDEX_CORRUPT) { @@ -2485,7 +2487,7 @@ err_exit: of the error condition, since the user may want to dump data from the clustered index. However we load the foreign key information only if all indexes were loaded. */ - if (!cached || table->ibd_file_missing) { + if (!cached || table->file_unreadable) { /* Don't attempt to load the indexes from disk. */ } else if (err == DB_SUCCESS) { err = dict_load_foreigns(table->name, NULL, true, true, @@ -2518,12 +2520,12 @@ err_exit: table = NULL; } else if (dict_index_is_corrupted(index) - && !table->ibd_file_missing) { + && !table->file_unreadable) { /* It is possible we force to load a corrupted clustered index if srv_load_corrupted is set. Mark the table as corrupted in this case */ - table->corrupted = TRUE; + table->corrupted = true; } } @@ -2532,7 +2534,7 @@ func_exit: ut_ad(!table || ignore_err != DICT_ERR_IGNORE_NONE - || table->ibd_file_missing + || table->file_unreadable || !table->corrupted); if (table && table->fts) { diff --git a/storage/xtradb/dict/dict0mem.cc b/storage/xtradb/dict/dict0mem.cc index bde72b83c54..fa7177c5137 100644 --- a/storage/xtradb/dict/dict0mem.cc +++ b/storage/xtradb/dict/dict0mem.cc @@ -2,6 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -137,8 +138,6 @@ dict_mem_table_create( table->fts = NULL; } - table->is_corrupt = FALSE; - #endif /* !UNIV_HOTBACKUP */ new(&table->foreign_set) dict_foreign_set(); diff --git a/storage/xtradb/dict/dict0stats.cc b/storage/xtradb/dict/dict0stats.cc index 33b6c2e23e0..d2e9a2618c0 100644 --- a/storage/xtradb/dict/dict0stats.cc +++ b/storage/xtradb/dict/dict0stats.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2009, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -920,7 +921,11 @@ dict_stats_update_transient_for_index( index->stat_n_leaf_pages = size; - btr_estimate_number_of_different_key_vals(index); + /* Do not continue if table decryption has failed or + table is already marked as corrupted. */ + if (index->is_readable()) { + btr_estimate_number_of_different_key_vals(index); + } } } @@ -974,8 +979,9 @@ dict_stats_update_transient( continue; } - /* Do not continue if table decryption has failed. */ - if (index->table->is_encrypted) { + /* Do not continue if table decryption has failed or + table is already marked as corrupted. */ + if (!index->is_readable()) { break; } @@ -2437,6 +2443,61 @@ dict_stats_save_index_stat( return(ret); } +/** Report error if statistic update for a table failed because +.ibd file is missing, table decryption failed or table is corrupted. +@param[in,out] table Table +@param[in] defragment true if statistics is for defragment +@return DB_DECRYPTION_FAILED, DB_TABLESPACE_DELETED or DB_CORRUPTION +@retval DB_DECRYPTION_FAILED if decryption of the table failed +@retval DB_TABLESPACE_DELETED if .ibd file is missing +@retval DB_CORRUPTION if table is marked as corrupted */ +static +dberr_t +dict_stats_report_error( + dict_table_t* table, + bool defragment = false) +{ + char buf[3 * NAME_LEN]; + dberr_t err; + + innobase_format_name(buf, sizeof buf, + table->name, + true); + + FilSpace space(table->space); + + if (space()) { + if (table->corrupted) { + ib_logf(IB_LOG_LEVEL_INFO, + "Cannot save%s statistics because " + " table %s in file %s is corrupted.", + defragment ? " defragment" : " ", + buf, space()->chain.start->name); + err = DB_CORRUPTION; + } else { + ib_logf(IB_LOG_LEVEL_INFO, + "Cannot save%s statistics because " + " table %s in file %s can't be decrypted.", + defragment ? " defragment" : " ", + buf, space()->chain.start->name); + err = DB_DECRYPTION_FAILED; + } + } else { + ib_logf(IB_LOG_LEVEL_INFO, + "Cannot save%s statistics for " + " table %s because .ibd file is missing." + " For help, please " + "refer to " REFMAN "innodb-troubleshooting.html.", + defragment ? " defragment" : " ", + buf); + err = DB_TABLESPACE_DELETED; + } + + dict_stats_empty_table(table, defragment); + + return (err); +} + /** Save the table's statistics into the persistent statistics storage. @param[in] table_orig table whose stats to save @param[in] only_for_index if this is non-NULL, then stats for indexes @@ -2457,6 +2518,11 @@ dict_stats_save( char db_utf8[MAX_DB_UTF8_LEN]; char table_utf8[MAX_TABLE_UTF8_LEN]; + if (table_orig->is_readable()) { + } else { + return (dict_stats_report_error(table_orig)); + } + table = dict_stats_snapshot_create(table_orig); dict_fs2utf8(table->name, db_utf8, sizeof(db_utf8), @@ -3192,15 +3258,8 @@ dict_stats_update( ut_ad(!mutex_own(&dict_sys->mutex)); - if (table->ibd_file_missing) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: cannot calculate statistics for table %s " - "because the .ibd file is missing. For help, please " - "refer to " REFMAN "innodb-troubleshooting.html\n", - ut_format_name(table->name, TRUE, buf, sizeof(buf))); - dict_stats_empty_table(table, true); - return(DB_TABLESPACE_DELETED); + if (!table->is_readable()) { + return (dict_stats_report_error(table)); } else if (srv_force_recovery >= SRV_FORCE_NO_IBUF_MERGE) { /* If we have set a high innodb_force_recovery level, do not calculate statistics, as a badly corrupted index can @@ -3946,19 +4005,10 @@ dict_stats_save_defrag_stats( { dberr_t ret; - if (index->table->ibd_file_missing) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Cannot save defragment stats because " - ".ibd file is missing.\n"); - return (DB_TABLESPACE_DELETED); - } - if (dict_index_is_corrupted(index)) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Cannot save defragment stats because " - "index is corrupted.\n"); - return(DB_CORRUPTION); + + if (index->is_readable()) { + } else { + return (dict_stats_report_error(index->table, true)); } if (dict_index_is_univ(index)) { diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index f37d5e19ad6..c1ec1c7b1fd 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -479,11 +479,12 @@ Parse a MLOG_FILE_WRITE_CRYPT_DATA log entry @param[in] block buffer block @return position on log buffer */ UNIV_INTERN -const byte* +byte* fil_parse_write_crypt_data( - const byte* ptr, + byte* ptr, const byte* end_ptr, - const buf_block_t* block) + const buf_block_t* block, + dberr_t* err) { /* check that redo log entry is complete */ size_t entry_size = @@ -495,6 +496,8 @@ fil_parse_write_crypt_data( 4 + // size of key_id 1; // fil_encryption_t + *err = DB_SUCCESS; + if (ptr + entry_size > end_ptr) { return NULL; } @@ -541,6 +544,11 @@ fil_parse_write_crypt_data( fil_space_release(space); } + /* Check is used key found from encryption plugin */ + if (crypt_data->should_encrypt() && !crypt_data->is_key_found()) { + *err = DB_DECRYPTION_FAILED; + } + return ptr; } @@ -710,10 +718,12 @@ fil_space_encrypt( } bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size, space); + memcpy(tmp_mem+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, src+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 8); bool different = memcmp(src, tmp_mem, size); if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) { - fprintf(stderr, "ok %d corrupted %d corrupted1 %d err %d different %d\n", ok , corrupted, corrupted1, err, different); + fprintf(stderr, "ok %d corrupted %d corrupted1 %d err %d different %d\n", + ok , corrupted, corrupted1, err, different); fprintf(stderr, "src_frame\n"); buf_page_print(src_frame, zip_size, BUF_PAGE_PRINT_NO_CRASH); fprintf(stderr, "encrypted_frame\n"); @@ -768,25 +778,6 @@ fil_space_decrypt( return false; } - if (crypt_data == NULL) { - if (!(space == 0 && offset == 0) && key_version != 0) { - /* FIL_PAGE_FILE_FLUSH_LSN field i.e. - FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION - should be only defined for the - first page in a system tablespace - data file (ibdata*, not *.ibd), if not - clear it. */ - - DBUG_PRINT("ib_crypt", - ("Page on space %lu offset %lu has key_version %u" - " when it shoud be undefined.", - space, offset, key_version)); - - mach_write_to_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0); - } - return false; - } - ut_a(crypt_data != NULL && crypt_data->is_encrypted()); /* read space & lsn */ @@ -838,9 +829,6 @@ fil_space_decrypt( memcpy(tmp_frame + page_size - FIL_PAGE_DATA_END, src_frame + page_size - FIL_PAGE_DATA_END, FIL_PAGE_DATA_END); - - // clear key-version & crypt-checksum from dst - memset(tmp_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); } srv_stats.pages_decrypted.inc(); @@ -962,11 +950,12 @@ fil_space_verify_crypt_checksum( /* If page is not encrypted, return false */ if (key_version == 0) { - return false; + return(false); } srv_checksum_algorithm_t algorithm = static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm); + /* If no checksum is used, can't continue checking. */ if (algorithm == SRV_CHECKSUM_ALGORITHM_NONE) { return(true); @@ -990,28 +979,31 @@ fil_space_verify_crypt_checksum( return (true); } - /* Compressed pages use different checksum method. We first store - the post encryption checksum on checksum location and after function - restore the original. */ - if (zip_size) { - ib_uint32_t old = static_cast<ib_uint32_t>(mach_read_from_4( - page + FIL_PAGE_SPACE_OR_CHKSUM)); - - mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); + ib_uint32_t cchecksum1 = 0; + ib_uint32_t cchecksum2 = 0; - bool valid = page_zip_verify_checksum(page, zip_size); + /* Calculate checksums */ + if (zip_size) { + cchecksum1 = page_zip_calc_checksum( + page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32); - mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, old); + if(cchecksum1 != checksum) { + cchecksum2 = page_zip_calc_checksum( + page, zip_size, + SRV_CHECKSUM_ALGORITHM_INNODB); + } + } else { + cchecksum1 = buf_calc_page_crc32(page); - return (valid); + if (cchecksum1 != checksum) { + cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum( + page); + } } /* If stored checksum matches one of the calculated checksums page is not corrupted. */ - ib_uint32_t cchecksum1 = buf_calc_page_crc32(page); - ib_uint32_t cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum( - page); bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2 || checksum == BUF_NO_CHECKSUM_MAGIC); @@ -1042,13 +1034,19 @@ fil_space_verify_crypt_checksum( ulint checksum1 = mach_read_from_4( page + FIL_PAGE_SPACE_OR_CHKSUM); - ulint checksum2 = mach_read_from_4( - page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); + ulint checksum2 = checksum1; + bool valid; - bool valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2) + if (zip_size) { + valid = (checksum1 == cchecksum1); + } else { + checksum1 = mach_read_from_4( + page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); + valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2) || buf_page_is_checksum_valid_none(page,checksum1,checksum2) || buf_page_is_checksum_valid_innodb(page,checksum1, checksum2)); + } if (encrypted && valid) { /* If page is encrypted and traditional checksums match, @@ -1964,11 +1962,11 @@ fil_crypt_rotate_page( bool modified = false; int needs_scrubbing = BTR_SCRUB_SKIP_PAGE; lsn_t block_lsn = block->page.newest_modification; - uint kv = block->page.key_version; + byte* frame = buf_block_get_frame(block); + uint kv = mach_read_from_4(frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); /* check if tablespace is closing after reading page */ - if (space->is_stopping()) { - byte* frame = buf_block_get_frame(block); + if (!space->is_stopping()) { if (kv == 0 && fil_crypt_is_page_uninitialized(frame, zip_size)) { @@ -1990,9 +1988,6 @@ fil_crypt_rotate_page( FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id, MLOG_4BYTES, &mtr); - /* update block */ - block->page.key_version = key_state->key_version; - /* statistics */ state->crypt_stat.pages_modified++; } else { @@ -2083,6 +2078,11 @@ fil_crypt_rotate_page( state->end_lsn = block_lsn; } } + } else { + /* If block read failed mtr memo and log should be empty. */ + ut_ad(dyn_array_get_data_size(&mtr.memo) == 0); + ut_ad(dyn_array_get_data_size(&mtr.log) == 0); + mtr_commit(&mtr); } if (sleeptime_ms) { diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index e504ab3947e..7ac08cc0e97 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -330,7 +330,11 @@ fil_write( } /*******************************************************************//** -Returns the table space by a given id, NULL if not found. */ +Returns the table space by a given id, NULL if not found. +It is unsafe to dereference the returned pointer. It is fine to check +for NULL. +@param[in] id Tablespace id +@return table space or NULL */ fil_space_t* fil_space_get_by_id( /*================*/ @@ -350,26 +354,6 @@ fil_space_get_by_id( return(space); } -/*******************************************************************//** -Returns the table space by a given id, NULL if not found. */ -fil_space_t* -fil_space_found_by_id( -/*==================*/ - ulint id) /*!< in: space id */ -{ - fil_space_t* space = NULL; - mutex_enter(&fil_system->mutex); - space = fil_space_get_by_id(id); - - /* Not found if space is being deleted */ - if (space && space->stop_new_ops) { - space = NULL; - } - - mutex_exit(&fil_system->mutex); - return space; -} - /****************************************************************//** Get space id from fil node */ ulint @@ -2516,74 +2500,6 @@ fil_read_first_page( /*================ SINGLE-TABLE TABLESPACES ==========================*/ -#ifndef UNIV_HOTBACKUP -/*******************************************************************//** -Increments the count of pending operation, if space is not being deleted. -@return TRUE if being deleted, and operation should be skipped */ -UNIV_INTERN -ibool -fil_inc_pending_ops( -/*================*/ - ulint id, /*!< in: space id */ - ibool print_err) /*!< in: need to print error or not */ -{ - fil_space_t* space; - - mutex_enter(&fil_system->mutex); - - space = fil_space_get_by_id(id); - - if (space == NULL) { - if (print_err) { - fprintf(stderr, - "InnoDB: Error: trying to do an operation on a" - " dropped tablespace %lu\n", - (ulong) id); - } - } - - if (space == NULL || space->stop_new_ops) { - mutex_exit(&fil_system->mutex); - - return(TRUE); - } - - space->n_pending_ops++; - - mutex_exit(&fil_system->mutex); - - return(FALSE); -} - -/*******************************************************************//** -Decrements the count of pending operations. */ -UNIV_INTERN -void -fil_decr_pending_ops( -/*=================*/ - ulint id) /*!< in: space id */ -{ - fil_space_t* space; - - mutex_enter(&fil_system->mutex); - - space = fil_space_get_by_id(id); - - if (space == NULL) { - fprintf(stderr, - "InnoDB: Error: decrementing pending operation" - " of a dropped tablespace %lu\n", - (ulong) id); - } - - if (space != NULL) { - space->n_pending_ops--; - } - - mutex_exit(&fil_system->mutex); -} -#endif /* !UNIV_HOTBACKUP */ - /********************************************************//** Creates the database directory for a table if it does not exist yet. */ static @@ -2975,7 +2891,7 @@ fil_check_pending_operations( fil_space_t* sp = fil_space_get_by_id(id); if (sp) { - sp->stop_new_ops = TRUE; + sp->stop_new_ops = true; /* space could be freed by other threads as soon as n_pending_ops reaches 0, thus increment pending ops here. */ @@ -5390,7 +5306,7 @@ fil_tablespace_deleted_or_being_deleted_in_mem( space = fil_space_get_by_id(id); - if (space == NULL || space->stop_new_ops) { + if (space == NULL || space->is_stopping()) { mutex_exit(&fil_system->mutex); return(TRUE); @@ -6143,15 +6059,19 @@ _fil_io( /* If we are deleting a tablespace we don't allow async read operations on that. However, we do allow write and sync read operations */ if (space == 0 - || (type == OS_FILE_READ && !sync && space->stop_new_ops)) { + || (type == OS_FILE_READ + && !sync + && space->stop_new_ops)) { mutex_exit(&fil_system->mutex); ib_logf(IB_LOG_LEVEL_ERROR, "Trying to do i/o to a tablespace which does " - "not exist. i/o type %lu, space id %lu, " - "page no. %lu, i/o length %lu bytes", - (ulong) type, (ulong) space_id, (ulong) block_offset, - (ulong) len); + "not exist. i/o type " ULINTPF + ", space id " ULINTPF " , " + "page no. " ULINTPF + ", i/o length " ULINTPF " bytes", + type, space_id, block_offset, + len); return(DB_TABLESPACE_DELETED); } @@ -6255,8 +6175,15 @@ _fil_io( mutex_exit(&fil_system->mutex); if (mode == OS_AIO_NORMAL) { ut_a(space->purpose == FIL_TABLESPACE); - buf_page_io_complete(static_cast<buf_page_t *> - (message)); + dberr_t err = buf_page_io_complete(static_cast<buf_page_t *> + (message)); + + if (err != DB_SUCCESS) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Write operation failed for tablespace %s (" + ULINTPF ") offset " ULINTPF " error=%d.", + space->name, space->id, byte_offset, err); + } } } @@ -6359,6 +6286,8 @@ fil_aio_wait( mutex_enter(&fil_system->mutex); fil_node_complete_io(fil_node, fil_system, type); + ulint purpose = fil_node->space->purpose; + space_id = fil_node->space->id; mutex_exit(&fil_system->mutex); @@ -6370,9 +6299,27 @@ fil_aio_wait( deadlocks in the i/o system. We keep tablespace 0 data files always open, and use a special i/o thread to serve insert buffer requests. */ - if (fil_node->space->purpose == FIL_TABLESPACE) { + if (purpose == FIL_TABLESPACE) { srv_set_io_thread_op_info(segment, "complete io for buf page"); - buf_page_io_complete(static_cast<buf_page_t*>(message)); + buf_page_t* bpage = static_cast<buf_page_t*>(message); + ulint offset = bpage->offset; + dberr_t err = buf_page_io_complete(bpage); + + if (err != DB_SUCCESS) { + ut_ad(type == OS_FILE_READ); + /* In crash recovery set log corruption on + and produce only an error to fail InnoDB startup. */ + if (recv_recovery_is_on() && !srv_force_recovery) { + recv_sys->found_corrupt_log = true; + } + + ib_logf(IB_LOG_LEVEL_ERROR, + "Read operation failed for tablespace %s" + " offset " ULINTPF " with error %s", + fil_node->name, + offset, + ut_strerr(err)); + } } else { srv_set_io_thread_op_info(segment, "complete io for log"); log_io_complete(static_cast<log_group_t*>(message)); @@ -6393,7 +6340,8 @@ fil_flush( mutex_enter(&fil_system->mutex); if (fil_space_t* space = fil_space_get_by_id(space_id)) { - if (!space->is_stopping()) { + if (!space->stop_new_ops) { + fil_flush_low(space); } } @@ -7331,76 +7279,6 @@ fil_space_set_corrupt( mutex_exit(&fil_system->mutex); } -/****************************************************************** -Get id of first tablespace or ULINT_UNDEFINED if none */ -UNIV_INTERN -ulint -fil_get_first_space() -/*=================*/ -{ - ulint out_id = ULINT_UNDEFINED; - fil_space_t* space; - - mutex_enter(&fil_system->mutex); - - space = UT_LIST_GET_FIRST(fil_system->space_list); - if (space != NULL) { - do - { - if (!space->stop_new_ops) { - out_id = space->id; - break; - } - space = UT_LIST_GET_NEXT(space_list, space); - } while (space != NULL); - } - - mutex_exit(&fil_system->mutex); - - return out_id; -} - -/****************************************************************** -Get id of next tablespace or ULINT_UNDEFINED if none */ -UNIV_INTERN -ulint -fil_get_next_space( -/*===============*/ - ulint id) /*!< in: previous space id */ -{ - bool found; - fil_space_t* space; - ulint out_id = ULINT_UNDEFINED; - - mutex_enter(&fil_system->mutex); - - space = fil_space_get_by_id(id); - if (space == NULL) { - /* we didn't find it...search for space with space->id > id */ - found = false; - space = UT_LIST_GET_FIRST(fil_system->space_list); - } else { - /* we found it, take next available space */ - found = true; - } - - while ((space = UT_LIST_GET_NEXT(space_list, space)) != NULL) { - - if (!found && space->id <= id) - continue; - - if (!space->stop_new_ops && UT_LIST_GET_LEN(space->chain) > 0) { - /* inc reference to prevent drop */ - out_id = space->id; - break; - } - } - - mutex_exit(&fil_system->mutex); - - return out_id; -} - /** Acquire a tablespace when it could be dropped concurrently. Used by background threads that do not necessarily hold proper locks for concurrency control. @@ -7410,9 +7288,9 @@ for concurrency control. (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted or truncated */ -inline +UNIV_INTERN fil_space_t* -fil_space_acquire_low(ulint id, bool silent, bool for_io = false) +fil_space_acquire_low(ulint id, bool silent, bool for_io) { fil_space_t* space; @@ -7436,32 +7314,6 @@ fil_space_acquire_low(ulint id, bool silent, bool for_io = false) return(space); } -/** Acquire a tablespace when it could be dropped concurrently. -Used by background threads that do not necessarily hold proper locks -for concurrency control. -@param[in] id tablespace ID -@param[in] for_io whether to look up the tablespace while performing I/O - (possibly executing TRUNCATE) -@return the tablespace -@retval NULL if missing or being deleted or truncated */ -fil_space_t* -fil_space_acquire(ulint id, bool for_io) -{ - return(fil_space_acquire_low(id, false, for_io)); -} - -/** Acquire a tablespace that may not exist. -Used by background threads that do not necessarily hold proper locks -for concurrency control. -@param[in] id tablespace ID -@return the tablespace -@retval NULL if missing or being deleted */ -fil_space_t* -fil_space_acquire_silent(ulint id) -{ - return(fil_space_acquire_low(id, true)); -} - /** Release a tablespace acquired with fil_space_acquire(). @param[in,out] space tablespace to release */ void @@ -7590,7 +7442,7 @@ fil_space_keyrotate_next( space->purpose == FIL_TABLESPACE. */ while (space != NULL && (UT_LIST_GET_LEN(space->chain) == 0 - || space->stop_new_ops)) { + || space->is_stopping())) { old = space; space = UT_LIST_GET_NEXT(rotation_list, space); diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 11779a66e7c..91c9fc7824b 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -32,7 +32,7 @@ this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA *****************************************************************************/ - + #define MYSQL_SERVER #include <sql_table.h> // explain_filename, nz2, EXPLAIN_PARTITIONS_AS_COMMENT, @@ -6233,20 +6233,23 @@ ha_innobase::innobase_initialize_autoinc() break; } case DB_RECORD_NOT_FOUND: - ut_print_timestamp(stderr); - fprintf(stderr, " InnoDB: MySQL and InnoDB data " - "dictionaries are out of sync.\n" - "InnoDB: Unable to find the AUTOINC column " - "%s in the InnoDB table %s.\n" - "InnoDB: We set the next AUTOINC column " - "value to 0,\n" - "InnoDB: in effect disabling the AUTOINC " - "next value generation.\n" - "InnoDB: You can either set the next " - "AUTOINC value explicitly using ALTER TABLE\n" - "InnoDB: or fix the data dictionary by " - "recreating the table.\n", - col_name, index->table->name); + char buf[MAX_FULL_NAME_LEN]; + ut_format_name(index->table->name, TRUE, buf, sizeof(buf)); + + ib_logf(IB_LOG_LEVEL_ERROR, + "MySQL and InnoDB data " + "dictionaries are out of sync." + " Unable to find the AUTOINC column " + " %s in the InnoDB table %s." + " We set the next AUTOINC column " + "value to 0" + " in effect disabling the AUTOINC " + "next value generation." + " You can either set the next " + "AUTOINC value explicitly using ALTER TABLE " + " or fix the data dictionary by " + "recreating the table.", + col_name, buf); /* This will disable the AUTOINC generation. */ auto_inc = 0; @@ -6305,7 +6308,7 @@ ha_innobase::open( DBUG_RETURN(1); } - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt && + if (UNIV_UNLIKELY(share->ib_table && share->ib_table->corrupted && srv_pass_corrupt_table <= 1)) { free_share(share); @@ -6341,23 +6344,24 @@ ha_innobase::open( && (table->s->fields != dict_table_get_n_user_cols(ib_table) - 1)))) { ib_logf(IB_LOG_LEVEL_WARN, - "table %s contains %lu user defined columns " - "in InnoDB, but %lu columns in MySQL. Please " + "table %s contains " ULINTPF " user defined columns " + "in InnoDB, but %u columns in MySQL. Please " "check INFORMATION_SCHEMA.INNODB_SYS_COLUMNS and " REFMAN "innodb-troubleshooting.html " "for how to resolve it", - norm_name, (ulong) dict_table_get_n_user_cols(ib_table), - (ulong) table->s->fields); + norm_name, dict_table_get_n_user_cols(ib_table), + table->s->fields); /* Mark this table as corrupted, so the drop table or force recovery can still use it, but not others. */ ib_table->corrupted = true; + ib_table->file_unreadable = true; dict_table_close(ib_table, FALSE, FALSE); ib_table = NULL; is_part = NULL; } - if (UNIV_UNLIKELY(ib_table && ib_table->is_corrupt && + if (UNIV_UNLIKELY(ib_table && ib_table->corrupted && srv_pass_corrupt_table <= 1)) { free_share(share); my_free(upd_buf); @@ -6467,7 +6471,7 @@ table_opened: ib_table->thd = (void*)thd; /* No point to init any statistics if tablespace is still encrypted. */ - if (!ib_table->is_encrypted) { + if (ib_table->is_readable()) { dict_stats_init(ib_table); } else { ib_table->stat_initialized = 1; @@ -6475,7 +6479,9 @@ table_opened: MONITOR_INC(MONITOR_TABLE_OPEN); - bool no_tablespace; + bool no_tablespace = false; + bool encrypted = false; + FilSpace space; if (dict_table_is_discarded(ib_table)) { @@ -6490,23 +6496,28 @@ table_opened: no_tablespace = false; - } else if (ib_table->ibd_file_missing) { + } else if (!ib_table->is_readable()) { + space = fil_space_acquire_silent(ib_table->space); - ib_senderrf( - thd, IB_LOG_LEVEL_WARN, - ER_TABLESPACE_MISSING, norm_name); - - /* This means we have no idea what happened to the tablespace - file, best to play it safe. */ - - no_tablespace = true; - } else if (ib_table->is_encrypted) { - /* This means that tablespace was found but we could not - decrypt encrypted page. */ - no_tablespace = true; - ib_table->ibd_file_missing = true; - } else { - no_tablespace = false; + if (space()) { + if (space()->crypt_data && space()->crypt_data->is_encrypted()) { + /* This means that tablespace was found but we could not + decrypt encrypted page. */ + no_tablespace = true; + encrypted = true; + } else { + no_tablespace = true; + } + } else { + ib_senderrf( + thd, IB_LOG_LEVEL_WARN, + ER_TABLESPACE_MISSING, norm_name); + + /* This means we have no idea what happened to the tablespace + file, best to play it safe. */ + + no_tablespace = true; + } } if (!thd_tablespace_op(thd) && no_tablespace) { @@ -6517,27 +6528,33 @@ table_opened: /* If table has no talespace but it has crypt data, check is tablespace made unaccessible because encryption service or used key_id is not available. */ - if (ib_table) { - fil_space_crypt_t* crypt_data = ib_table->crypt_data; + if (encrypted) { + bool warning_pushed = false; + char buf[MAX_FULL_NAME_LEN]; + ut_format_name(ib_table->name, TRUE, buf, sizeof(buf)); - if (crypt_data && crypt_data->should_encrypt()) { + if (!encryption_key_id_exists(space()->crypt_data->key_id)) { + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + HA_ERR_DECRYPTION_FAILED, + "Table %s in file %s is encrypted but encryption service or" + " used key_id %u is not available. " + " Can't continue reading table.", + buf, space()->chain.start->name, + space()->crypt_data->key_id); + ret_err = HA_ERR_DECRYPTION_FAILED; + warning_pushed = true; + } - if (!encryption_key_id_exists(crypt_data->key_id)) { - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - HA_ERR_DECRYPTION_FAILED, - "Table %s is encrypted but encryption service or" - " used key_id %u is not available. " - " Can't continue reading table.", - ib_table->name, crypt_data->key_id); - ret_err = HA_ERR_DECRYPTION_FAILED; - } - } else if (ib_table->is_encrypted) { + /* If table is marked as encrypted then we push + warning if it has not been already done as used + key_id might be found but it is incorrect. */ + if (!warning_pushed) { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, HA_ERR_DECRYPTION_FAILED, - "Table %s is encrypted but encryption service or" + "Table %s in file %s is encrypted but encryption service or" " used key_id is not available. " " Can't continue reading table.", - ib_table->name); + buf, space()->chain.start->name); ret_err = HA_ERR_DECRYPTION_FAILED; } } @@ -6673,7 +6690,7 @@ table_opened: /* Only if the table has an AUTOINC column. */ if (prebuilt->table != NULL - && !prebuilt->table->ibd_file_missing + && prebuilt->table->is_readable() && table->found_next_number_field != NULL) { dict_table_autoinc_lock(prebuilt->table); @@ -8658,16 +8675,6 @@ ha_innobase::write_row( ha_statistic_increment(&SSV::ha_write_count); - if (share->ib_table != prebuilt->table) { - fprintf(stderr, - "InnoDB: Warning: share->ib_table %p prebuilt->table %p table %s is_corrupt %lu.", - share->ib_table, prebuilt->table, prebuilt->table->name, prebuilt->table->is_corrupt); - } - - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - sql_command = thd_sql_command(user_thd); if ((sql_command == SQLCOM_ALTER_TABLE @@ -9005,16 +9012,6 @@ wsrep_error: func_exit: innobase_active_small(); - if (share->ib_table != prebuilt->table) { - fprintf(stderr, - "InnoDB: Warning: share->ib_table %p prebuilt->table %p table %s is_corrupt %lu.", - share->ib_table, prebuilt->table, prebuilt->table->name, prebuilt->table->is_corrupt); - } - - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - DBUG_RETURN(error_result); } @@ -9425,16 +9422,6 @@ ha_innobase::update_row( ha_statistic_increment(&SSV::ha_update_count); - if (share->ib_table != prebuilt->table) { - fprintf(stderr, - "InnoDB: Warning: share->ib_table %p prebuilt->table %p table %s is_corrupt %lu.", - share->ib_table, prebuilt->table, prebuilt->table->name, prebuilt->table->is_corrupt); - } - - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - if (prebuilt->upd_node) { uvect = prebuilt->upd_node->update; } else { @@ -9540,16 +9527,6 @@ func_exit: wsrep_error: #endif /* WITH_WSREP */ - if (share->ib_table != prebuilt->table) { - fprintf(stderr, - "InnoDB: Warning: share->ib_table %p prebuilt->table %p table %s is_corrupt %lu.", - share->ib_table, prebuilt->table, prebuilt->table->name, prebuilt->table->is_corrupt); - } - - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - DBUG_RETURN(err); } @@ -9578,11 +9555,6 @@ ha_innobase::delete_row( ha_statistic_increment(&SSV::ha_delete_count); - if (UNIV_UNLIKELY(share && share->ib_table - && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - if (!prebuilt->upd_node) { row_get_prebuilt_update_vector(prebuilt); } @@ -9617,11 +9589,6 @@ ha_innobase::delete_row( wsrep_error: #endif /* WITH_WSREP */ - if (UNIV_UNLIKELY(share && share->ib_table - && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - DBUG_RETURN(convert_error_code_to_mysql( error, prebuilt->table->flags, user_thd)); } @@ -9872,17 +9839,13 @@ ha_innobase::index_read( ha_statistic_increment(&SSV::ha_read_key_count); - if (UNIV_UNLIKELY(srv_pass_corrupt_table <= 1 && share - && share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - index = prebuilt->index; if (UNIV_UNLIKELY(index == NULL) || dict_index_is_corrupted(index)) { prebuilt->index_usable = FALSE; DBUG_RETURN(HA_ERR_CRASHED); } + if (UNIV_UNLIKELY(!prebuilt->index_usable)) { DBUG_RETURN(dict_index_is_corrupted(index) ? HA_ERR_INDEX_CORRUPT @@ -9949,11 +9912,6 @@ ha_innobase::index_read( ret = DB_UNSUPPORTED; } - if (UNIV_UNLIKELY(srv_pass_corrupt_table <= 1 && share - && share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - switch (ret) { case DB_SUCCESS: error = 0; @@ -10113,11 +10071,6 @@ ha_innobase::change_active_index( { DBUG_ENTER("change_active_index"); - if (UNIV_UNLIKELY(srv_pass_corrupt_table <= 1 && share - && share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - ut_ad(user_thd == ha_thd()); ut_a(prebuilt->trx == thd_to_trx(user_thd)); @@ -10237,13 +10190,23 @@ ha_innobase::general_fetch( DBUG_RETURN(HA_ERR_END_OF_FILE); } - if (UNIV_UNLIKELY(srv_pass_corrupt_table <= 1 && share - && share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - ut_a(prebuilt->trx == thd_to_trx(user_thd)); + if (prebuilt->table->is_readable()) { + } else { + if (prebuilt->table->corrupted) { + DBUG_RETURN(HA_ERR_CRASHED); + } else { + FilSpace space(prebuilt->table->space, true); + + if (space()) { + DBUG_RETURN(HA_ERR_DECRYPTION_FAILED); + } else { + DBUG_RETURN(HA_ERR_NO_SUCH_TABLE); + } + } + } + innobase_srv_conc_enter_innodb(prebuilt->trx); ret = row_search_for_mysql( @@ -10251,11 +10214,6 @@ ha_innobase::general_fetch( innobase_srv_conc_exit_innodb(prebuilt->trx); - if (UNIV_UNLIKELY(srv_pass_corrupt_table <= 1 && share - && share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - switch (ret) { case DB_SUCCESS: error = 0; @@ -12984,7 +12942,7 @@ ha_innobase::discard_or_import_tablespace( user may want to set the DISCARD flag in order to IMPORT a new tablespace. */ - if (dict_table->ibd_file_missing) { + if (!dict_table->is_readable()) { ib_senderrf( prebuilt->trx->mysql_thd, IB_LOG_LEVEL_WARN, ER_TABLESPACE_MISSING, @@ -12994,7 +12952,7 @@ ha_innobase::discard_or_import_tablespace( err = row_discard_tablespace_for_mysql( dict_table->name, prebuilt->trx); - } else if (!dict_table->ibd_file_missing) { + } else if (dict_table->is_readable()) { /* Commit the transaction in order to release the table lock. */ trx_commit_for_mysql(prebuilt->trx); @@ -13071,15 +13029,7 @@ ha_innobase::truncate() update_thd(ha_thd()); - if (share->ib_table != prebuilt->table) { - fprintf(stderr, - "InnoDB: Warning: share->ib_table %p prebuilt->table %p table %s is_corrupt %lu.", - share->ib_table, prebuilt->table, prebuilt->table->name, prebuilt->table->is_corrupt); - } - - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } + DBUG_ASSERT(share->ib_table == prebuilt->table); if (UNIV_UNLIKELY(prebuilt->trx->fake_changes)) { DBUG_RETURN(HA_ERR_WRONG_COMMAND); @@ -13092,16 +13042,6 @@ ha_innobase::truncate() err = row_truncate_table_for_mysql(prebuilt->table, prebuilt->trx); - if (share->ib_table != prebuilt->table) { - fprintf(stderr, - "InnoDB: Warning: share->ib_table %p prebuilt->table %p table %s is_corrupt %lu.", - share->ib_table, prebuilt->table, prebuilt->table->name, prebuilt->table->is_corrupt); - } - - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - switch (err) { case DB_TABLESPACE_DELETED: @@ -14502,15 +14442,7 @@ ha_innobase::analyze( { int ret; - if (share->ib_table != prebuilt->table) { - fprintf(stderr, - "InnoDB: Warning: share->ib_table %p prebuilt->table %p table %s is_corrupt %lu.", - share->ib_table, prebuilt->table, prebuilt->table->name, prebuilt->table->is_corrupt); - } - - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt)) { - return(HA_ADMIN_CORRUPT); - } + DBUG_ASSERT(share->ib_table == prebuilt->table); /* Simply call this->info_low() with all the flags and request recalculation of the statistics */ @@ -14518,15 +14450,7 @@ ha_innobase::analyze( HA_STATUS_TIME | HA_STATUS_CONST | HA_STATUS_VARIABLE, true /* this is ANALYZE */); - if (share->ib_table != prebuilt->table) { - fprintf(stderr, - "InnoDB: Warning: share->ib_table %p prebuilt->table %p table %s is_corrupt %lu.", - share->ib_table, prebuilt->table, prebuilt->table->name, prebuilt->table->is_corrupt); - } - - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt)) { - return(HA_ADMIN_CORRUPT); - } + DBUG_ASSERT(share->ib_table == prebuilt->table); if (ret != 0) { return(HA_ADMIN_FAILED); @@ -14630,7 +14554,8 @@ ha_innobase::check( DBUG_RETURN(HA_ADMIN_CORRUPT); - } else if (prebuilt->table->ibd_file_missing) { + } else if (!prebuilt->table->is_readable() && + fil_space_get(prebuilt->table->space) == NULL) { ib_senderrf( thd, IB_LOG_LEVEL_ERROR, @@ -14704,6 +14629,7 @@ ha_innobase::check( server_mutex, srv_fatal_semaphore_wait_threshold, SRV_SEMAPHORE_WAIT_EXTENSION); + dberr_t err = btr_validate_index(index, prebuilt->trx); /* Restore the fatal lock wait timeout after @@ -14802,6 +14728,7 @@ ha_innobase::check( if (!(index->type & DICT_CLUSTERED)) { check_result = false; }); + if (!check_result) { innobase_format_name( index_name, sizeof index_name, @@ -15831,15 +15758,7 @@ ha_innobase::transactional_table_lock( update_thd(thd); - if (share->ib_table != prebuilt->table) { - fprintf(stderr, - "InnoDB: Warning: share->ib_table %p prebuilt->table %p table %s is_corrupt %lu.", - share->ib_table, prebuilt->table, prebuilt->table->name, prebuilt->table->is_corrupt); - } - - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } + DBUG_ASSERT(share->ib_table == prebuilt->table); if (!thd_tablespace_op(thd)) { @@ -15850,7 +15769,7 @@ ha_innobase::transactional_table_lock( ER_TABLESPACE_DISCARDED, table->s->table_name.str); - } else if (prebuilt->table->ibd_file_missing) { + } else if (!prebuilt->table->is_readable()) { ib_senderrf( thd, IB_LOG_LEVEL_ERROR, diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc index a521db3ce25..898db72be2e 100644 --- a/storage/xtradb/handler/handler0alter.cc +++ b/storage/xtradb/handler/handler0alter.cc @@ -3137,7 +3137,7 @@ prepare_inplace_alter_table_dict( clustered index of the old table, later. */ if (new_clustered || !ctx->online - || user_table->ibd_file_missing + || !user_table->is_readable() || dict_table_is_discarded(user_table)) { /* No need to allocate a modification log. */ ut_ad(!ctx->add_index[a]->online_log); @@ -3557,6 +3557,43 @@ ha_innobase::prepare_inplace_alter_table( goto func_exit; } + indexed_table = prebuilt->table; + + if (indexed_table->is_readable()) { + } else { + if (indexed_table->corrupted) { + /* Handled below */ + } else { + FilSpace space(indexed_table->space, true); + + if (space()) { + String str; + const char* engine= table_type(); + char buf[MAX_FULL_NAME_LEN]; + ut_format_name(indexed_table->name, TRUE, buf, sizeof(buf)); + + push_warning_printf(user_thd, Sql_condition::WARN_LEVEL_WARN, + HA_ERR_DECRYPTION_FAILED, + "Table %s in file %s is encrypted but encryption service or" + " used key_id is not available. " + " Can't continue reading table.", + buf, space()->chain.start->name); + + my_error(ER_GET_ERRMSG, MYF(0), HA_ERR_DECRYPTION_FAILED, str.c_ptr(), engine); + DBUG_RETURN(true); + } + } + } + + if (indexed_table->corrupted + || dict_table_get_first_index(indexed_table) == NULL + || dict_index_is_corrupted( + dict_table_get_first_index(indexed_table))) { + /* The clustered index is corrupted. */ + my_error(ER_CHECK_NO_SUCH_TABLE, MYF(0)); + DBUG_RETURN(true); + } + if (ha_alter_info->handler_flags & Alter_inplace_info::CHANGE_CREATE_OPTION) { /* Check engine specific table options */ @@ -4209,7 +4246,7 @@ ok_exit: DBUG_ASSERT(ctx->trx); DBUG_ASSERT(ctx->prebuilt == prebuilt); - if (prebuilt->table->ibd_file_missing + if (prebuilt->table->file_unreadable || dict_table_is_discarded(prebuilt->table)) { goto all_done; } @@ -5231,7 +5268,7 @@ commit_try_rebuild( /* The new table must inherit the flag from the "parent" table. */ if (dict_table_is_discarded(user_table)) { - rebuilt_table->ibd_file_missing = true; + rebuilt_table->file_unreadable = true; rebuilt_table->flags2 |= DICT_TF2_DISCARDED; } @@ -5763,9 +5800,9 @@ ha_innobase::commit_inplace_alter_table( if (ha_alter_info->group_commit_ctx) { ctx_array = ha_alter_info->group_commit_ctx; } else { - ctx_single[0] = ctx0; - ctx_single[1] = NULL; - ctx_array = ctx_single; + ctx_single[0] = ctx0; + ctx_single[1] = NULL; + ctx_array = ctx_single; } DBUG_ASSERT(ctx0 == ctx_array[0]); @@ -5794,6 +5831,19 @@ ha_innobase::commit_inplace_alter_table( = static_cast<ha_innobase_inplace_ctx*>(*pctx); DBUG_ASSERT(ctx->prebuilt->trx == prebuilt->trx); + /* If decryption failed for old table or new table + fail here. */ + if ((ctx->old_table->file_unreadable && + fil_space_get(ctx->old_table->space) != NULL)|| + (ctx->new_table->file_unreadable && + fil_space_get(ctx->new_table->space) != NULL)) { + String str; + const char* engine= table_type(); + get_error_message(HA_ERR_DECRYPTION_FAILED, &str); + my_error(ER_GET_ERRMSG, MYF(0), HA_ERR_DECRYPTION_FAILED, str.c_ptr(), engine); + DBUG_RETURN(true); + } + /* Exclusively lock the table, to ensure that no other transaction is holding locks on the table while we change the table definition. The MySQL meta-data lock diff --git a/storage/xtradb/include/btr0btr.ic b/storage/xtradb/include/btr0btr.ic index 62a24873482..0f5f025d6a3 100644 --- a/storage/xtradb/include/btr0btr.ic +++ b/storage/xtradb/include/btr0btr.ic @@ -61,7 +61,7 @@ btr_block_get_func( if (err == DB_DECRYPTION_FAILED) { if (index && index->table) { - index->table->is_encrypted = true; + index->table->file_unreadable = true; } } diff --git a/storage/xtradb/include/buf0buf.h b/storage/xtradb/include/buf0buf.h index 4a632e2345f..08c3a765a8b 100644 --- a/storage/xtradb/include/buf0buf.h +++ b/storage/xtradb/include/buf0buf.h @@ -690,17 +690,6 @@ buf_page_is_corrupted( const fil_space_t* space) MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** -Check if page is maybe compressed, encrypted or both when we encounter -corrupted page. Note that we can't be 100% sure if page is corrupted -or decrypt/decompress just failed. -@param[in] bpage Page -@return true if page corrupted, false if not */ -bool -buf_page_check_corrupt( - buf_page_t* bpage) /*!< in/out: buffer page read from disk */ - MY_ATTRIBUTE(( warn_unused_result)); - -/********************************************************************//** Checks if a page is all zeroes. @return TRUE if the page is all zeroes */ bool @@ -1273,12 +1262,15 @@ buf_page_init_for_read( /********************************************************************//** Completes an asynchronous read or write request of a file page to or from the buffer pool. -@return true if successful */ +@param[in,out] bpage pointer to the block in question +@return DB_SUCCESS if page has been read and is not corrupted, +DB_PAGE_CORRUPTED if page based on checksum check is corrupted, +DB_DECRYPTION_FAILED if page post encryption checksum matches but +after decryption normal page checksum does not match.*/ UNIV_INTERN -bool +dberr_t buf_page_io_complete( -/*=================*/ - buf_page_t* bpage); /*!< in: pointer to the block in question */ + buf_page_t* bpage); /********************************************************************//** Calculates a folded value of a file page address to use in the page hash table. @@ -1680,7 +1672,6 @@ struct buf_page_t{ if written again we check is TRIM operation needed. */ - unsigned key_version; /*!< key version for this block */ bool encrypted; /*!< page is still encrypted */ ulint real_size; /*!< Real size of the page diff --git a/storage/xtradb/include/buf0rea.h b/storage/xtradb/include/buf0rea.h index f0652b5d2cd..ab73108a71e 100644 --- a/storage/xtradb/include/buf0rea.h +++ b/storage/xtradb/include/buf0rea.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -35,29 +35,37 @@ High-level function which reads a page asynchronously from a file to the buffer buf_pool if it is not already there. Sets the io_fix flag and sets an exclusive lock on the buffer frame. The flag is cleared and the x-lock released by the i/o-handler thread. -@return TRUE if page has been read in, FALSE in case of failure */ + +@param[in] space space_id +@param[in] zip_size compressed page size in bytes, or 0 +@param[in] offset page number +@param[in] trx transaction +@return DB_SUCCESS if page has been read and is not corrupted, +@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted, +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but +after decryption normal page checksum does not match. +@retval DB_TABLESPACE_DELETED if tablespace .ibd file is missing */ UNIV_INTERN -ibool +dberr_t buf_read_page( -/*==========*/ - ulint space, /*!< in: space id */ - ulint zip_size, /*!< in: compressed page size in bytes, or 0 */ - ulint offset, /*!< in: page number */ - trx_t* trx, /*!< in: trx */ - buf_page_t** bpage /*!< out: page */ -); + ulint space, + ulint zip_size, + ulint offset, + trx_t* trx); + /********************************************************************//** High-level function which reads a page asynchronously from a file to the buffer buf_pool if it is not already there. Sets the io_fix flag and sets an exclusive lock on the buffer frame. The flag is cleared and the x-lock released by the i/o-handler thread. -@return TRUE if page has been read in, FALSE in case of failure */ +@param[in] space Tablespace id +@param[in] offset Page number */ UNIV_INTERN -ibool +void buf_read_page_async( -/*================*/ - ulint space, /*!< in: space id */ - ulint offset);/*!< in: page number */ + ulint space, + ulint offset); + /********************************************************************//** Applies a random read-ahead in buf_pool if there are at least a threshold value of accessed pages from the random read-ahead area. Does not read any diff --git a/storage/xtradb/include/db0err.h b/storage/xtradb/include/db0err.h index b11c5f4ea1a..8bd3beda110 100644 --- a/storage/xtradb/include/db0err.h +++ b/storage/xtradb/include/db0err.h @@ -138,6 +138,8 @@ enum dberr_t { of missing key management plugin, or missing or incorrect key or incorret AES method or algorithm. */ + DB_PAGE_CORRUPTED, /* Page read from tablespace is + corrupted. */ /* The following are partial failure codes */ DB_FAIL = 1000, DB_OVERFLOW, diff --git a/storage/xtradb/include/dict0dict.h b/storage/xtradb/include/dict0dict.h index 1622b927a76..6da8eb892d9 100644 --- a/storage/xtradb/include/dict0dict.h +++ b/storage/xtradb/include/dict0dict.h @@ -1767,16 +1767,6 @@ dict_close(void); /*============*/ #ifndef UNIV_HOTBACKUP /**********************************************************************//** -Check whether the table is corrupted. -@return nonzero for corrupted table, zero for valid tables */ -UNIV_INLINE -ulint -dict_table_is_corrupted( -/*====================*/ - const dict_table_t* table) /*!< in: table */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); - -/**********************************************************************//** Check whether the index is corrupted. @return nonzero for corrupted index, zero for valid indexes */ UNIV_INLINE @@ -1820,6 +1810,15 @@ dict_set_corrupted_by_space( /*========================*/ ulint space_id); /*!< in: space ID */ +/**********************************************************************//** +Flags a table with specified space_id encrypted in the data dictionary +cache +@param[in] space_id Tablespace id */ +UNIV_INTERN +void +dict_set_encrypted_by_space( + ulint space_id); + /********************************************************************//** Validate the table flags. @return true if valid. */ @@ -1900,14 +1899,6 @@ dict_table_get_index_on_first_col( in table */ #endif /* !UNIV_HOTBACKUP */ -/************************************************************************* -set is_corrupt flag by space_id*/ - -void -dict_table_set_corrupt_by_space( -/*============================*/ - ulint space_id, - ibool need_mutex); #ifndef UNIV_NONINL #include "dict0dict.ic" diff --git a/storage/xtradb/include/dict0dict.ic b/storage/xtradb/include/dict0dict.ic index e3de7a33123..4ed1afc8094 100644 --- a/storage/xtradb/include/dict0dict.ic +++ b/storage/xtradb/include/dict0dict.ic @@ -1538,21 +1538,6 @@ dict_max_field_len_store_undo( } /********************************************************************//** -Check whether the table is corrupted. -@return nonzero for corrupted table, zero for valid tables */ -UNIV_INLINE -ulint -dict_table_is_corrupted( -/*====================*/ - const dict_table_t* table) /*!< in: table */ -{ - ut_ad(table); - ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); - - return(table->corrupted); -} - -/********************************************************************//** Check whether the index is corrupted. @return nonzero for corrupted index, zero for valid indexes */ UNIV_INLINE @@ -1564,7 +1549,7 @@ dict_index_is_corrupted( ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return((index->type & DICT_CORRUPT) - || (index->table && index->table->corrupted)); + || (index->table && index->table->corrupted)); } /********************************************************************//** diff --git a/storage/xtradb/include/dict0mem.h b/storage/xtradb/include/dict0mem.h index 96c85cd8a99..a32581a0e90 100644 --- a/storage/xtradb/include/dict0mem.h +++ b/storage/xtradb/include/dict0mem.h @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -795,6 +795,9 @@ struct dict_index_t{ to first_blob_page_no; protected by blobs_mutex; @see btr_blob_dbg_t */ #endif /* UNIV_BLOB_DEBUG */ + + bool is_readable() const; + #ifdef UNIV_DEBUG ulint magic_n;/*!< magic number */ /** Value of dict_index_t::magic_n */ @@ -1062,11 +1065,13 @@ struct dict_table_t{ table is placed */ unsigned flags:DICT_TF_BITS; /*!< DICT_TF_... */ unsigned flags2:DICT_TF2_BITS; /*!< DICT_TF2_... */ - unsigned ibd_file_missing:1; - /*!< TRUE if this is in a single-table - tablespace and the .ibd file is missing; then - we must return in ha_innodb.cc an error if the - user tries to query such an orphaned table */ + unsigned file_unreadable:1; + /*!< true if this is in a single-table + tablespace and the .ibd file is missing + or page decryption failed and page is + corrupted; then we must return in + ha_innodb.cc an error if the + user tries to query such table */ unsigned cached:1;/*!< TRUE if the table object has been added to the dictionary cache */ unsigned to_be_dropped:1; @@ -1364,10 +1369,19 @@ struct dict_table_t{ UT_LIST_BASE_NODE_T(lock_t) locks; /*!< list of locks on the table; protected by lock_sys->mutex */ - ibool is_corrupt; - ibool is_encrypted; + #endif /* !UNIV_HOTBACKUP */ + /* Returns true if this is a single-table tablespace + and the .ibd file is missing or page decryption failed + and/or page is corrupted. + @return true if table is readable + @retval false if table is not readable */ + inline bool is_readable() const + { + return(UNIV_LIKELY(!file_unreadable)); + } + #ifdef UNIV_DEBUG ulint magic_n;/*!< magic number */ /** Value of dict_table_t::magic_n */ @@ -1375,6 +1389,16 @@ struct dict_table_t{ #endif /* UNIV_DEBUG */ }; +/* Returns true if this is a single-table tablespace +and the .ibd file is missing or page decryption failed +and/or page is corrupted. +@return true if table is readable +@retval false if table is not readable */ +inline bool dict_index_t::is_readable() const +{ + return(UNIV_LIKELY(!table->file_unreadable)); +} + /** A function object to add the foreign key constraint to the referenced set of the referenced table, if it exists in the dictionary cache. */ struct dict_foreign_add_to_referenced_table { diff --git a/storage/xtradb/include/fil0crypt.h b/storage/xtradb/include/fil0crypt.h index e7e9676aa3a..228dfb895fe 100644 --- a/storage/xtradb/include/fil0crypt.h +++ b/storage/xtradb/include/fil0crypt.h @@ -117,8 +117,7 @@ struct fil_space_crypt_t : st_encryption_scheme min_key_version(new_min_key_version), page0_offset(0), encryption(new_encryption), - mutex(), - key_found(new_min_key_version), + key_found(0), rotate_state() { key_id = new_key_id; @@ -136,6 +135,8 @@ struct fil_space_crypt_t : st_encryption_scheme type = CRYPT_SCHEME_1; min_key_version = key_get_latest_version(); } + + key_found = min_key_version; } /** Destructor */ @@ -298,13 +299,15 @@ Parse a MLOG_FILE_WRITE_CRYPT_DATA log entry @param[in] ptr Log entry start @param[in] end_ptr Log entry end @param[in] block buffer block +@param[out] err DB_SUCCESS or DB_DECRYPTION_FAILED @return position on log buffer */ UNIV_INTERN -const byte* +byte* fil_parse_write_crypt_data( - const byte* ptr, + byte* ptr, const byte* end_ptr, - const buf_block_t* block) + const buf_block_t* block, + dberr_t* err) MY_ATTRIBUTE((warn_unused_result)); /****************************************************************** diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index d73a68d9d34..0f7526a8e5e 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -650,23 +650,45 @@ fil_write_flushed_lsn_to_data_files( Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID +@param[in] silent whether to silently ignore missing tablespaces @param[in] for_io whether to look up the tablespace while performing I/O (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted or truncated */ +UNIV_INTERN fil_space_t* -fil_space_acquire(ulint id, bool for_io = false) +fil_space_acquire_low(ulint id, bool silent, bool for_io = false) MY_ATTRIBUTE((warn_unused_result)); +/** Acquire a tablespace when it could be dropped concurrently. +Used by background threads that do not necessarily hold proper locks +for concurrency control. +@param[in] id tablespace ID +@param[in] for_io whether to look up the tablespace while performing I/O + (possibly executing TRUNCATE) +@return the tablespace +@retval NULL if missing or being deleted or truncated */ +inline +fil_space_t* +fil_space_acquire(ulint id, bool for_io = false) +{ + return (fil_space_acquire_low(id, false, for_io)); +} + /** Acquire a tablespace that may not exist. Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID +@param[in] for_io whether to look up the tablespace while performing I/O + (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted */ +inline fil_space_t* -fil_space_acquire_silent(ulint id) - MY_ATTRIBUTE((warn_unused_result)); +fil_space_acquire_silent(ulint id, bool for_io = false) +{ + return (fil_space_acquire_low(id, true, for_io)); +} /** Release a tablespace acquired with fil_space_acquire(). @param[in,out] space tablespace to release */ @@ -699,6 +721,64 @@ fil_space_keyrotate_next( fil_space_t* prev_space) MY_ATTRIBUTE((warn_unused_result)); +/** Wrapper with reference-counting for a fil_space_t. */ +class FilSpace +{ +public: + /** Default constructor: Use this when reference counting + is done outside this wrapper. */ + FilSpace() : m_space(NULL) {} + + /** Constructor: Look up the tablespace and increment the + reference count if found. + @param[in] space_id tablespace ID + @param[in] silent whether not print any errors + @param[in] for_io whether to look up the tablespace + while performing I/O + (possibly executing TRUNCATE) */ + explicit FilSpace(ulint space_id, bool silent = false, bool for_io = false) + : m_space(fil_space_acquire_low(space_id, silent, for_io)) {} + + /** Assignment operator: This assumes that fil_space_acquire() + has already been done for the fil_space_t. The caller must + assign NULL if it calls fil_space_release(). + @param[in] space tablespace to assign */ + class FilSpace& operator=(fil_space_t* space) + { + /* fil_space_acquire() must have been invoked. */ + ut_ad(space == NULL || space->n_pending_ops > 0); + m_space = space; + return(*this); + } + + /** Destructor - Decrement the reference count if a fil_space_t + is still assigned. */ + ~FilSpace() + { + if (m_space != NULL) { + fil_space_release(m_space); + } + } + + /** Implicit type conversion + @return the wrapped object */ + operator const fil_space_t*() const + { + return(m_space); + } + + /** Explicit type conversion + @return the wrapped object */ + const fil_space_t* operator()() const + { + return(m_space); + } + +private: + /** The wrapped pointer */ + fil_space_t* m_space; +}; + /*******************************************************************//** Reads the flushed lsn, arch no, and tablespace flag fields from a data file at database startup. @@ -721,22 +801,6 @@ fil_read_first_page( fil_space_crypt_t** crypt_data) /*!< out: crypt data */ __attribute__((warn_unused_result)); -/*******************************************************************//** -Increments the count of pending operation, if space is not being deleted. -@return TRUE if being deleted, and operation should be skipped */ -UNIV_INTERN -ibool -fil_inc_pending_ops( -/*================*/ - ulint id, /*!< in: space id */ - ibool print_err); /*!< in: need to print error or not */ -/*******************************************************************//** -Decrements the count of pending operations. */ -UNIV_INTERN -void -fil_decr_pending_ops( -/*=================*/ - ulint id); /*!< in: space id */ #endif /* !UNIV_HOTBACKUP */ /*******************************************************************//** Parses the body of a log record written about an .ibd file operation. That is, @@ -1428,36 +1492,6 @@ fil_space_get_by_id( /*================*/ ulint id); /*!< in: space id */ -/****************************************************************** -Get id of first tablespace or ULINT_UNDEFINED if none */ -UNIV_INTERN -ulint -fil_get_first_space(); -/*=================*/ - -/****************************************************************** -Get id of next tablespace or ULINT_UNDEFINED if none */ -UNIV_INTERN -ulint -fil_get_next_space( -/*===============*/ - ulint id); /*!< in: space id */ - -/****************************************************************** -Get id of first tablespace that has node or ULINT_UNDEFINED if none */ -UNIV_INTERN -ulint -fil_get_first_space_safe(); -/*======================*/ - -/****************************************************************** -Get id of next tablespace that has node or ULINT_UNDEFINED if none */ -UNIV_INTERN -ulint -fil_get_next_space_safe( -/*====================*/ - ulint id); /*!< in: previous space id */ - #endif /* UNIV_INNOCHECKSUM */ /****************************************************************//** diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index 120f1432ccf..d6d31b0c572 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -1383,7 +1383,12 @@ recv_parse_or_apply_log_rec_body( } break; case MLOG_FILE_WRITE_CRYPT_DATA: - ptr = const_cast<byte*>(fil_parse_write_crypt_data(ptr, end_ptr, block)); + dberr_t err; + ptr = const_cast<byte*>(fil_parse_write_crypt_data(ptr, end_ptr, block, &err)); + + if (err != DB_SUCCESS) { + recv_sys->found_corrupt_log = TRUE; + } break; default: ptr = NULL; @@ -1868,6 +1873,11 @@ recv_apply_hashed_log_recs(bool last_batch) break; } + if (recv_sys->found_corrupt_log) { + mutex_exit(&recv_sys->mutex); + return; + } + mutex_exit(&recv_sys->mutex); os_thread_sleep(500000); } @@ -1932,6 +1942,10 @@ recv_apply_hashed_log_recs(bool last_batch) mutex_exit(&(recv_sys->mutex)); + if (recv_sys->found_corrupt_log) { + return; + } + os_thread_sleep(500000); mutex_enter(&(recv_sys->mutex)); diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index baa7bcbea09..81d6fda9e53 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -2030,15 +2030,12 @@ PageConverter::validate( buf_block_t* block) UNIV_NOTHROW { buf_frame_t* page = get_frame(block); - ulint space_id = mach_read_from_4( - page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - fil_space_t* space = fil_space_found_by_id(space_id); /* Check that the page number corresponds to the offset in the file. Flag as corrupt if it doesn't. Disable the check for LSN in buf_page_is_corrupted() */ - if (buf_page_is_corrupted(false, page, get_zip_size(), space) + if (buf_page_is_corrupted(false, page, get_zip_size(), NULL) || (page_get_page_no(page) != offset / m_page_size && page_get_page_no(page) != 0)) { @@ -2201,7 +2198,7 @@ row_import_discard_changes( index->space = FIL_NULL; } - table->ibd_file_missing = TRUE; + table->file_unreadable = true; fil_close_tablespace(trx, table->space); } @@ -3447,7 +3444,7 @@ row_import_for_mysql( ut_a(table->space); ut_ad(prebuilt->trx); - ut_a(table->ibd_file_missing); + ut_a(table->file_unreadable); trx_start_if_not_started(prebuilt->trx); @@ -3752,7 +3749,7 @@ row_import_for_mysql( return(row_import_error(prebuilt, trx, err)); } - table->ibd_file_missing = false; + table->file_unreadable = false; table->flags2 &= ~DICT_TF2_DISCARDED; if (autoinc != 0) { diff --git a/storage/xtradb/row/row0ins.cc b/storage/xtradb/row/row0ins.cc index 8dbf093a17d..f4f96d32c50 100644 --- a/storage/xtradb/row/row0ins.cc +++ b/storage/xtradb/row/row0ins.cc @@ -1518,7 +1518,7 @@ run_again: } if (check_table == NULL - || check_table->ibd_file_missing + || check_table->file_unreadable || check_index == NULL) { if (!srv_read_only_mode && check_ref) { @@ -2410,8 +2410,7 @@ row_ins_clust_index_entry_low( &cursor, 0, __FILE__, __LINE__, &mtr); if (err != DB_SUCCESS) { - index->table->is_encrypted = true; - index->table->ibd_file_missing = true; + index->table->file_unreadable = true; mtr_commit(&mtr); goto func_exit; } @@ -2771,7 +2770,7 @@ row_ins_sec_index_entry_low( " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } goto func_exit; } diff --git a/storage/xtradb/row/row0log.cc b/storage/xtradb/row/row0log.cc index a17ee405720..666b59b42db 100644 --- a/storage/xtradb/row/row0log.cc +++ b/storage/xtradb/row/row0log.cc @@ -3674,7 +3674,7 @@ row_log_apply( rw_lock_x_lock(dict_index_get_lock(index)); - if (!dict_table_is_corrupted(index->table)) { + if (!index->table->corrupted) { error = row_log_apply_ops(trx, index, &dup); } else { error = DB_SUCCESS; diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index 24cd9687a47..57b08801225 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -1620,7 +1620,8 @@ row_merge_read_clustered_index( page_cur_t* cur = btr_pcur_get_page_cur(&pcur); /* Do not continue if table pages are still encrypted */ - if (old_table->is_encrypted || new_table->is_encrypted) { + if (!old_table->is_readable() || + !new_table->is_readable()) { err = DB_DECRYPTION_FAILED; trx->error_key_num = 0; goto func_exit; @@ -3695,7 +3696,7 @@ row_merge_rename_tables_dict( table is in a non-system tablespace where space > 0. */ if (err == DB_SUCCESS && old_table->space != TRX_SYS_SPACE - && !old_table->ibd_file_missing) { + && fil_space_get(old_table->space) != NULL) { /* Make pathname to update SYS_DATAFILES. */ char* tmp_path = row_make_new_pathname(old_table, tmp_name); @@ -4005,6 +4006,7 @@ row_merge_build_indexes( os_mem_alloc_large(&block_size)); if (crypt_block == NULL) { + fil_space_release(space); DBUG_RETURN(DB_OUT_OF_MEMORY); } } else { @@ -4070,13 +4072,15 @@ row_merge_build_indexes( pct_cost = COST_READ_CLUSTERED_INDEX * 100 / (total_static_cost + total_dynamic_cost); /* Do not continue if we can't encrypt table pages */ - if (old_table->is_encrypted || new_table->is_encrypted) { + if (!old_table->is_readable() || + !new_table->is_readable()) { error = DB_DECRYPTION_FAILED; ib_push_warning(trx->mysql_thd, DB_DECRYPTION_FAILED, "Table %s is encrypted but encryption service or" " used key_id is not available. " " Can't continue reading table.", - old_table->is_encrypted ? old_table->name : new_table->name); + !old_table->is_readable() ? old_table->name : + new_table->name); goto func_exit; } diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index 7c8636d354f..a4141c36eda 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -663,6 +663,7 @@ handle_new_error: abort(); case DB_CORRUPTION: + case DB_PAGE_CORRUPTED: fputs("InnoDB: We detected index corruption" " in an InnoDB type table.\n" "InnoDB: You have to dump + drop + reimport" @@ -1280,6 +1281,71 @@ run_again: } /*********************************************************************//** +Determine is tablespace encrypted but decryption failed, is table corrupted +or is tablespace .ibd file missing. +@param[in] table Table +@param[in] trx Transaction +@param[in] push_warning true if we should push warning to user +@return DB_DECRYPTION_FAILED table is encrypted but decryption failed +DB_CORRUPTION table is corrupted +DB_TABLESPACE_NOT_FOUND tablespace .ibd file not found */ +static +dberr_t +row_mysql_get_table_status( + const dict_table_t* table, + trx_t* trx, + bool push_warning = true) +{ + dberr_t err = DB_SUCCESS; + FilSpace space(table->space, true); + char buf[MAX_FULL_NAME_LEN]; + ut_format_name(table->name, TRUE, buf, sizeof(buf)); + + if (space()) { + + if (space()->crypt_data && space()->crypt_data->is_encrypted()) { + // maybe we cannot access the table due to failing + // to decrypt + if (push_warning) { + ib_push_warning(trx,HA_ERR_DECRYPTION_FAILED, + "Table %s in file %s is encrypted but encryption service or" + " used key_id %u is not available. " + " Can't continue reading table.", + buf, space()->chain.start->name, + space()->crypt_data->key_id); + } + + err = DB_DECRYPTION_FAILED; + } else { + if (push_warning) { + ib_push_warning(trx, DB_CORRUPTION, + "Table %s in file %s corrupted.", + buf, space()->chain.start->name); + } + + err = DB_CORRUPTION; + } + + } else { + ib_logf(IB_LOG_LEVEL_ERROR, + "InnoDB: MySQL is trying to use a table handle" + " but the .ibd file for" + " table %s does not exist." + " Have you deleted the .ibd file" + " from the database directory under" + " the MySQL datadir, or have you" + " used DISCARD TABLESPACE?" + " Look from " REFMAN "innodb-troubleshooting.html" + " how you can resolve the problem.", + buf); + + err = DB_TABLESPACE_NOT_FOUND; + } + + return (err); +} + +/*********************************************************************//** Does an insert for MySQL. @return error code or DB_SUCCESS */ UNIV_INTERN @@ -1308,20 +1374,8 @@ row_insert_for_mysql( return(DB_TABLESPACE_DELETED); - } else if (prebuilt->table->ibd_file_missing) { - - ib_logf(IB_LOG_LEVEL_ERROR, - ".ibd file is missing for table %s", - prebuilt->table->name); - - return(DB_TABLESPACE_NOT_FOUND); - } else if (prebuilt->table->is_encrypted) { - ib_push_warning(trx, DB_DECRYPTION_FAILED, - "Table %s in tablespace %lu encrypted." - "However key management plugin or used key_id is not found or" - " used encryption algorithm or method does not match.", - prebuilt->table->name, prebuilt->table->space); - return(DB_DECRYPTION_FAILED); + } else if (!prebuilt->table->is_readable()) { + return (row_mysql_get_table_status(prebuilt->table, trx, true)); } else if (prebuilt->magic_n != ROW_PREBUILT_ALLOCATED) { fprintf(stderr, "InnoDB: Error: trying to free a corrupt\n" @@ -1712,28 +1766,8 @@ row_update_for_mysql( ut_ad(trx != NULL); UT_NOT_USED(mysql_rec); - if (prebuilt->table->ibd_file_missing) { - ut_print_timestamp(stderr); - fprintf(stderr, " InnoDB: Error:\n" - "InnoDB: MySQL is trying to use a table handle" - " but the .ibd file for\n" - "InnoDB: table %s does not exist.\n" - "InnoDB: Have you deleted the .ibd file" - " from the database directory under\n" - "InnoDB: the MySQL datadir, or have you" - " used DISCARD TABLESPACE?\n" - "InnoDB: Look from\n" - "InnoDB: " REFMAN "innodb-troubleshooting.html\n" - "InnoDB: how you can resolve the problem.\n", - prebuilt->table->name); - return(DB_ERROR); - } else if (prebuilt->table->is_encrypted) { - ib_push_warning(trx, DB_DECRYPTION_FAILED, - "Table %s in tablespace %lu encrypted." - "However key management plugin or used key_id is not found or" - " used encryption algorithm or method does not match.", - prebuilt->table->name, prebuilt->table->space); - return (DB_TABLE_NOT_FOUND); + if (!table->is_readable()) { + return (row_mysql_get_table_status(table, trx, true)); } if (UNIV_UNLIKELY(prebuilt->magic_n != ROW_PREBUILT_ALLOCATED)) { @@ -3128,7 +3162,7 @@ row_discard_tablespace( /* All persistent operations successful, update the data dictionary memory cache. */ - table->ibd_file_missing = TRUE; + table->file_unreadable = true; table->flags2 |= DICT_TF2_DISCARDED; @@ -3185,8 +3219,6 @@ row_discard_tablespace_for_mysql( if (table == 0) { err = DB_TABLE_NOT_FOUND; - } else if (table->is_encrypted) { - err = DB_DECRYPTION_FAILED; } else if (table->space == TRX_SYS_SPACE) { char table_name[MAX_FULL_NAME_LEN + 1]; @@ -3401,10 +3433,8 @@ row_truncate_table_for_mysql( if (dict_table_is_discarded(table)) { return(DB_TABLESPACE_DELETED); - } else if (table->is_encrypted) { - return(DB_DECRYPTION_FAILED); - } else if (table->ibd_file_missing) { - return(DB_TABLESPACE_NOT_FOUND); + } else if (!table->is_readable()) { + return (row_mysql_get_table_status(table, trx, true)); } trx_start_for_ddl(trx, TRX_DICT_OP_TABLE); @@ -3570,7 +3600,7 @@ row_truncate_table_for_mysql( "create a new tablespace", table->name); - table->ibd_file_missing = 1; + table->file_unreadable = true; err = DB_ERROR; goto funct_exit; } @@ -3994,19 +4024,6 @@ row_drop_table_for_mysql( goto funct_exit; } - /* If table is encrypted and table page encryption failed - return error. */ - if (table->is_encrypted) { - - if (table->can_be_evicted) { - dict_table_move_from_lru_to_non_lru(table); - } - - dict_table_close(table, TRUE, FALSE); - err = DB_DECRYPTION_FAILED; - goto funct_exit; - } - /* Turn on this drop bit before we could release the dictionary latch */ table->to_be_dropped = true; @@ -4378,7 +4395,7 @@ row_drop_table_for_mysql( from table->heap, which will be freed by dict_table_remove_from_cache(table) below. */ space_id = table->space; - ibd_file_missing = table->ibd_file_missing; + ibd_file_missing = table->file_unreadable; table_flags = table->flags; is_temp = DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY); @@ -4389,7 +4406,8 @@ row_drop_table_for_mysql( have a temp flag but not know the temp path */ ut_a(table->dir_path_of_temp_table == NULL || is_temp); if (dict_table_is_discarded(table) - || table->ibd_file_missing) { + || (!table->is_readable() + && fil_space_get(table->space) == NULL)) { /* Do not attempt to drop known-to-be-missing tablespaces. */ space_id = 0; @@ -4801,7 +4819,8 @@ loop: "'%s.frm' was lost.", table->name); } - if (table->ibd_file_missing) { + if (!table->is_readable() + && fil_space_get(table->space) == NULL) { ib_logf(IB_LOG_LEVEL_WARN, "Missing %s.ibd file for table %s.", table->name, table->name); @@ -5075,7 +5094,8 @@ row_rename_table_for_mysql( stderr); goto funct_exit; - } else if (table->ibd_file_missing + } else if (!table->is_readable() + && fil_space_get(table->space) == NULL && !dict_table_is_discarded(table)) { err = DB_TABLE_NOT_FOUND; @@ -5145,7 +5165,7 @@ row_rename_table_for_mysql( which have space IDs > 0. */ if (err == DB_SUCCESS && table->space != TRX_SYS_SPACE - && !table->ibd_file_missing) { + && table->is_readable()) { /* Make a new pathname to update SYS_DATAFILES. */ char* new_path = row_make_new_pathname(table, new_name); diff --git a/storage/xtradb/row/row0purge.cc b/storage/xtradb/row/row0purge.cc index 35b3520749b..8a1dbd6f69f 100644 --- a/storage/xtradb/row/row0purge.cc +++ b/storage/xtradb/row/row0purge.cc @@ -777,7 +777,7 @@ row_purge_parse_undo_rec( goto err_exit; } - if (node->table->ibd_file_missing) { + if (node->table->file_unreadable) { /* We skip purge of missing .ibd files */ dict_table_close(node->table, FALSE, FALSE); diff --git a/storage/xtradb/row/row0sel.cc b/storage/xtradb/row/row0sel.cc index 29569bb31e7..9a583e2023a 100644 --- a/storage/xtradb/row/row0sel.cc +++ b/storage/xtradb/row/row0sel.cc @@ -62,6 +62,7 @@ Created 12/19/1997 Heikki Tuuri #include "srv0start.h" #include "m_string.h" /* for my_sys.h */ #include "my_sys.h" /* DEBUG_SYNC_C */ +#include "fil0fil.h" #include "my_compare.h" /* enum icp_result */ @@ -3728,13 +3729,12 @@ row_search_for_mysql( return(DB_TABLESPACE_DELETED); - } else if (prebuilt->table->ibd_file_missing) { - - return(DB_TABLESPACE_NOT_FOUND); - - } else if (prebuilt->table->is_encrypted) { - - return(DB_DECRYPTION_FAILED); + } else if (!prebuilt->table->is_readable()) { + if (fil_space_get(prebuilt->table->space) == NULL) { + return(DB_TABLESPACE_NOT_FOUND); + } else { + return(DB_DECRYPTION_FAILED); + } } else if (!prebuilt->index_usable) { return(DB_MISSING_HISTORY); @@ -4193,7 +4193,7 @@ wait_table_again: " used key_id is not available. " " Can't continue reading table.", prebuilt->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } rec = NULL; goto lock_wait_or_error; @@ -4213,7 +4213,7 @@ rec_loop: rec = btr_pcur_get_rec(pcur); - if (!rec) { + if (!index->table->is_readable()) { err = DB_DECRYPTION_FAILED; goto lock_wait_or_error; } @@ -4305,7 +4305,7 @@ rec_loop: wrong_offs: if (srv_pass_corrupt_table && index->table->space != 0 && index->table->space < SRV_LOG_SPACE_FIRST_ID) { - index->table->is_corrupt = TRUE; + index->table->file_unreadable = TRUE; fil_space_set_corrupt(index->table->space); } @@ -4366,20 +4366,22 @@ wrong_offs: offsets = rec_get_offsets(rec, index, offsets, ULINT_UNDEFINED, &heap); if (UNIV_UNLIKELY(srv_force_recovery > 0 - || (index->table->is_corrupt && + || (!index->table->is_readable() && srv_pass_corrupt_table == 2))) { if (!rec_validate(rec, offsets) || !btr_index_rec_validate(rec, index, FALSE)) { - fprintf(stderr, - "InnoDB: Index corruption: rec offs %lu" - " next offs %lu, page no %lu,\n" - "InnoDB: ", - (ulong) page_offset(rec), - (ulong) next_offs, - (ulong) page_get_page_no(page_align(rec))); - dict_index_name_print(stderr, trx, index); - fputs(". We try to skip the record.\n", - stderr); + char buf[MAX_FULL_NAME_LEN]; + ut_format_name(index->table->name, FALSE, buf, sizeof(buf)); + + ib_logf(IB_LOG_LEVEL_ERROR, + "Index %s corrupted: rec offs " ULINTPF + " next offs " ULINTPF + ", page no " ULINTPF " ." + " We try to skip the record.", + buf, + page_offset(rec), + next_offs, + page_get_page_no(page_align(rec))); goto next_rec; } diff --git a/storage/xtradb/row/row0uins.cc b/storage/xtradb/row/row0uins.cc index 651042fb820..f14a4ef9bcf 100644 --- a/storage/xtradb/row/row0uins.cc +++ b/storage/xtradb/row/row0uins.cc @@ -320,7 +320,7 @@ row_undo_ins_parse_undo_rec( /* Skip the UNDO if we can't find the table or the .ibd file. */ if (UNIV_UNLIKELY(node->table == NULL)) { - } else if (UNIV_UNLIKELY(node->table->ibd_file_missing)) { + } else if (UNIV_UNLIKELY(node->table->file_unreadable)) { close_table: dict_table_close(node->table, dict_locked, FALSE); node->table = NULL; diff --git a/storage/xtradb/row/row0umod.cc b/storage/xtradb/row/row0umod.cc index 19576d976bd..8deba4f00a5 100644 --- a/storage/xtradb/row/row0umod.cc +++ b/storage/xtradb/row/row0umod.cc @@ -1068,7 +1068,7 @@ row_undo_mod_parse_undo_rec( return; } - if (node->table->ibd_file_missing) { + if (node->table->file_unreadable) { dict_table_close(node->table, dict_locked, FALSE); /* We skip undo operations to missing .ibd files */ diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index 5255a7454ea..c5c594c82a7 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -2531,6 +2531,11 @@ files_checked: recv_group_scan_log_recs(). */ recv_apply_hashed_log_recs(true); + + if (recv_sys->found_corrupt_log) { + return (DB_CORRUPTION); + } + DBUG_PRINT("ib_log", ("apply completed")); } diff --git a/storage/xtradb/trx/trx0trx.cc b/storage/xtradb/trx/trx0trx.cc index 92d7525ea84..ed80bafa6c6 100644 --- a/storage/xtradb/trx/trx0trx.cc +++ b/storage/xtradb/trx/trx0trx.cc @@ -661,7 +661,7 @@ trx_resurrect_table_locks( i != tables.end(); i++) { if (dict_table_t* table = dict_table_open_on_id( *i, FALSE, DICT_TABLE_OP_LOAD_TABLESPACE)) { - if (table->ibd_file_missing + if (table->file_unreadable || dict_table_is_temporary(table)) { mutex_enter(&dict_sys->mutex); dict_table_close(table, TRUE, FALSE); diff --git a/storage/xtradb/ut/ut0ut.cc b/storage/xtradb/ut/ut0ut.cc index acedb56879a..fd52537ae11 100644 --- a/storage/xtradb/ut/ut0ut.cc +++ b/storage/xtradb/ut/ut0ut.cc @@ -852,6 +852,8 @@ ut_strerr( return("BLOB record length is greater than 10%% of redo log"); case DB_DECRYPTION_FAILED: return("Table is encrypted but decrypt failed."); + case DB_PAGE_CORRUPTED: + return("Page read from tablespace is corrupted."); /* do not add default: in order to produce a warning if new code is added to the enum but not added here */ |