diff options
author | Jan Lindström <jan.lindstrom@mariadb.com> | 2016-09-22 16:32:26 +0300 |
---|---|---|
committer | Jan Lindström <jan.lindstrom@mariadb.com> | 2016-09-22 16:38:24 +0300 |
commit | 2bedc3978b90bf5abe1029df393c63ced1849bed (patch) | |
tree | 1afa5fe2f562ae577566cf7a67e90fb110682c99 | |
parent | e387bfafbbb01ccfabeb2beb86efb199ca2ca3ac (diff) | |
download | mariadb-git-2bedc3978b90bf5abe1029df393c63ced1849bed.tar.gz |
MDEV-9931: InnoDB reads first page of every .ibd file at startup
Analysis: By design InnoDB was reading first page of every .ibd file
at startup to find out is tablespace encrypted or not. This is
because tablespace could have been encrypted always, not
encrypted newer or encrypted based on configuration and this
information can be find realible only from first page of .ibd file.
Fix: Do not read first page of every .ibd file at startup. Instead
whenever tablespace is first time accedded we will read the first
page to find necessary information about tablespace encryption
status.
TODO: Add support for SYS_TABLEOPTIONS where all table options
encryption information included will be stored.
40 files changed, 1173 insertions, 132 deletions
diff --git a/mysql-test/suite/encryption/r/innodb-bad-key-change.result b/mysql-test/suite/encryption/r/innodb-bad-key-change.result index cf9791887cc..ec8dee89230 100644 --- a/mysql-test/suite/encryption/r/innodb-bad-key-change.result +++ b/mysql-test/suite/encryption/r/innodb-bad-key-change.result @@ -36,10 +36,13 @@ SELECT * FROM t1; ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB SHOW WARNINGS; Level Code Message -Warning 1812 Tablespace is missing for table 'test/t1' -Warning 192 Table test/t1 is encrypted but encryption service or used key_id 2 is not available. Can't continue reading table. +Warning 192 Table test/t1 in tablespace 6 is encrypted but encryption service or used key_id is not available. Can't continue reading table. +Warning 192 Table test/t1 is encrypted but encryption service or used key_id is not available. Can't continue reading table. Error 1296 Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB DROP TABLE t1; +Warnings: +Warning 192 Table in tablespace 6 encrypted.However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match. Can't continue opening the table. +Warning 192 Table in tablespace 6 encrypted.However key management plugin or used key_id 1 is not found or used encryption algorithm or method does not match. Can't continue opening the table. # Start server with keys.txt CREATE TABLE t2 (c VARCHAR(8), id int not null primary key, b int, key(b)) ENGINE=InnoDB ENCRYPTED=YES; INSERT INTO t2 VALUES ('foobar',1,2); diff --git a/mysql-test/suite/encryption/r/innodb_encryption_row_compressed.result b/mysql-test/suite/encryption/r/innodb_encryption_row_compressed.result new file mode 100644 index 00000000000..355271ef9ca --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb_encryption_row_compressed.result @@ -0,0 +1,153 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +create table innodb_compressed1(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed encrypted=yes; +create table innodb_compressed2(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed key_block_size=1 encrypted=yes; +create table innodb_compressed3(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed key_block_size=2 encrypted=yes; +create table innodb_compressed4(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed key_block_size=4 encrypted=yes; +insert into innodb_compressed1 values (1, 20, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (2, 20, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (3, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (4, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (5, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (6, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (7, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (8, 20, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (9, 20, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (10, 20, 'private', 'evenmoreprivate'); +insert into innodb_compressed2 select * from innodb_compressed1; +insert into innodb_compressed3 select * from innodb_compressed1; +insert into innodb_compressed4 select * from innodb_compressed1; +# t1 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed1.ibd +# t2 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed2.ibd +# t3 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed3.ibd +# t4 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed4.ibd +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +select * from innodb_compressed1 where d = 20; +c1 d a b +1 20 private evenmoreprivate +2 20 private evenmoreprivate +8 20 private evenmoreprivate +9 20 private evenmoreprivate +10 20 private evenmoreprivate +select * from innodb_compressed1 where d = 30; +c1 d a b +3 30 private evenmoreprivate +4 30 private evenmoreprivate +5 30 private evenmoreprivate +6 30 private evenmoreprivate +7 30 private evenmoreprivate +select * from innodb_compressed2 where d = 20; +c1 d a b +1 20 private evenmoreprivate +2 20 private evenmoreprivate +8 20 private evenmoreprivate +9 20 private evenmoreprivate +10 20 private evenmoreprivate +select * from innodb_compressed2 where d = 30; +c1 d a b +3 30 private evenmoreprivate +4 30 private evenmoreprivate +5 30 private evenmoreprivate +6 30 private evenmoreprivate +7 30 private evenmoreprivate +select * from innodb_compressed3 where d = 20; +c1 d a b +1 20 private evenmoreprivate +2 20 private evenmoreprivate +8 20 private evenmoreprivate +9 20 private evenmoreprivate +10 20 private evenmoreprivate +select * from innodb_compressed3 where d = 30; +c1 d a b +3 30 private evenmoreprivate +4 30 private evenmoreprivate +5 30 private evenmoreprivate +6 30 private evenmoreprivate +7 30 private evenmoreprivate +select * from innodb_compressed4 where d = 20; +c1 d a b +1 20 private evenmoreprivate +2 20 private evenmoreprivate +8 20 private evenmoreprivate +9 20 private evenmoreprivate +10 20 private evenmoreprivate +select * from innodb_compressed4 where d = 30; +c1 d a b +3 30 private evenmoreprivate +4 30 private evenmoreprivate +5 30 private evenmoreprivate +6 30 private evenmoreprivate +7 30 private evenmoreprivate +update innodb_compressed1 set d = d + 10 where d = 30; +update innodb_compressed2 set d = d + 10 where d = 30; +update innodb_compressed3 set d = d + 10 where d = 30; +update innodb_compressed4 set d = d + 10 where d = 30; +insert into innodb_compressed1 values (20, 60, 'newprivate', 'newevenmoreprivate'); +insert into innodb_compressed2 values (20, 60, 'newprivate', 'newevenmoreprivate'); +insert into innodb_compressed3 values (20, 60, 'newprivate', 'newevenmoreprivate'); +insert into innodb_compressed4 values (20, 60, 'newprivate', 'newevenmoreprivate'); +# t1 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed1.ibd +# t2 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed2.ibd +# t3 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed3.ibd +# t4 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed4.ibd +select * from innodb_compressed1 where d = 40; +c1 d a b +3 40 private evenmoreprivate +4 40 private evenmoreprivate +5 40 private evenmoreprivate +6 40 private evenmoreprivate +7 40 private evenmoreprivate +select * from innodb_compressed1 where d = 60; +c1 d a b +20 60 newprivate newevenmoreprivate +select * from innodb_compressed2 where d = 40; +c1 d a b +3 40 private evenmoreprivate +4 40 private evenmoreprivate +5 40 private evenmoreprivate +6 40 private evenmoreprivate +7 40 private evenmoreprivate +select * from innodb_compressed2 where d = 60; +c1 d a b +20 60 newprivate newevenmoreprivate +select * from innodb_compressed3 where d = 40; +c1 d a b +3 40 private evenmoreprivate +4 40 private evenmoreprivate +5 40 private evenmoreprivate +6 40 private evenmoreprivate +7 40 private evenmoreprivate +select * from innodb_compressed3 where d = 60; +c1 d a b +20 60 newprivate newevenmoreprivate +select * from innodb_compressed4 where d = 40; +c1 d a b +3 40 private evenmoreprivate +4 40 private evenmoreprivate +5 40 private evenmoreprivate +6 40 private evenmoreprivate +7 40 private evenmoreprivate +select * from innodb_compressed4 where d = 60; +c1 d a b +20 60 newprivate newevenmoreprivate +# t1 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed1.ibd +# t2 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed2.ibd +# t3 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed3.ibd +# t4 yes on expecting NOT FOUND +NOT FOUND /private/ in innodb_compressed4.ibd +drop table innodb_compressed1; +drop table innodb_compressed2; +drop table innodb_compressed3; +drop table innodb_compressed4; diff --git a/mysql-test/suite/encryption/r/innodb_lotoftables.result b/mysql-test/suite/encryption/r/innodb_lotoftables.result new file mode 100644 index 00000000000..5e3eaef550f --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb_lotoftables.result @@ -0,0 +1,154 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +SHOW VARIABLES LIKE 'innodb_encrypt%'; +Variable_name Value +innodb_encrypt_log OFF +innodb_encrypt_tables OFF +innodb_encryption_rotate_key_age 1 +innodb_encryption_rotation_iops 100 +innodb_encryption_threads 0 +create database innodb_encrypted_1; +use innodb_encrypted_1; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 1 +set autocommit=0; +set autocommit=1; +commit work; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 1 +# should be 100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE NAME LIKE 'innodb_encrypted%'; +COUNT(*) +100 +create database innodb_encrypted_2; +use innodb_encrypted_2; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +set autocommit=0; +commit work; +set autocommit=1; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +# should be 100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +100 +# should be 100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +100 +create database innodb_encrypted_3; +use innodb_encrypted_3; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +set autocommit=0; +commit work; +set autocommit=1; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +# should be 100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +100 +# should be 200 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +200 +use test; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +200 +SET GLOBAL innodb_encrypt_tables = on; +SET GLOBAL innodb_encryption_threads=4; +# Wait until all encrypted tables have been encrypted +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +200 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +100 +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +# Success! +# Restart mysqld --innodb_encrypt_tables=0 --innodb_encryption_threads=0 +# Restart Success! +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +use test; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +use innodb_encrypted_1; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +use innodb_encrypted_2; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +use innodb_encrypted_3; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +use innodb_encrypted_1; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 3 +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 103 +use innodb_encrypted_2; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 103 +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 203 +use innodb_encrypted_3; +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 203 +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 203 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +200 +SET GLOBAL innodb_encrypt_tables = off; +SET GLOBAL innodb_encryption_threads=4; +# Wait until all default encrypted tables have been decrypted +# should be 100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +100 +# should be 200 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; +COUNT(*) +200 +show status like 'innodb_pages0_read%'; +Variable_name Value +Innodb_pages0_read 303 +use test; +drop database innodb_encrypted_1; +drop database innodb_encrypted_2; +drop database innodb_encrypted_3; diff --git a/mysql-test/suite/encryption/t/innodb_encryption_row_compressed.opt b/mysql-test/suite/encryption/t/innodb_encryption_row_compressed.opt new file mode 100644 index 00000000000..7ebf81a07f3 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb_encryption_row_compressed.opt @@ -0,0 +1,4 @@ +--innodb-encrypt-tables=ON +--innodb-encryption-rotate-key-age=15 +--innodb-encryption-threads=4 +--innodb-tablespaces-encryption diff --git a/mysql-test/suite/encryption/t/innodb_encryption_row_compressed.test b/mysql-test/suite/encryption/t/innodb_encryption_row_compressed.test new file mode 100644 index 00000000000..0a28c1690a2 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb_encryption_row_compressed.test @@ -0,0 +1,125 @@ +-- source include/have_innodb.inc +-- source include/have_file_key_management_plugin.inc +-- source include/not_embedded.inc + +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; +--enable_query_log + +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + +create table innodb_compressed1(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed encrypted=yes; +create table innodb_compressed2(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed key_block_size=1 encrypted=yes; +create table innodb_compressed3(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed key_block_size=2 encrypted=yes; +create table innodb_compressed4(c1 bigint not null primary key, d int, a varchar(20), b char(200)) engine=innodb row_format=compressed key_block_size=4 encrypted=yes; + +insert into innodb_compressed1 values (1, 20, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (2, 20, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (3, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (4, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (5, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (6, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (7, 30, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (8, 20, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (9, 20, 'private', 'evenmoreprivate'); +insert into innodb_compressed1 values (10, 20, 'private', 'evenmoreprivate'); + +insert into innodb_compressed2 select * from innodb_compressed1; +insert into innodb_compressed3 select * from innodb_compressed1; +insert into innodb_compressed4 select * from innodb_compressed1; + +--source include/restart_mysqld.inc + +--let $MYSQLD_DATADIR=`select @@datadir` +--let t1_IBD = $MYSQLD_DATADIR/test/innodb_compressed1.ibd +--let t2_IBD = $MYSQLD_DATADIR/test/innodb_compressed2.ibd +--let t3_IBD = $MYSQLD_DATADIR/test/innodb_compressed3.ibd +--let t4_IBD = $MYSQLD_DATADIR/test/innodb_compressed4.ibd +--let SEARCH_RANGE = 10000000 +--let SEARCH_PATTERN=private +--echo # t1 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t1_IBD +-- source include/search_pattern_in_file.inc +--echo # t2 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t2_IBD +-- source include/search_pattern_in_file.inc +--echo # t3 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t3_IBD +-- source include/search_pattern_in_file.inc +--echo # t4 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t4_IBD +-- source include/search_pattern_in_file.inc + +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + +select * from innodb_compressed1 where d = 20; +select * from innodb_compressed1 where d = 30; +select * from innodb_compressed2 where d = 20; +select * from innodb_compressed2 where d = 30; +select * from innodb_compressed3 where d = 20; +select * from innodb_compressed3 where d = 30; +select * from innodb_compressed4 where d = 20; +select * from innodb_compressed4 where d = 30; + +update innodb_compressed1 set d = d + 10 where d = 30; +update innodb_compressed2 set d = d + 10 where d = 30; +update innodb_compressed3 set d = d + 10 where d = 30; +update innodb_compressed4 set d = d + 10 where d = 30; + +insert into innodb_compressed1 values (20, 60, 'newprivate', 'newevenmoreprivate'); +insert into innodb_compressed2 values (20, 60, 'newprivate', 'newevenmoreprivate'); +insert into innodb_compressed3 values (20, 60, 'newprivate', 'newevenmoreprivate'); +insert into innodb_compressed4 values (20, 60, 'newprivate', 'newevenmoreprivate'); + +--let SEARCH_PATTERN=private +--echo # t1 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t1_IBD +-- source include/search_pattern_in_file.inc +--echo # t2 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t2_IBD +-- source include/search_pattern_in_file.inc +--echo # t3 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t3_IBD +-- source include/search_pattern_in_file.inc +--echo # t4 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t4_IBD +-- source include/search_pattern_in_file.inc + +--source include/restart_mysqld.inc + +select * from innodb_compressed1 where d = 40; +select * from innodb_compressed1 where d = 60; +select * from innodb_compressed2 where d = 40; +select * from innodb_compressed2 where d = 60; +select * from innodb_compressed3 where d = 40; +select * from innodb_compressed3 where d = 60; +select * from innodb_compressed4 where d = 40; +select * from innodb_compressed4 where d = 60; + +--let SEARCH_PATTERN=private +--echo # t1 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t1_IBD +-- source include/search_pattern_in_file.inc +--echo # t2 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t2_IBD +-- source include/search_pattern_in_file.inc +--echo # t3 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t3_IBD +-- source include/search_pattern_in_file.inc +--echo # t4 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t4_IBD +-- source include/search_pattern_in_file.inc + +drop table innodb_compressed1; +drop table innodb_compressed2; +drop table innodb_compressed3; +drop table innodb_compressed4; + +# reset system +--disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; +--enable_query_log diff --git a/mysql-test/suite/encryption/t/innodb_lotoftables.opt b/mysql-test/suite/encryption/t/innodb_lotoftables.opt new file mode 100644 index 00000000000..ffb5a2957f8 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb_lotoftables.opt @@ -0,0 +1,3 @@ +--innodb-tablespaces-encryption +--innodb-encrypt-tables=off +--innodb-encryption-threads=0 diff --git a/mysql-test/suite/encryption/t/innodb_lotoftables.test b/mysql-test/suite/encryption/t/innodb_lotoftables.test new file mode 100644 index 00000000000..cad3cb54326 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb_lotoftables.test @@ -0,0 +1,274 @@ +-- source include/have_innodb.inc +-- source include/have_example_key_management_plugin.inc +-- source include/big_test.inc + +# embedded does not support restart +-- source include/not_embedded.inc + +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; +let $innodb_encryption_threads_orig = `SELECT @@global.innodb_encryption_threads`; +--enable_query_log + +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; + +SHOW VARIABLES LIKE 'innodb_encrypt%'; + +# +# This will create 100 tables where that could be +# encrypted an unencrypt +# +create database innodb_encrypted_1; +use innodb_encrypted_1; +show status like 'innodb_pages0_read%'; +set autocommit=0; +let $tables = 100; + +--disable_query_log +while ($tables) +{ + eval create table t_$tables (a int not null primary key, b varchar(200)) engine=innodb; + commit; + let $rows = 100; + while($rows) + { + eval insert into t_$tables values ($rows, substring(MD5(RAND()), -64)); + dec $rows; + } + commit; + dec $tables; +} +--enable_query_log + +set autocommit=1; +commit work; +show status like 'innodb_pages0_read%'; +# +# Verify +# +--echo # should be 100 + +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE NAME LIKE 'innodb_encrypted%'; + +# +# This will create 100 tables that are encrypted always +# +create database innodb_encrypted_2; +use innodb_encrypted_2; +show status like 'innodb_pages0_read%'; +set autocommit=0; + +--disable_query_log +let $tables = 100; +while ($tables) +{ + eval create table t_$tables (a int not null primary key, b varchar(200)) engine=innodb encrypted=yes; + commit; + let $rows = 100; + while($rows) + { + eval insert into t_$tables values ($rows, substring(MD5(RAND()), -64)); + dec $rows; + } + commit; + dec $tables; +} +--enable_query_log + +commit work; +set autocommit=1; +show status like 'innodb_pages0_read%'; +# +# Verify +# +--echo # should be 100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +--echo # should be 100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; + +# +# This will create 100 tables that are not encrypted +# +create database innodb_encrypted_3; +use innodb_encrypted_3; +show status like 'innodb_pages0_read%'; +set autocommit=0; + +--disable_query_log +let $tables = 100; +while ($tables) +{ + eval create table t_$tables (a int not null primary key, b varchar(200)) engine=innodb encrypted=no; + commit; + let $rows = 100; + while($rows) + { + eval insert into t_$tables values ($rows, substring(MD5(RAND()), -64)); + dec $rows; + } + commit; + dec $tables; +} +--enable_query_log + +commit work; +set autocommit=1; +show status like 'innodb_pages0_read%'; +# +# Verify +# +--echo # should be 100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +--echo # should be 200 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; + +use test; +show status like 'innodb_pages0_read%'; + +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; + +SET GLOBAL innodb_encrypt_tables = on; +SET GLOBAL innodb_encryption_threads=4; + +--echo # Wait until all encrypted tables have been encrypted +let $cnt=600; +while ($cnt) +{ + let $success=`SELECT COUNT(*) = 100 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0`; + if ($success) + { + let $cnt=0; + } + if (!$success) + { + real_sleep 1; + dec $cnt; + } +} +if (!$success) +{ + SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0; + SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0; + SHOW STATUS LIKE 'innodb_encryption%'; + -- die Timeout waiting for encryption threads +} + +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; +show status like 'innodb_pages0_read%'; + +--echo # Success! +--echo # Restart mysqld --innodb_encrypt_tables=0 --innodb_encryption_threads=0 +-- let $restart_parameters=--innodb_encrypt_tables=0 --innodb_encryption_threads=0 +-- source include/restart_mysqld.inc + +--echo # Restart Success! +show status like 'innodb_pages0_read%'; + +show status like 'innodb_pages0_read%'; +use test; +show status like 'innodb_pages0_read%'; +use innodb_encrypted_1; +show status like 'innodb_pages0_read%'; +use innodb_encrypted_2; +show status like 'innodb_pages0_read%'; +use innodb_encrypted_3; +show status like 'innodb_pages0_read%'; + +use innodb_encrypted_1; +show status like 'innodb_pages0_read%'; +--disable_result_log +--disable_query_log +let $tables = 100; +while ($tables) +{ + eval select * from t_$tables; + dec $tables; +} +--enable_query_log +--enable_result_log + +show status like 'innodb_pages0_read%'; + +use innodb_encrypted_2; +show status like 'innodb_pages0_read%'; + +--disable_result_log +--disable_query_log +let $tables = 100; +while ($tables) +{ + eval select * from t_$tables; + dec $tables; +} +--enable_query_log +--enable_result_log + +show status like 'innodb_pages0_read%'; + +use innodb_encrypted_3; +show status like 'innodb_pages0_read%'; +--disable_result_log +--disable_query_log +let $tables = 100; +while ($tables) +{ + eval select * from t_$tables; + dec $tables; +} +--enable_query_log +--enable_result_log + +show status like 'innodb_pages0_read%'; + +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; + +SET GLOBAL innodb_encrypt_tables = off; +SET GLOBAL innodb_encryption_threads=4; + +--echo # Wait until all default encrypted tables have been decrypted +let $cnt=600; +while ($cnt) +{ + let $success=`SELECT COUNT(*) = 100 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0`; + if ($success) + { + let $cnt=0; + } + if (!$success) + { + real_sleep 1; + dec $cnt; + } +} +if (!$success) +{ + SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0; + SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0; + SHOW STATUS LIKE 'innodb_encryption%'; + -- die Timeout waiting for encryption threads +} + +--echo # should be 100 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND NAME LIKE 'innodb_encrypted%'; +--echo # should be 200 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; +show status like 'innodb_pages0_read%'; + +# +# Cleanup +# +use test; +drop database innodb_encrypted_1; +drop database innodb_encrypted_2; +drop database innodb_encrypted_3; + +--disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; +EVAL SET GLOBAL innodb_encryption_threads = $innodb_encryption_threads_orig; +--enable_query_log diff --git a/mysql-test/suite/sys_vars/r/innodb_monitor_disable_basic.result b/mysql-test/suite/sys_vars/r/innodb_monitor_disable_basic.result index 6c7051dc3d0..f7a02ed3548 100644 --- a/mysql-test/suite/sys_vars/r/innodb_monitor_disable_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_monitor_disable_basic.result @@ -40,6 +40,7 @@ buffer_pages_written disabled buffer_index_pages_written disabled buffer_non_index_pages_written disabled buffer_pages_read disabled +buffer_pages0_read disabled buffer_index_sec_rec_cluster_reads disabled buffer_index_sec_rec_cluster_reads_avoided disabled buffer_data_reads disabled diff --git a/mysql-test/suite/sys_vars/r/innodb_monitor_enable_basic.result b/mysql-test/suite/sys_vars/r/innodb_monitor_enable_basic.result index 6c7051dc3d0..f7a02ed3548 100644 --- a/mysql-test/suite/sys_vars/r/innodb_monitor_enable_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_monitor_enable_basic.result @@ -40,6 +40,7 @@ buffer_pages_written disabled buffer_index_pages_written disabled buffer_non_index_pages_written disabled buffer_pages_read disabled +buffer_pages0_read disabled buffer_index_sec_rec_cluster_reads disabled buffer_index_sec_rec_cluster_reads_avoided disabled buffer_data_reads disabled diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index 33ca57c9654..470825fa246 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -2,7 +2,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2014, 2015, MariaDB Corporation +Copyright (c) 2014, 2016, MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -737,14 +737,16 @@ btr_root_block_get( block = btr_block_get(space, zip_size, root_page_no, mode, (dict_index_t*)index, mtr); if (!block) { - index->table->is_encrypted = TRUE; - index->table->corrupted = FALSE; - - ib_push_warning(index->table->thd, DB_DECRYPTION_FAILED, - "Table %s in tablespace %lu is encrypted but encryption service or" - " used key_id is not available. " - " Can't continue reading table.", - index->table->name, space); + if (index && index->table) { + index->table->is_encrypted = TRUE; + index->table->corrupted = FALSE; + + ib_push_warning(index->table->thd, DB_DECRYPTION_FAILED, + "Table %s in tablespace %lu is encrypted but encryption service or" + " used key_id is not available. " + " Can't continue reading table.", + index->table->name, space); + } return NULL; } @@ -1819,6 +1821,12 @@ leaf_loop: root = btr_page_get(space, zip_size, root_page_no, RW_X_LATCH, NULL, &mtr); + + if (!root) { + mtr_commit(&mtr); + return; + } + #ifdef UNIV_BTR_DEBUG ut_a(btr_root_fseg_validate(FIL_PAGE_DATA + PAGE_BTR_SEG_LEAF + root, space)); @@ -1875,15 +1883,17 @@ btr_free_root( block = btr_block_get(space, zip_size, root_page_no, RW_X_LATCH, NULL, mtr); - btr_search_drop_page_hash_index(block); + if (block) { + btr_search_drop_page_hash_index(block); - header = buf_block_get_frame(block) + PAGE_HEADER + PAGE_BTR_SEG_TOP; + header = buf_block_get_frame(block) + PAGE_HEADER + PAGE_BTR_SEG_TOP; #ifdef UNIV_BTR_DEBUG - ut_a(btr_root_fseg_validate(header, space)); + ut_a(btr_root_fseg_validate(header, space)); #endif /* UNIV_BTR_DEBUG */ - while (!fseg_free_step(header, mtr)) { - /* Free the entire segment in small steps. */ + while (!fseg_free_step(header, mtr)) { + /* Free the entire segment in small steps. */ + } } } #endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index 3c72efdb6f3..8baab7ad536 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -1154,11 +1154,14 @@ loop: space_id, name); } - /* We need to read page 0 to get (optional) IV - regardless if encryptions is turned on or not, - since if it's off we should decrypt a potentially - already encrypted table */ - bool read_page_0 = true; + /* We could read page 0 to get (optional) IV + if encryption is turned on, if it's off + we will read the page 0 later and find out + if we should decrypt a potentially + already encrypted table + bool read_page_0 = srv_encrypt_tables; */ + + bool read_page_0 = false; /* We set the 2nd param (fix_dict = true) here because we already have an x-lock on diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index ceffa950739..bedaf7cfe49 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -258,20 +258,6 @@ fil_space_read_crypt_data( } if (memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) { -#ifdef UNIV_DEBUG - ib_logf(IB_LOG_LEVEL_WARN, - "Found potentially bogus bytes on " - "page 0 offset %lu for space %lu : " - "[ %.2x %.2x %.2x %.2x %.2x %.2x ]. " - "Assuming space is not encrypted!.", - offset, space, - page[offset + 0], - page[offset + 1], - page[offset + 2], - page[offset + 3], - page[offset + 4], - page[offset + 5]); -#endif /* Crypt data is not stored. */ return NULL; } @@ -666,6 +652,61 @@ fil_space_encrypt( byte* tmp = fil_encrypt_buf(crypt_data, space, offset, lsn, src_frame, zip_size, dst_frame); +#ifdef UNIV_DEBUG + if (tmp) { + /* Verify that encrypted buffer is not corrupted */ + byte* tmp_mem = (byte *)malloc(UNIV_PAGE_SIZE); + dberr_t err = DB_SUCCESS; + byte* src = src_frame; + bool page_compressed_encrypted = (mach_read_from_2(tmp+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); + byte* comp_mem = NULL; + byte* uncomp_mem = NULL; + ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; + + if (page_compressed_encrypted) { + comp_mem = (byte *)malloc(UNIV_PAGE_SIZE); + uncomp_mem = (byte *)malloc(UNIV_PAGE_SIZE); + memcpy(comp_mem, src_frame, UNIV_PAGE_SIZE); + fil_decompress_page(uncomp_mem, comp_mem, UNIV_PAGE_SIZE, NULL); + src = uncomp_mem; + } + + bool corrupted1 = buf_page_is_corrupted(true, src, zip_size); + bool ok = fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err); + + /* Need to decompress the page if it was also compressed */ + if (page_compressed_encrypted) { + memcpy(comp_mem, tmp_mem, UNIV_PAGE_SIZE); + fil_decompress_page(tmp_mem, comp_mem, UNIV_PAGE_SIZE, NULL); + } + + bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size); + bool different = memcmp(src, tmp_mem, size); + + if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) { + fprintf(stderr, "JAN: ok %d corrupted %d corrupted1 %d err %d different %d\n", ok , corrupted, corrupted1, err, different); + fprintf(stderr, "JAN1: src_frame\n"); + buf_page_print(src_frame, zip_size, BUF_PAGE_PRINT_NO_CRASH); + fprintf(stderr, "JAN2: encrypted_frame\n"); + buf_page_print(tmp, zip_size, BUF_PAGE_PRINT_NO_CRASH); + fprintf(stderr, "JAN1: decrypted_frame\n"); + buf_page_print(tmp_mem, zip_size, BUF_PAGE_PRINT_NO_CRASH); + ut_error; + } + + free(tmp_mem); + + if (comp_mem) { + free(comp_mem); + } + + if (uncomp_mem) { + free(uncomp_mem); + } + } + +#endif /* UNIV_DEBUG */ + return tmp; } @@ -2426,7 +2467,8 @@ UNIV_INTERN void fil_space_crypt_mark_space_closing( /*===============================*/ - ulint space) /*!< in: Space id */ + ulint space, /*!< in: tablespace id */ + fil_space_crypt_t* crypt_data) /*!< in: crypt_data or NULL */ { if (!fil_crypt_threads_inited) { return; @@ -2434,7 +2476,9 @@ fil_space_crypt_mark_space_closing( mutex_enter(&fil_crypt_threads_mutex); - fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); + if (!crypt_data) { + crypt_data = fil_space_get_crypt_data(space); + } if (crypt_data == NULL) { mutex_exit(&fil_crypt_threads_mutex); diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index b92ac02da10..43aecf45b60 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1156,7 +1156,8 @@ fil_space_create( ulint id, /*!< in: space id */ ulint flags, /*!< in: tablespace flags */ ulint purpose,/*!< in: FIL_TABLESPACE, or FIL_LOG if log */ - fil_space_crypt_t* crypt_data) /*!< in: crypt data */ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + bool create_table) /*!< in: true if create table */ { fil_space_t* space; @@ -1240,6 +1241,21 @@ fil_space_create( space->magic_n = FIL_SPACE_MAGIC_N; space->printed_compression_failure = false; + space->crypt_data = crypt_data; + + /* In create table we write page 0 so we have already + "read" it and for system tablespaces we have read + crypt data at startup. */ + if (create_table || crypt_data != NULL) { + space->page_0_crypt_read = true; + } + + ib_logf(IB_LOG_LEVEL_INFO, + "Created tablespace for space %lu name %s key_id %u encryption %d\n", + space->id, + space->name, + space->crypt_data ? space->crypt_data->key_id : 0, + space->crypt_data ? space->crypt_data->encryption : 0); rw_lock_create(fil_space_latch_key, &space->latch, SYNC_FSP); @@ -1251,7 +1267,6 @@ fil_space_create( UT_LIST_ADD_LAST(space_list, fil_system->space_list, space); - space->crypt_data = crypt_data; mutex_exit(&fil_system->mutex); @@ -2020,6 +2035,8 @@ fil_read_first_page( os_file_read(data_file, page, 0, UNIV_PAGE_SIZE); + srv_stats.page0_read.add(1); + /* The FSP_HEADER on page 0 is only valid for the first file in a tablespace. So if this is not the first datafile, leave *flags and *space_id as they were read from the first file and @@ -2039,6 +2056,7 @@ fil_read_first_page( ulint space = fsp_header_get_space_id(page); ulint offset = fsp_header_get_crypt_offset( fsp_flags_get_zip_size(*flags), NULL); + cdata = fil_space_read_crypt_data(space, page, offset); if (crypt_data) { @@ -3595,7 +3613,7 @@ fil_create_new_single_table_tablespace( } success = fil_space_create(tablename, space_id, flags, FIL_TABLESPACE, - crypt_data); + crypt_data, true); if (!success || !fil_node_create(path, size, space_id, FALSE)) { err = DB_ERROR; @@ -3832,6 +3850,7 @@ fil_open_single_table_tablespace( if (table) { table->crypt_data = def.crypt_data; + table->page_0_read = true; } /* Validate this single-table-tablespace with SYS_TABLES, @@ -3871,6 +3890,7 @@ fil_open_single_table_tablespace( if (table) { table->crypt_data = remote.crypt_data; + table->page_0_read = true; } /* Validate this single-table-tablespace with SYS_TABLES, @@ -3910,6 +3930,7 @@ fil_open_single_table_tablespace( if (table) { table->crypt_data = dict.crypt_data; + table->page_0_read = true; } /* Validate this single-table-tablespace with SYS_TABLES, @@ -4081,7 +4102,7 @@ skip_validate: if (err != DB_SUCCESS) { ; // Don't load the tablespace into the cache } else if (!fil_space_create(tablename, id, flags, FIL_TABLESPACE, - crypt_data)) { + crypt_data, false)) { err = DB_ERROR; } else { /* We do not measure the size of the file, that is why @@ -4698,7 +4719,7 @@ will_not_choose: #endif /* UNIV_HOTBACKUP */ ibool file_space_create_success = fil_space_create( tablename, fsp->id, fsp->flags, FIL_TABLESPACE, - fsp->crypt_data); + fsp->crypt_data, false); if (!file_space_create_success) { if (srv_force_recovery > 0) { @@ -7224,7 +7245,46 @@ fil_space_get_crypt_data( space = fil_space_get_by_id(id); if (space != NULL) { + /* If we have not yet read the page0 + of this tablespace we will do it now. */ + if (!space->crypt_data && !space->page_0_crypt_read) { + ulint flags; + ulint space_id; + lsn_t min_flushed_lsn; + lsn_t max_flushed_lsn; + fil_node_t* node; + + ut_a(space->crypt_data == NULL); + node = UT_LIST_GET_FIRST(space->chain); + + fil_node_prepare_for_io(node, fil_system, space); + + const char* msg = fil_read_first_page(node->handle, + false, + &flags, + &space_id, + &min_flushed_lsn, + &max_flushed_lsn, + &space->crypt_data); + + fil_node_complete_io(node, fil_system, OS_FILE_READ); + + ib_logf(IB_LOG_LEVEL_INFO, + "Read page 0 from tablespace for space %lu name %s key_id %u encryption %d handle %d\n", + space_id, + space->name, + space->crypt_data ? space->crypt_data->key_id : 0, + space->crypt_data ? space->crypt_data->encryption : 0, + node->handle); + + ut_a(space->id == space_id); + + space->page_0_crypt_read = true; + } + crypt_data = space->crypt_data; + + ut_ad(space->page_0_crypt_read); } mutex_exit(&fil_system->mutex); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 0eaf8d35463..fd15092d96c 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -943,6 +943,8 @@ static SHOW_VAR innodb_status_variables[]= { (char*) &export_vars.innodb_pages_created, SHOW_LONG}, {"pages_read", (char*) &export_vars.innodb_pages_read, SHOW_LONG}, + {"pages0_read", + (char*) &export_vars.innodb_page0_read, SHOW_LONG}, {"pages_written", (char*) &export_vars.innodb_pages_written, SHOW_LONG}, {"row_lock_current_waits", diff --git a/storage/innobase/include/btr0btr.ic b/storage/innobase/include/btr0btr.ic index 64b3d5a0975..e9410310213 100644 --- a/storage/innobase/include/btr0btr.ic +++ b/storage/innobase/include/btr0btr.ic @@ -60,7 +60,9 @@ btr_block_get_func( NULL, BUF_GET, file, line, mtr, &err); if (err == DB_DECRYPTION_FAILED) { - index->table->is_encrypted = true; + if (index && index->table) { + index->table->is_encrypted = true; + } } if (block) { diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index f964447fb8f..f2d72de39e5 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -1030,6 +1030,8 @@ struct dict_table_t{ mem_heap_t* heap; /*!< memory heap */ char* name; /*!< table name */ void* thd; /*!< thd */ + bool page_0_read; /*!< true if page 0 has + been already read */ fil_space_crypt_t *crypt_data; /*!< crypt data if present */ const char* dir_path_of_temp_table;/*!< NULL or the directory path where a TEMPORARY table that was explicitly diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index fdc413e7520..8ffa4e2073f 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -316,7 +316,8 @@ UNIV_INTERN void fil_space_crypt_mark_space_closing( /*===============================*/ - ulint space); /*!< in: tablespace id */ + ulint space, /*!< in: tablespace id */ + fil_space_crypt_t* crypt_data); /*!< in: crypt_data or NULL */ /********************************************************************* Wait for crypt threads to stop accessing space */ diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index c97143235bc..ae8224d77bb 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -330,10 +330,17 @@ struct fil_space_t { bool printed_compression_failure; /*!< true if we have already printed compression failure */ + fil_space_crypt_t* crypt_data; + /*!< tablespace crypt data or NULL */ + bool page_0_crypt_read; + /*!< tablespace crypt data has been + read */ + ulint file_block_size; + /*!< file system block size */ + UT_LIST_NODE_T(fil_space_t) space_list; /*!< list of all spaces */ - fil_space_crypt_t* crypt_data; - ulint file_block_size;/*!< file system block size */ + ulint magic_n;/*!< FIL_SPACE_MAGIC_N */ }; @@ -470,7 +477,8 @@ fil_space_create( ulint zip_size,/*!< in: compressed page size, or 0 for uncompressed tablespaces */ ulint purpose, /*!< in: FIL_TABLESPACE, or FIL_LOG if log */ - fil_space_crypt_t* crypt_data); /*!< in: crypt data */ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + bool create_table); /*!< in: true if create table */ /*******************************************************************//** Assigns a new space id for a new single-table tablespace. This works simply by diff --git a/storage/innobase/include/page0page.ic b/storage/innobase/include/page0page.ic index cde3cad33f0..d7f1db82858 100644 --- a/storage/innobase/include/page0page.ic +++ b/storage/innobase/include/page0page.ic @@ -52,6 +52,7 @@ page_align( { return((page_t*) ut_align_down(ptr, UNIV_PAGE_SIZE)); } + #ifndef UNIV_INNOCHECKSUM /************************************************************//** Gets the offset within a page. @@ -230,18 +231,6 @@ page_header_reset_last_insert( #endif /* !UNIV_HOTBACKUP */ #endif /* !UNIV_INNOCHECKSUM */ -/************************************************************//** -Determine whether the page is in new-style compact format. -@return nonzero if the page is in compact format, zero if it is in -old-style format */ -UNIV_INLINE -ulint -page_is_comp( -/*=========*/ - const page_t* page) /*!< in: index page */ -{ - return(page_header_get_field(page, PAGE_N_HEAP) & 0x8000); -} #ifndef UNIV_INNOCHECKSUM /************************************************************//** @@ -973,6 +962,20 @@ page_rec_get_base_extra_size( } #endif /* !UNIV_INNOCHECKSUM */ + +/************************************************************//** +Determine whether the page is in new-style compact format. +@return nonzero if the page is in compact format, zero if it is in +old-style format */ +UNIV_INLINE +ulint +page_is_comp( +/*=========*/ + const page_t* page) /*!< in: index page */ +{ + return(page_header_get_field(page, PAGE_N_HEAP) & 0x8000); +} + /************************************************************//** Returns the sum of the sizes of the records in the record list, excluding the infimum and supremum records. diff --git a/storage/innobase/include/srv0mon.h b/storage/innobase/include/srv0mon.h index 422cfc3eaf6..d15110726b9 100644 --- a/storage/innobase/include/srv0mon.h +++ b/storage/innobase/include/srv0mon.h @@ -167,6 +167,7 @@ enum monitor_id_t { MONITOR_OVLD_INDEX_PAGES_WRITTEN, MONITOR_OVLD_NON_INDEX_PAGES_WRITTEN, MONITOR_OVLD_PAGES_READ, + MONITOR_OVLD_PAGES0_READ, MONITOR_OVLD_INDEX_SEC_REC_CLUSTER_READS, MONITOR_OVLD_INDEX_SEC_REC_CLUSTER_READS_AVOIDED, MONITOR_OVLD_BYTE_READ, diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 04c8cbecf99..d75f60944cc 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -179,6 +179,9 @@ struct srv_stats_t { /** Number of times prefix optimization avoided triggering cluster lookup */ ulint_ctr_64_t n_sec_rec_cluster_reads_avoided; + + /** Number of times page 0 is read from tablespace */ + ulint_ctr_64_t page0_read; }; extern const char* srv_main_thread_op_info; @@ -950,7 +953,8 @@ struct export_var_t{ ulint innodb_os_log_pending_fsyncs; /*!< fil_n_pending_log_flushes */ ulint innodb_page_size; /*!< UNIV_PAGE_SIZE */ ulint innodb_pages_created; /*!< buf_pool->stat.n_pages_created */ - ulint innodb_pages_read; /*!< buf_pool->stat.n_pages_read */ + ulint innodb_pages_read; /*!< buf_pool->stat.n_pages_read*/ + ulint innodb_page0_read; /*!< srv_stats.page0_read */ ulint innodb_pages_written; /*!< buf_pool->stat.n_pages_written */ ulint innodb_row_lock_waits; /*!< srv_n_lock_wait_count */ ulint innodb_row_lock_current_waits; /*!< srv_n_lock_wait_current_count */ diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index a5ab4f4911e..b2c96a7ed7b 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -3279,7 +3279,7 @@ fil_wait_crypt_bg_threads( uint last = start; if (table->space != 0) { - fil_space_crypt_mark_space_closing(table->space); + fil_space_crypt_mark_space_closing(table->space, table->crypt_data); } while (table->n_ref_count > 0) { @@ -4211,6 +4211,12 @@ row_drop_table_for_mysql( rw_lock_x_unlock(dict_index_get_lock(index)); } + /* If table has not yet have crypt_data, try to read it to + make freeing the table easier. */ + if (!table->crypt_data) { + table->crypt_data = fil_space_get_crypt_data(table->space); + } + /* We use the private SQL parser of Innobase to generate the query graphs needed in deleting the dictionary data from system tables in Innobase. Deleting a row from SYS_INDEXES table also diff --git a/storage/innobase/srv/srv0mon.cc b/storage/innobase/srv/srv0mon.cc index 3375ea40658..6bf4ad8af01 100644 --- a/storage/innobase/srv/srv0mon.cc +++ b/storage/innobase/srv/srv0mon.cc @@ -309,6 +309,12 @@ static monitor_info_t innodb_counter_info[] = MONITOR_EXISTING | MONITOR_DEFAULT_ON), MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES_READ}, + {"buffer_pages0_read", "buffer", + "Number of page 0 read (innodb_pages0_read)", + static_cast<monitor_type_t>( + MONITOR_EXISTING | MONITOR_DEFAULT_ON), + MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES0_READ}, + {"buffer_index_sec_rec_cluster_reads", "buffer", "Number of secondary record reads triggered cluster read", static_cast<monitor_type_t>( @@ -1718,6 +1724,11 @@ srv_mon_process_existing_counter( value = stat.n_pages_read; break; + /* innodb_pages0_read */ + case MONITOR_OVLD_PAGES0_READ: + value = srv_stats.page0_read; + break; + /* Number of times secondary index lookup triggered cluster lookup */ case MONITOR_OVLD_INDEX_SEC_REC_CLUSTER_READS: value = srv_stats.n_sec_rec_cluster_reads; diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index e17b27b44fc..076bc366132 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -3,7 +3,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2016, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1549,6 +1549,7 @@ srv_export_innodb_status(void) export_vars.innodb_pages_created = stat.n_pages_created; export_vars.innodb_pages_read = stat.n_pages_read; + export_vars.innodb_page0_read = srv_stats.page0_read; export_vars.innodb_pages_written = stat.n_pages_written; diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 0b81ad86f1c..135846384f6 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -3,7 +3,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2015, MariaDB Corporation +Copyright (c) 2013, 2016, MariaDB Corporation Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -675,7 +675,8 @@ create_log_files( logfilename, SRV_LOG_SPACE_FIRST_ID, fsp_flags_set_page_size(0, UNIV_PAGE_SIZE), FIL_LOG, - NULL /* no encryption yet */); + NULL /* no encryption yet */, + true /* this is create */); ut_a(fil_validate()); logfile0 = fil_node_create( @@ -813,7 +814,7 @@ open_or_create_data_files( ulint space; ulint rounded_size_pages; char name[10000]; - fil_space_crypt_t* crypt_data; + fil_space_crypt_t* crypt_data=NULL; if (srv_n_data_files >= 1000) { @@ -1150,18 +1151,20 @@ check_first_page: } *sum_of_new_sizes += srv_data_file_sizes[i]; - - crypt_data = fil_space_create_crypt_data(FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY); } ret = os_file_close(files[i]); ut_a(ret); if (i == 0) { + if (!crypt_data) { + crypt_data = fil_space_create_crypt_data(FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY); + } + flags = fsp_flags_set_page_size(0, UNIV_PAGE_SIZE); + fil_space_create(name, 0, flags, FIL_TABLESPACE, - crypt_data); - crypt_data = NULL; + crypt_data, (*create_new_db) == true); } ut_a(fil_validate()); @@ -1308,7 +1311,8 @@ srv_undo_tablespace_open( /* Set the compressed page size to 0 (non-compressed) */ flags = fsp_flags_set_page_size(0, UNIV_PAGE_SIZE); fil_space_create(name, space, flags, FIL_TABLESPACE, - NULL /* no encryption */); + NULL /* no encryption */, + true /* create */); ut_a(fil_validate()); @@ -2293,7 +2297,8 @@ innobase_start_or_create_for_mysql(void) SRV_LOG_SPACE_FIRST_ID, fsp_flags_set_page_size(0, UNIV_PAGE_SIZE), FIL_LOG, - NULL /* no encryption yet */); + NULL /* no encryption yet */, + true /* create */); ut_a(fil_validate()); @@ -2315,7 +2320,7 @@ innobase_start_or_create_for_mysql(void) /* Create the file space object for archived logs. Under MySQL, no archiving ever done. */ fil_space_create("arch_log_space", SRV_LOG_SPACE_FIRST_ID + 1, - 0, FIL_LOG); + 0, FIL_LOG, NULL, true); #endif /* UNIV_LOG_ARCHIVE */ log_group_init(0, i, srv_log_file_size * UNIV_PAGE_SIZE, SRV_LOG_SPACE_FIRST_ID, diff --git a/storage/xtradb/btr/btr0btr.cc b/storage/xtradb/btr/btr0btr.cc index 00a04e75c49..c2a70cce7aa 100644 --- a/storage/xtradb/btr/btr0btr.cc +++ b/storage/xtradb/btr/btr0btr.cc @@ -2,7 +2,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2014, 2015, MariaDB Corporation +Copyright (c) 2014, 2016, MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -743,14 +743,16 @@ btr_root_block_get( block = btr_block_get(space, zip_size, root_page_no, mode, (dict_index_t*)index, mtr); if (!block) { - index->table->is_encrypted = TRUE; - index->table->corrupted = FALSE; - - ib_push_warning(index->table->thd, DB_DECRYPTION_FAILED, - "Table %s in tablespace %lu is encrypted but encryption service or" - " used key_id is not available. " - " Can't continue reading table.", - index->table->name, space); + if (index && index->table) { + index->table->is_encrypted = TRUE; + index->table->corrupted = FALSE; + + ib_push_warning(index->table->thd, DB_DECRYPTION_FAILED, + "Table %s in tablespace %lu is encrypted but encryption service or" + " used key_id is not available. " + " Can't continue reading table.", + index->table->name, space); + } return NULL; } @@ -1840,6 +1842,11 @@ leaf_loop: root = btr_page_get(space, zip_size, root_page_no, RW_X_LATCH, NULL, &mtr); + if (!root) { + mtr_commit(&mtr); + return; + } + SRV_CORRUPT_TABLE_CHECK(root, { mtr_commit(&mtr); @@ -1909,17 +1916,19 @@ btr_free_root( block = btr_block_get(space, zip_size, root_page_no, RW_X_LATCH, NULL, mtr); - SRV_CORRUPT_TABLE_CHECK(block, return;); + if (block) { + SRV_CORRUPT_TABLE_CHECK(block, return;); - btr_search_drop_page_hash_index(block); + btr_search_drop_page_hash_index(block); - header = buf_block_get_frame(block) + PAGE_HEADER + PAGE_BTR_SEG_TOP; + header = buf_block_get_frame(block) + PAGE_HEADER + PAGE_BTR_SEG_TOP; #ifdef UNIV_BTR_DEBUG - ut_a(btr_root_fseg_validate(header, space)); + ut_a(btr_root_fseg_validate(header, space)); #endif /* UNIV_BTR_DEBUG */ - while (!fseg_free_step(header, mtr)) { - /* Free the entire segment in small steps. */ + while (!fseg_free_step(header, mtr)) { + /* Free the entire segment in small steps. */ + } } } #endif /* !UNIV_HOTBACKUP */ diff --git a/storage/xtradb/dict/dict0load.cc b/storage/xtradb/dict/dict0load.cc index 3dcdfc7c61a..8b7034e5eba 100644 --- a/storage/xtradb/dict/dict0load.cc +++ b/storage/xtradb/dict/dict0load.cc @@ -1154,11 +1154,14 @@ loop: space_id, name); } - /* We need to read page 0 to get (optional) IV - regardless if encryptions is turned on or not, - since if it's off we should decrypt a potentially - already encrypted table */ - bool read_page_0 = true; + /* We could read page 0 to get (optional) IV + if encryption is turned on, if it's off + we will read the page 0 later and find out + if we should decrypt a potentially + already encrypted table + bool read_page_0 = srv_encrypt_tables; */ + + bool read_page_0 = false; /* We set the 2nd param (fix_dict = true) here because we already have an x-lock on diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index ceffa950739..bedaf7cfe49 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -258,20 +258,6 @@ fil_space_read_crypt_data( } if (memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) { -#ifdef UNIV_DEBUG - ib_logf(IB_LOG_LEVEL_WARN, - "Found potentially bogus bytes on " - "page 0 offset %lu for space %lu : " - "[ %.2x %.2x %.2x %.2x %.2x %.2x ]. " - "Assuming space is not encrypted!.", - offset, space, - page[offset + 0], - page[offset + 1], - page[offset + 2], - page[offset + 3], - page[offset + 4], - page[offset + 5]); -#endif /* Crypt data is not stored. */ return NULL; } @@ -666,6 +652,61 @@ fil_space_encrypt( byte* tmp = fil_encrypt_buf(crypt_data, space, offset, lsn, src_frame, zip_size, dst_frame); +#ifdef UNIV_DEBUG + if (tmp) { + /* Verify that encrypted buffer is not corrupted */ + byte* tmp_mem = (byte *)malloc(UNIV_PAGE_SIZE); + dberr_t err = DB_SUCCESS; + byte* src = src_frame; + bool page_compressed_encrypted = (mach_read_from_2(tmp+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); + byte* comp_mem = NULL; + byte* uncomp_mem = NULL; + ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE; + + if (page_compressed_encrypted) { + comp_mem = (byte *)malloc(UNIV_PAGE_SIZE); + uncomp_mem = (byte *)malloc(UNIV_PAGE_SIZE); + memcpy(comp_mem, src_frame, UNIV_PAGE_SIZE); + fil_decompress_page(uncomp_mem, comp_mem, UNIV_PAGE_SIZE, NULL); + src = uncomp_mem; + } + + bool corrupted1 = buf_page_is_corrupted(true, src, zip_size); + bool ok = fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err); + + /* Need to decompress the page if it was also compressed */ + if (page_compressed_encrypted) { + memcpy(comp_mem, tmp_mem, UNIV_PAGE_SIZE); + fil_decompress_page(tmp_mem, comp_mem, UNIV_PAGE_SIZE, NULL); + } + + bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size); + bool different = memcmp(src, tmp_mem, size); + + if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) { + fprintf(stderr, "JAN: ok %d corrupted %d corrupted1 %d err %d different %d\n", ok , corrupted, corrupted1, err, different); + fprintf(stderr, "JAN1: src_frame\n"); + buf_page_print(src_frame, zip_size, BUF_PAGE_PRINT_NO_CRASH); + fprintf(stderr, "JAN2: encrypted_frame\n"); + buf_page_print(tmp, zip_size, BUF_PAGE_PRINT_NO_CRASH); + fprintf(stderr, "JAN1: decrypted_frame\n"); + buf_page_print(tmp_mem, zip_size, BUF_PAGE_PRINT_NO_CRASH); + ut_error; + } + + free(tmp_mem); + + if (comp_mem) { + free(comp_mem); + } + + if (uncomp_mem) { + free(uncomp_mem); + } + } + +#endif /* UNIV_DEBUG */ + return tmp; } @@ -2426,7 +2467,8 @@ UNIV_INTERN void fil_space_crypt_mark_space_closing( /*===============================*/ - ulint space) /*!< in: Space id */ + ulint space, /*!< in: tablespace id */ + fil_space_crypt_t* crypt_data) /*!< in: crypt_data or NULL */ { if (!fil_crypt_threads_inited) { return; @@ -2434,7 +2476,9 @@ fil_space_crypt_mark_space_closing( mutex_enter(&fil_crypt_threads_mutex); - fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); + if (!crypt_data) { + crypt_data = fil_space_get_crypt_data(space); + } if (crypt_data == NULL) { mutex_exit(&fil_crypt_threads_mutex); diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 58e08f11778..1db342d2bc7 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -1191,7 +1191,8 @@ fil_space_create( ulint id, /*!< in: space id */ ulint flags, /*!< in: tablespace flags */ ulint purpose,/*!< in: FIL_TABLESPACE, or FIL_LOG if log */ - fil_space_crypt_t* crypt_data) /*!< in: crypt data */ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + bool create_table) /*!< in: true if create table */ { fil_space_t* space; @@ -1285,10 +1286,23 @@ fil_space_create( space->is_in_unflushed_spaces = false; space->is_corrupt = FALSE; + space->crypt_data = crypt_data; - UT_LIST_ADD_LAST(space_list, fil_system->space_list, space); + /* In create table we write page 0 so we have already + "read" it and for system tablespaces we have read + crypt data at startup. */ + if (create_table || crypt_data != NULL) { + space->page_0_crypt_read = true; + } - space->crypt_data = crypt_data; + ib_logf(IB_LOG_LEVEL_INFO, + "Created tablespace for space %lu name %s key_id %u encryption %d\n", + space->id, + space->name, + space->crypt_data ? space->crypt_data->key_id : 0, + space->crypt_data ? space->crypt_data->encryption : 0); + + UT_LIST_ADD_LAST(space_list, fil_system->space_list, space); mutex_exit(&fil_system->mutex); @@ -2057,6 +2071,8 @@ fil_read_first_page( os_file_read(data_file, page, 0, UNIV_PAGE_SIZE); + srv_stats.page0_read.add(1); + /* The FSP_HEADER on page 0 is only valid for the first file in a tablespace. So if this is not the first datafile, leave *flags and *space_id as they were read from the first file and @@ -2077,6 +2093,7 @@ fil_read_first_page( ulint space = fsp_header_get_space_id(page); ulint offset = fsp_header_get_crypt_offset( fsp_flags_get_zip_size(*flags), NULL); + cdata = fil_space_read_crypt_data(space, page, offset); if (crypt_data) { @@ -3627,7 +3644,7 @@ fil_create_new_single_table_tablespace( } success = fil_space_create(tablename, space_id, flags, FIL_TABLESPACE, - crypt_data); + crypt_data, true); if (!success || !fil_node_create(path, size, space_id, FALSE)) { err = DB_ERROR; @@ -3861,6 +3878,7 @@ fil_open_single_table_tablespace( if (table) { table->crypt_data = def.crypt_data; + table->page_0_read = true; } /* Validate this single-table-tablespace with SYS_TABLES, @@ -3897,6 +3915,7 @@ fil_open_single_table_tablespace( if (table) { table->crypt_data = remote.crypt_data; + table->page_0_read = true; } /* Validate this single-table-tablespace with SYS_TABLES, @@ -3933,6 +3952,7 @@ fil_open_single_table_tablespace( if (table) { table->crypt_data = dict.crypt_data; + table->page_0_read = true; } /* Validate this single-table-tablespace with SYS_TABLES, @@ -4104,7 +4124,7 @@ skip_validate: if (err != DB_SUCCESS) { ; // Don't load the tablespace into the cache } else if (!fil_space_create(tablename, id, flags, FIL_TABLESPACE, - crypt_data)) { + crypt_data, false)) { err = DB_ERROR; } else { /* We do not measure the size of the file, that is why @@ -4710,7 +4730,7 @@ will_not_choose: #endif /* UNIV_HOTBACKUP */ ibool file_space_create_success = fil_space_create( tablename, fsp->id, fsp->flags, FIL_TABLESPACE, - fsp->crypt_data); + fsp->crypt_data, false); if (!file_space_create_success) { if (srv_force_recovery > 0) { @@ -7323,7 +7343,46 @@ fil_space_get_crypt_data( space = fil_space_get_by_id(id); if (space != NULL) { + /* If we have not yet read the page0 + of this tablespace we will do it now. */ + if (!space->crypt_data && !space->page_0_crypt_read) { + ulint flags; + ulint space_id; + lsn_t min_flushed_lsn; + lsn_t max_flushed_lsn; + fil_node_t* node; + + ut_a(space->crypt_data == NULL); + node = UT_LIST_GET_FIRST(space->chain); + + fil_node_prepare_for_io(node, fil_system, space); + + const char* msg = fil_read_first_page(node->handle, + false, + &flags, + &space_id, + &min_flushed_lsn, + &max_flushed_lsn, + &space->crypt_data); + + fil_node_complete_io(node, fil_system, OS_FILE_READ); + + ib_logf(IB_LOG_LEVEL_INFO, + "Read page 0 from tablespace for space %lu name %s key_id %u encryption %d handle %d\n", + space_id, + space->name, + space->crypt_data ? space->crypt_data->key_id : 0, + space->crypt_data ? space->crypt_data->encryption : 0, + node->handle); + + ut_a(space->id == space_id); + + space->page_0_crypt_read = true; + } + crypt_data = space->crypt_data; + + ut_ad(space->page_0_crypt_read); } mutex_exit(&fil_system->mutex); diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 868ece7dbe9..14870659b0e 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -1116,6 +1116,8 @@ static SHOW_VAR innodb_status_variables[]= { (char*) &export_vars.innodb_pages_created, SHOW_LONG}, {"pages_read", (char*) &export_vars.innodb_pages_read, SHOW_LONG}, + {"pages0_read", + (char*) &export_vars.innodb_page0_read, SHOW_LONG}, {"pages_written", (char*) &export_vars.innodb_pages_written, SHOW_LONG}, {"purge_trx_id", diff --git a/storage/xtradb/include/btr0btr.ic b/storage/xtradb/include/btr0btr.ic index 8c9c3bead09..34e0d36e230 100644 --- a/storage/xtradb/include/btr0btr.ic +++ b/storage/xtradb/include/btr0btr.ic @@ -60,7 +60,9 @@ btr_block_get_func( NULL, BUF_GET, file, line, mtr, &err); if (err == DB_DECRYPTION_FAILED) { - index->table->is_encrypted = true; + if (index && index->table) { + index->table->is_encrypted = true; + } } if (block) { diff --git a/storage/xtradb/include/dict0mem.h b/storage/xtradb/include/dict0mem.h index 24219ac8799..29f5b8fe814 100644 --- a/storage/xtradb/include/dict0mem.h +++ b/storage/xtradb/include/dict0mem.h @@ -1046,6 +1046,8 @@ struct dict_table_t{ mem_heap_t* heap; /*!< memory heap */ char* name; /*!< table name */ void* thd; /*!< thd */ + bool page_0_read; /*!< true if page 0 has + been already read */ fil_space_crypt_t *crypt_data; /*!< crypt data if present */ const char* dir_path_of_temp_table;/*!< NULL or the directory path where a TEMPORARY table that was explicitly diff --git a/storage/xtradb/include/fil0crypt.h b/storage/xtradb/include/fil0crypt.h index 5deed1f001c..b656cd3985d 100644 --- a/storage/xtradb/include/fil0crypt.h +++ b/storage/xtradb/include/fil0crypt.h @@ -316,7 +316,8 @@ UNIV_INTERN void fil_space_crypt_mark_space_closing( /*===============================*/ - ulint space); /*!< in: tablespace id */ + ulint space, /*!< in: tablespace id */ + fil_space_crypt_t* crypt_data); /*!< in: crypt_data or NULL */ /********************************************************************* Wait for crypt threads to stop accessing space */ diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 20e8303f764..95011ae6125 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -321,13 +321,21 @@ struct fil_space_t { /*!< true if this space is currently in unflushed_spaces */ ibool is_corrupt; + /*!< true if tablespace corrupted */ bool printed_compression_failure; /*!< true if we have already printed compression failure */ + fil_space_crypt_t* crypt_data; + /*!< tablespace crypt data or NULL */ + bool page_0_crypt_read; + /*!< tablespace crypt data has been + read */ + ulint file_block_size; + /*!< file system block size */ + UT_LIST_NODE_T(fil_space_t) space_list; /*!< list of all spaces */ - fil_space_crypt_t* crypt_data; - ulint file_block_size;/*!< file system block size */ + ulint magic_n;/*!< FIL_SPACE_MAGIC_N */ }; @@ -471,7 +479,8 @@ fil_space_create( ulint zip_size,/*!< in: compressed page size, or 0 for uncompressed tablespaces */ ulint purpose, /*!< in: FIL_TABLESPACE, or FIL_LOG if log */ - fil_space_crypt_t* crypt_data); /*!< in: crypt data */ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + bool create_table); /*!< in: true if create table */ /*******************************************************************//** Assigns a new space id for a new single-table tablespace. This works simply by diff --git a/storage/xtradb/include/srv0mon.h b/storage/xtradb/include/srv0mon.h index 33ae7749ca5..3b030d56d29 100644 --- a/storage/xtradb/include/srv0mon.h +++ b/storage/xtradb/include/srv0mon.h @@ -167,6 +167,7 @@ enum monitor_id_t { MONITOR_OVLD_INDEX_PAGES_WRITTEN, MONITOR_OVLD_NON_INDEX_PAGES_WRITTEN, MONITOR_OVLD_PAGES_READ, + MONITOR_OVLD_PAGES0_READ, MONITOR_OVLD_INDEX_SEC_REC_CLUSTER_READS, MONITOR_OVLD_INDEX_SEC_REC_CLUSTER_READS_AVOIDED, MONITOR_OVLD_BYTE_READ, diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index bd7a8731bdf..d95adf00814 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -186,6 +186,9 @@ struct srv_stats_t { /** Number of lock waits that have been up to max time (i.e.) lock wait timeout */ ulint_ctr_1_t n_lock_max_wait_time; + + /** Number of times page 0 is read from tablespace */ + ulint_ctr_64_t page0_read; }; extern const char* srv_main_thread_op_info; @@ -1161,7 +1164,8 @@ struct export_var_t{ ulint innodb_os_log_pending_fsyncs; /*!< fil_n_pending_log_flushes */ ulint innodb_page_size; /*!< UNIV_PAGE_SIZE */ ulint innodb_pages_created; /*!< buf_pool->stat.n_pages_created */ - ulint innodb_pages_read; /*!< buf_pool->stat.n_pages_read */ + ulint innodb_pages_read; /*!< buf_pool->stat.n_pages_read*/ + ulint innodb_page0_read; /*!< srv_stats.page0_read */ ulint innodb_pages_written; /*!< buf_pool->stat.n_pages_written */ ib_int64_t innodb_purge_trx_id; ib_int64_t innodb_purge_undo_no; diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index ba2e0047fe9..059254a3c49 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -3293,7 +3293,7 @@ fil_wait_crypt_bg_threads( uint last = start; if (table->space != 0) { - fil_space_crypt_mark_space_closing(table->space); + fil_space_crypt_mark_space_closing(table->space, table->crypt_data); } while (table->n_ref_count > 0) { @@ -4225,6 +4225,12 @@ row_drop_table_for_mysql( rw_lock_x_unlock(dict_index_get_lock(index)); } + /* If table has not yet have crypt_data, try to read it to + make freeing the table easier. */ + if (!table->crypt_data) { + table->crypt_data = fil_space_get_crypt_data(table->space); + } + /* We use the private SQL parser of Innobase to generate the query graphs needed in deleting the dictionary data from system tables in Innobase. Deleting a row from SYS_INDEXES table also diff --git a/storage/xtradb/srv/srv0mon.cc b/storage/xtradb/srv/srv0mon.cc index e72868c6450..1e0d21d4a9e 100644 --- a/storage/xtradb/srv/srv0mon.cc +++ b/storage/xtradb/srv/srv0mon.cc @@ -309,6 +309,12 @@ static monitor_info_t innodb_counter_info[] = MONITOR_EXISTING | MONITOR_DEFAULT_ON), MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES_READ}, + {"buffer_pages0_read", "buffer", + "Number of page 0 read (innodb_pages0_read)", + static_cast<monitor_type_t>( + MONITOR_EXISTING | MONITOR_DEFAULT_ON), + MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES0_READ}, + {"buffer_index_sec_rec_cluster_reads", "buffer", "Number of secondary record reads triggered cluster read", static_cast<monitor_type_t>( @@ -1715,6 +1721,11 @@ srv_mon_process_existing_counter( value = stat.n_pages_read; break; + /* innodb_pages0_read */ + case MONITOR_OVLD_PAGES0_READ: + value = srv_stats.page0_read; + break; + /* Number of times secondary index lookup triggered cluster lookup */ case MONITOR_OVLD_INDEX_SEC_REC_CLUSTER_READS: value = srv_stats.n_sec_rec_cluster_reads; diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index 58b442e1aa8..a836442eb70 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -3,7 +3,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2016, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1951,6 +1951,7 @@ srv_export_innodb_status(void) export_vars.innodb_pages_created = stat.n_pages_created; export_vars.innodb_pages_read = stat.n_pages_read; + export_vars.innodb_page0_read = srv_stats.page0_read; export_vars.innodb_pages_written = stat.n_pages_written; diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index ca1eb194b00..6b1e8c39b38 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -705,7 +705,8 @@ create_log_files( logfilename, SRV_LOG_SPACE_FIRST_ID, fsp_flags_set_page_size(0, UNIV_PAGE_SIZE), FIL_LOG, - NULL /* no encryption yet */); + NULL /* no encryption yet */, + true /* this is create */); ut_a(fil_validate()); logfile0 = fil_node_create( @@ -727,7 +728,7 @@ create_log_files( #ifdef UNIV_LOG_ARCHIVE /* Create the file space object for archived logs. */ fil_space_create("arch_log_space", SRV_LOG_SPACE_FIRST_ID + 1, - 0, FIL_LOG, NULL /* no encryption yet */); + 0, FIL_LOG, NULL /* no encryption yet */, true /* create */); #endif log_group_init(0, srv_n_log_files, srv_log_file_size * UNIV_PAGE_SIZE, @@ -853,7 +854,7 @@ open_or_create_data_files( ulint space; ulint rounded_size_pages; char name[10000]; - fil_space_crypt_t* crypt_data; + fil_space_crypt_t* crypt_data=NULL; if (srv_n_data_files >= 1000) { @@ -1184,18 +1185,20 @@ check_first_page: } *sum_of_new_sizes += srv_data_file_sizes[i]; - - crypt_data = fil_space_create_crypt_data(FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY); } ret = os_file_close(files[i]); ut_a(ret); if (i == 0) { + if (!crypt_data) { + crypt_data = fil_space_create_crypt_data(FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY); + } + flags = fsp_flags_set_page_size(0, UNIV_PAGE_SIZE); + fil_space_create(name, 0, flags, FIL_TABLESPACE, - crypt_data); - crypt_data = NULL; + crypt_data, (*create_new_db) == true); } ut_a(fil_validate()); @@ -1342,7 +1345,8 @@ srv_undo_tablespace_open( /* Set the compressed page size to 0 (non-compressed) */ flags = fsp_flags_set_page_size(0, UNIV_PAGE_SIZE); fil_space_create(name, space, flags, FIL_TABLESPACE, - NULL /* no encryption */); + NULL /* no encryption */, + true /* create */); ut_a(fil_validate()); @@ -2371,7 +2375,8 @@ innobase_start_or_create_for_mysql(void) SRV_LOG_SPACE_FIRST_ID, fsp_flags_set_page_size(0, UNIV_PAGE_SIZE), FIL_LOG, - NULL /* no encryption yet */); + NULL /* no encryption yet */, + true /* create */); ut_a(fil_validate()); @@ -2393,7 +2398,8 @@ innobase_start_or_create_for_mysql(void) /* Create the file space object for archived logs. Under MySQL, no archiving ever done. */ fil_space_create("arch_log_space", SRV_LOG_SPACE_FIRST_ID + 1, - 0, FIL_LOG, NULL /* no encryption yet */); + 0, FIL_LOG, NULL /* no encryption yet */, + true /* create */); #endif /* UNIV_LOG_ARCHIVE */ log_group_init(0, i, srv_log_file_size * UNIV_PAGE_SIZE, SRV_LOG_SPACE_FIRST_ID, |