diff options
author | Thirunarayanan Balathandayuthapani <thiru@mariadb.com> | 2018-08-01 14:58:03 +0530 |
---|---|---|
committer | Thirunarayanan Balathandayuthapani <thiru@mariadb.com> | 2018-08-01 14:58:03 +0530 |
commit | 236ae858ef502d92c4c8c736569ec621e5f92a6d (patch) | |
tree | 7f5c9af4540a3a4a7deb4a5d36c09853c11bec8a | |
parent | f4eac2deeb046a68e4a26ed1a08f8df51bbf817b (diff) | |
download | mariadb-git-bb-10.2-mdev-14398.tar.gz |
MDEV-14398 When innodb_encryption_rotate_key_age=0 is set,bb-10.2-mdev-14398
server won't encrypt tablespaces
- Introduced new variable called "innodb_encrypt_tables_deferred" which avoids the
opening of first page by rotation thread. It only does decrypt/encrypt the already
opened table depends on innodb_encrypt_tables variable value.
- Removed the rotation list from fil_system and is_in_rotation_list from fil_space_t.
- It removes the relation between innodb_encrypt_tables and
innodb_encryption_rotate_key_age.
14 files changed, 161 insertions, 145 deletions
diff --git a/mysql-test/suite/encryption/r/debug_key_management.result b/mysql-test/suite/encryption/r/debug_key_management.result index 02e05b4d221..f3abb346281 100644 --- a/mysql-test/suite/encryption/r/debug_key_management.result +++ b/mysql-test/suite/encryption/r/debug_key_management.result @@ -3,6 +3,7 @@ show variables like 'innodb_encrypt%'; Variable_name Value innodb_encrypt_log ON innodb_encrypt_tables ON +innodb_encrypt_tables_deferred OFF innodb_encryption_rotate_key_age 2 innodb_encryption_rotation_iops 100 innodb_encryption_threads 4 diff --git a/mysql-test/suite/encryption/r/innodb-key-rotation-disable.result b/mysql-test/suite/encryption/r/innodb-key-rotation-disable.result index 680db692c17..124ccad1773 100644 --- a/mysql-test/suite/encryption/r/innodb-key-rotation-disable.result +++ b/mysql-test/suite/encryption/r/innodb-key-rotation-disable.result @@ -1,7 +1,6 @@ -SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0; -NAME SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0; NAME +innodb_system create database enctests; use enctests; create table t1(a int not null primary key, b char(200)) engine=innodb; diff --git a/mysql-test/suite/encryption/r/innodb_encrypt_deferred.result b/mysql-test/suite/encryption/r/innodb_encrypt_deferred.result new file mode 100644 index 00000000000..2eee1452233 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb_encrypt_deferred.result @@ -0,0 +1,41 @@ +# Restart the server with encryption and +# innodb_encrypt_tables_deferred enabled. +CREATE TABLE t1 (pk INT PRIMARY KEY, c VARCHAR(256))engine=innodb; +INSERT INTO t1 VALUES(1, "MariaDB"); +CREATE TABLE t2(pk INT PRIMARY KEY)ENGINE=InnoDB; +INSERT INTO t2 VALUES(1); +SELECT * FROM t1 LIMIT 1; +pk c +1 MariaDB +# Wait until encryption threads have encrypted all tablespaces +SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0; +NAME +SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0; +NAME +innodb_system +mysql/innodb_index_stats +mysql/innodb_table_stats +test/t1 +test/t2 +# Now turn off encryption and wait for threads to decrypt all tablespaces +SET GLOBAL innodb_encryption_rotate_key_age = 1; +SET GLOBAL innodb_encrypt_tables = off; +SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0; +NAME +innodb_system +mysql/innodb_index_stats +mysql/innodb_table_stats +test/t1 +test/t2 +SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0; +NAME +SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0; +NAME +innodb_system +mysql/innodb_index_stats +mysql/innodb_table_stats +test/t1 +test/t2 +SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0; +NAME +DROP TABLE t2, t1; diff --git a/mysql-test/suite/encryption/r/innodb_encryption.result b/mysql-test/suite/encryption/r/innodb_encryption.result index 7b7601a289c..a5d436e6a9a 100644 --- a/mysql-test/suite/encryption/r/innodb_encryption.result +++ b/mysql-test/suite/encryption/r/innodb_encryption.result @@ -3,6 +3,7 @@ SHOW VARIABLES LIKE 'innodb_encrypt%'; Variable_name Value innodb_encrypt_log ON innodb_encrypt_tables ON +innodb_encrypt_tables_deferred OFF innodb_encryption_rotate_key_age 15 innodb_encryption_rotation_iops 100 innodb_encryption_threads 4 @@ -57,6 +58,7 @@ SHOW VARIABLES LIKE 'innodb_encrypt%'; Variable_name Value innodb_encrypt_log ON innodb_encrypt_tables OFF +innodb_encrypt_tables_deferred OFF innodb_encryption_rotate_key_age 15 innodb_encryption_rotation_iops 100 innodb_encryption_threads 0 diff --git a/mysql-test/suite/encryption/t/innodb-key-rotation-disable.opt b/mysql-test/suite/encryption/t/innodb-key-rotation-disable.opt index 03a0028d371..da1b94aeec5 100644 --- a/mysql-test/suite/encryption/t/innodb-key-rotation-disable.opt +++ b/mysql-test/suite/encryption/t/innodb-key-rotation-disable.opt @@ -2,4 +2,5 @@ --innodb-encrypt-log --innodb-encryption-rotate-key-age=0 --innodb-encryption-threads=4 +--innodb-encrypt-tables-deferred=ON --innodb-tablespaces-encryption diff --git a/mysql-test/suite/encryption/t/innodb-key-rotation-disable.test b/mysql-test/suite/encryption/t/innodb-key-rotation-disable.test index 574e0c3becc..41959da1dba 100644 --- a/mysql-test/suite/encryption/t/innodb-key-rotation-disable.test +++ b/mysql-test/suite/encryption/t/innodb-key-rotation-disable.test @@ -3,7 +3,6 @@ # not embedded because of restarts -- source include/not_embedded.inc -SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0; SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0; --disable_query_log diff --git a/mysql-test/suite/encryption/t/innodb_encrypt_deferred.opt b/mysql-test/suite/encryption/t/innodb_encrypt_deferred.opt new file mode 100644 index 00000000000..7d3f2da7971 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb_encrypt_deferred.opt @@ -0,0 +1 @@ +--innodb-tablespaces-encryption diff --git a/mysql-test/suite/encryption/t/innodb_encrypt_deferred.test b/mysql-test/suite/encryption/t/innodb_encrypt_deferred.test new file mode 100644 index 00000000000..3ca947fe2cd --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb_encrypt_deferred.test @@ -0,0 +1,45 @@ +-- source include/have_innodb.inc +-- source include/not_embedded.inc +-- source include/have_example_key_management_plugin.inc + +--echo # Restart the server with encryption and +--echo # innodb_encrypt_tables_deferred enabled. + +CREATE TABLE t1 (pk INT PRIMARY KEY, c VARCHAR(256))engine=innodb; +INSERT INTO t1 VALUES(1, "MariaDB"); + +let $restart_parameters= --innodb_encrypt_tables=ON --innodb_encryption_threads=1 --innodb_encryption_rotate_key_age=0 --innodb_buffer_pool_load_at_startup=0 --innodb_encrypt_tables_deferred=1; +--source include/restart_mysqld.inc + +CREATE TABLE t2(pk INT PRIMARY KEY)ENGINE=InnoDB; +INSERT INTO t2 VALUES(1); + +SELECT * FROM t1 LIMIT 1; + + +--echo # Wait until encryption threads have encrypted all tablespaces + +--let $tables_count= `select count(*) + 1 from information_schema.tables where engine = 'InnoDB'` +--let $wait_timeout= 600 +--let $wait_condition=SELECT COUNT(*) = $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0 AND ROTATING_OR_FLUSHING = 0; +--source include/wait_condition.inc + +SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0; +SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0; + +--echo # Now turn off encryption and wait for threads to decrypt all tablespaces +SET GLOBAL innodb_encryption_rotate_key_age = 1; +SET GLOBAL innodb_encrypt_tables = off; + +--let $wait_condition=SELECT COUNT(*) = $tables_count FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND ROTATING_OR_FLUSHING = 0; +--source include/wait_condition.inc + +SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0; +SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0; + +--let $restart_parameters= +-- source include/restart_mysqld.inc + +SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0; +SELECT NAME FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0; +DROP TABLE t2, t1; diff --git a/mysql-test/suite/sys_vars/r/innodb_encrypt_tables_deferred_basic.result b/mysql-test/suite/sys_vars/r/innodb_encrypt_tables_deferred_basic.result new file mode 100644 index 00000000000..c4581f74757 --- /dev/null +++ b/mysql-test/suite/sys_vars/r/innodb_encrypt_tables_deferred_basic.result @@ -0,0 +1,17 @@ +0 Expected +SELECT @@GLOBAL.innodb_encrypt_tables_deferred; +@@GLOBAL.innodb_encrypt_tables_deferred +0 +SET @@GLOBAL.innodb_encrypt_tables_deferred=123; +ERROR 42000: Variable 'innodb_encrypt_tables_deferred' can't be set to the value of '123' +SET GLOBAL innodb_encrypt_tables_deferred='foo'; +ERROR 42000: Variable 'innodb_encrypt_tables_deferred' can't be set to the value of 'foo' +SET GLOBAL innodb_encrypt_tables_deferred=ON; +SELECT @@global.innodb_encrypt_tables_deferred; +@@global.innodb_encrypt_tables_deferred +1 +SET GLOBAL innodb_encrypt_tables_deferred=OFF; +SELECT @@global.innodb_encrypt_tables_deferred; +@@global.innodb_encrypt_tables_deferred +0 +SET GLOBAL innodb_encrypt_tables_deferred=default; diff --git a/mysql-test/suite/sys_vars/t/innodb_encrypt_tables_deferred_basic.test b/mysql-test/suite/sys_vars/t/innodb_encrypt_tables_deferred_basic.test new file mode 100644 index 00000000000..42810dc52b9 --- /dev/null +++ b/mysql-test/suite/sys_vars/t/innodb_encrypt_tables_deferred_basic.test @@ -0,0 +1,22 @@ +--source include/have_innodb.inc + +# Display default value +--echo 0 Expected +SELECT @@GLOBAL.innodb_encrypt_tables_deferred; + +# Check if value can be set +--error ER_WRONG_VALUE_FOR_VAR +SET @@GLOBAL.innodb_encrypt_tables_deferred=123; + +-- error ER_WRONG_VALUE_FOR_VAR +SET GLOBAL innodb_encrypt_tables_deferred='foo'; + +# Check that changing value works and that setting the same value again +# is as expected +SET GLOBAL innodb_encrypt_tables_deferred=ON; +SELECT @@global.innodb_encrypt_tables_deferred; + +SET GLOBAL innodb_encrypt_tables_deferred=OFF; +SELECT @@global.innodb_encrypt_tables_deferred; + +SET GLOBAL innodb_encrypt_tables_deferred=default; diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 3c9640c8e15..c8e4574745d 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -93,6 +93,8 @@ static ib_mutex_t crypt_stat_mutex; extern my_bool srv_background_scrub_data_uncompressed; extern my_bool srv_background_scrub_data_compressed; +UNIV_INTERN my_bool srv_encrypt_tables_deferred; + /*********************************************************************** Check if a key needs rotation given a key_state @param[in] crypt_data Encryption information @@ -1465,16 +1467,18 @@ fil_crypt_find_space_to_rotate( state->space = NULL; } - /* If key rotation is enabled (default) we iterate all tablespaces. - If key rotation is not enabled we iterate only the tablespaces - added to keyrotation list. */ - if (srv_fil_crypt_rotate_key_age) { - state->space = fil_space_next(state->space); - } else { - state->space = fil_space_keyrotate_next(state->space); - } + state->space = fil_space_next(state->space); while (!state->should_shutdown() && state->space) { + + /* If innodb_encrypt_tables_deferred is enabled then + no need to read page 0 if it is not yet read. */ + if (srv_encrypt_tables_deferred + && !(state->space->size || state->space->crypt_data)) { + state->space = fil_space_next(state->space); + continue; + } + /* If there is no crypt data and we have not yet read page 0 for this tablespace, we need to read it before we can continue. */ @@ -1490,11 +1494,7 @@ fil_crypt_find_space_to_rotate( return true; } - if (srv_fil_crypt_rotate_key_age) { - state->space = fil_space_next(state->space); - } else { - state->space = fil_space_keyrotate_next(state->space); - } + state->space = fil_space_next(state->space); } /* if we didn't find any space return iops */ @@ -2161,9 +2161,9 @@ DECLARE_THREAD(fil_crypt_thread)( } if (recheck) { - /* check recheck here, after sleep, so - * that we don't busy loop while when one thread is starting - * a space*/ + /* check recheck here, after sleep, so that we + don't busy loop while when one thread is + starting a space */ break; } diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index feebe5d2162..42f768f5e35 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -156,6 +156,10 @@ UNIV_INTERN fil_system_t* fil_system = NULL; UNIV_INTERN extern uint srv_fil_crypt_rotate_key_age; UNIV_INTERN extern ib_mutex_t fil_crypt_threads_mutex; +/** If it is enabled then innodb rotation thread should avoid the +opening of page 0 for the tablespace. */ +UNIV_INTERN extern my_bool srv_encrypt_tables_deferred; + /** Determine if user has explicitly disabled fsync(). */ # define fil_buffering_disabled(s) \ ((s)->purpose == FIL_TYPE_TABLESPACE \ @@ -1324,12 +1328,6 @@ fil_space_detach( UT_LIST_REMOVE(fil_system->unflushed_spaces, space); } - if (space->is_in_rotation_list) { - space->is_in_rotation_list = false; - - UT_LIST_REMOVE(fil_system->rotation_list, space); - } - UT_LIST_REMOVE(fil_system->space_list, space); ut_a(space->magic_n == FIL_SPACE_MAGIC_N); @@ -1548,23 +1546,7 @@ fil_space_create( fil_system->max_assigned_id = id; } - /* Inform key rotation that there could be something - to do */ - if (purpose == FIL_TYPE_TABLESPACE - && !srv_fil_crypt_rotate_key_age && fil_crypt_threads_event && - (mode == FIL_ENCRYPTION_ON || mode == FIL_ENCRYPTION_OFF || - srv_encrypt_tables)) { - /* Key rotation is not enabled, need to inform background - encryption threads. */ - UT_LIST_ADD_LAST(fil_system->rotation_list, space); - space->is_in_rotation_list = true; - mutex_exit(&fil_system->mutex); - mutex_enter(&fil_crypt_threads_mutex); - os_event_set(fil_crypt_threads_event); - mutex_exit(&fil_crypt_threads_mutex); - } else { - mutex_exit(&fil_system->mutex); - } + mutex_exit(&fil_system->mutex); return(space); } @@ -1901,7 +1883,6 @@ fil_init( UT_LIST_INIT(fil_system->LRU, &fil_node_t::LRU); UT_LIST_INIT(fil_system->space_list, &fil_space_t::space_list); - UT_LIST_INIT(fil_system->rotation_list, &fil_space_t::rotation_list); UT_LIST_INIT(fil_system->unflushed_spaces, &fil_space_t::unflushed_spaces); UT_LIST_INIT(fil_system->named_spaces, &fil_space_t::named_spaces); @@ -6121,90 +6102,6 @@ fil_space_next(fil_space_t* prev_space) return(space); } -/** -Remove space from key rotation list if there are no more -pending operations. -@param[in,out] space Tablespace */ -static -void -fil_space_remove_from_keyrotation(fil_space_t* space) -{ - ut_ad(mutex_own(&fil_system->mutex)); - ut_ad(space); - - if (space->n_pending_ops == 0 && space->is_in_rotation_list) { - space->is_in_rotation_list = false; - ut_a(UT_LIST_GET_LEN(fil_system->rotation_list) > 0); - UT_LIST_REMOVE(fil_system->rotation_list, space); - } -} - - -/** Return the next fil_space_t from key rotation list. -Once started, the caller must keep calling this until it returns NULL. -fil_space_acquire() and fil_space_release() are invoked here which -blocks a concurrent operation from dropping the tablespace. -@param[in] prev_space Pointer to the previous fil_space_t. -If NULL, use the first fil_space_t on fil_system->space_list. -@return pointer to the next fil_space_t. -@retval NULL if this was the last*/ -fil_space_t* -fil_space_keyrotate_next( - fil_space_t* prev_space) -{ - fil_space_t* space = prev_space; - fil_space_t* old = NULL; - - mutex_enter(&fil_system->mutex); - - if (UT_LIST_GET_LEN(fil_system->rotation_list) == 0) { - if (space) { - ut_ad(space->n_pending_ops > 0); - space->n_pending_ops--; - fil_space_remove_from_keyrotation(space); - } - mutex_exit(&fil_system->mutex); - return(NULL); - } - - if (prev_space == NULL) { - space = UT_LIST_GET_FIRST(fil_system->rotation_list); - - /* We can trust that space is not NULL because we - checked list length above */ - } else { - ut_ad(space->n_pending_ops > 0); - - /* Move on to the next fil_space_t */ - space->n_pending_ops--; - - old = space; - space = UT_LIST_GET_NEXT(rotation_list, space); - - fil_space_remove_from_keyrotation(old); - } - - /* Skip spaces that are being created by fil_ibd_create(), - or dropped or truncated. Note that rotation_list contains only - space->purpose == FIL_TYPE_TABLESPACE. */ - while (space != NULL - && (UT_LIST_GET_LEN(space->chain) == 0 - || space->is_stopping())) { - - old = space; - space = UT_LIST_GET_NEXT(rotation_list, space); - fil_space_remove_from_keyrotation(old); - } - - if (space != NULL) { - space->n_pending_ops++; - } - - mutex_exit(&fil_system->mutex); - - return(space); -} - /** Determine the block size of the data file. @param[in] space tablespace @param[in] offset page number diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index a8594fd37b3..3d9e8d35294 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -256,6 +256,7 @@ static char* innodb_version_str = (char*) INNODB_VERSION_STR; extern uint srv_fil_crypt_rotate_key_age; extern uint srv_n_fil_crypt_iops; +extern my_bool srv_encrypt_tables_deferred; extern my_bool srv_immediate_scrub_data_uncompressed; extern my_bool srv_background_scrub_data_uncompressed; extern my_bool srv_background_scrub_data_compressed; @@ -21142,6 +21143,13 @@ static MYSQL_SYSVAR_UINT(encryption_rotation_iops, srv_n_fil_crypt_iops, innodb_encryption_rotation_iops_update, srv_n_fil_crypt_iops, 0, UINT_MAX32, 0); +static MYSQL_SYSVAR_BOOL(encrypt_tables_deferred, srv_encrypt_tables_deferred, + PLUGIN_VAR_OPCMDARG, + "It avoids the opening of page 0 for rotation thread. " + "Does encryption/decryption of already opened table " + "depends on innodb_encrypt_tables variable.", + NULL, NULL, FALSE); + static MYSQL_SYSVAR_BOOL(scrub_log, srv_scrub_log, PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY, "Enable background redo log (ib_logfile0, ib_logfile1...) scrubbing", @@ -21411,6 +21419,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(encryption_threads), MYSQL_SYSVAR(encryption_rotate_key_age), MYSQL_SYSVAR(encryption_rotation_iops), + MYSQL_SYSVAR(encrypt_tables_deferred), MYSQL_SYSVAR(scrub_log), MYSQL_SYSVAR(scrub_log_speed), MYSQL_SYSVAR(encrypt_log), diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 8aa8a746ce1..52dcb25259d 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -167,11 +167,6 @@ struct fil_space_t { unflushed_spaces */ UT_LIST_NODE_T(fil_space_t) space_list; /*!< list of all spaces */ - /** other tablespaces needing key rotation */ - UT_LIST_NODE_T(fil_space_t) rotation_list; - /** whether this tablespace needs key rotation */ - bool is_in_rotation_list; - /** MariaDB encryption data */ fil_space_crypt_t* crypt_data; @@ -793,19 +788,6 @@ fil_space_next( fil_space_t* prev_space) MY_ATTRIBUTE((warn_unused_result)); -/** Return the next fil_space_t from key rotation list. -Once started, the caller must keep calling this until it returns NULL. -fil_space_acquire() and fil_space_release() are invoked here which -blocks a concurrent operation from dropping the tablespace. -@param[in,out] prev_space Pointer to the previous fil_space_t. -If NULL, use the first fil_space_t on fil_system->space_list. -@return pointer to the next fil_space_t. -@retval NULL if this was the last*/ -fil_space_t* -fil_space_keyrotate_next( - fil_space_t* prev_space) - MY_ATTRIBUTE((warn_unused_result)); - /** Wrapper with reference-counting for a fil_space_t. */ class FilSpace { |