diff options
author | Jan Lindström <jan.lindstrom@mariadb.com> | 2017-09-14 09:23:20 +0300 |
---|---|---|
committer | Jan Lindström <jan.lindstrom@mariadb.com> | 2017-09-14 09:23:20 +0300 |
commit | fa2701c6f7b028782cf231565f578b2fc0f10d51 (patch) | |
tree | 9fdefbda86d247e32f8c497b716e58eb4412af11 | |
parent | 112d721a74c1f9dcf3321006418fd4abc2884751 (diff) | |
download | mariadb-git-fa2701c6f7b028782cf231565f578b2fc0f10d51.tar.gz |
MDEV-12634: Uninitialised ROW_MERGE_RESERVE_SIZE bytes written to tem…
…porary file
Fixed by removing writing key version to start of every block that
was encrypted. Instead we will use single key version from log_sys
crypt info.
After this MDEV also blocks writen to row log are encrypted and blocks
read from row log aren decrypted if encryption is configured for the
table.
innodb_status_variables[], struct srv_stats_t
Added status variables for merge block and row log block
encryption and decryption amounts.
Removed ROW_MERGE_RESERVE_SIZE define.
row_merge_fts_doc_tokenize
Remove ROW_MERGE_RESERVE_SIZE
row_log_t
Add index, crypt_tail, crypt_head to be used in case of
encryption.
row_log_online_op, row_log_table_close_func
Before writing a block encrypt it if encryption is enabled
row_log_table_apply_ops, row_log_apply_ops
After reading a block decrypt it if encryption is enabled
row_log_allocate
Allocate temporary buffers crypt_head and crypt_tail
if needed.
row_log_free
Free temporary buffers crypt_head and crypt_tail if they
exist.
row_merge_encrypt_buf, row_merge_decrypt_buf
Removed.
row_merge_buf_create, row_merge_buf_write
Remove ROW_MERGE_RESERVE_SIZE
row_merge_build_indexes
Allocate temporary buffer used in decryption and encryption
if needed.
log_tmp_blocks_crypt, log_tmp_block_encrypt, log_temp_block_decrypt
New functions used in block encryption and decryption
log_tmp_is_encrypted
New function to check is encryption enabled.
Added test case innodb-rowlog to force creating a row log and
verify that operations are done using introduced status
variables.
23 files changed, 971 insertions, 467 deletions
diff --git a/mysql-test/suite/encryption/r/innodb-rowlog.result b/mysql-test/suite/encryption/r/innodb-rowlog.result new file mode 100644 index 00000000000..873041ab421 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb-rowlog.result @@ -0,0 +1,30 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +create table t1(id int not null primary key auto_increment, credit_card varchar(200), private varchar(50)) engine=innodb; +SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; +alter table t1 add index secret (credit_card), ALGORITHM=INPLACE, LOCK=NONE; +SET GLOBAL innodb_encryption_rotate_key_age = 1; +create table t2(id int) engine=innodb; +SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; +alter table t1 add index secret2 (private), ALGORITHM=INPLACE, LOCK=NONE; +insert into t1(credit_card) select credit_card from t1; +insert into t1(credit_card) select credit_card from t1; +insert into t1(credit_card) select credit_card from t1; +drop table t2; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; +variable_value > 0 +1 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'; +variable_value > 0 +1 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'; +variable_value > 0 +1 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'; +variable_value > 0 +1 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'; +variable_value > 0 +1 +drop table t1; +SET GLOBAL innodb_file_format=Antelope; +SET GLOBAL innodb_encryption_rotate_key_age=15; diff --git a/mysql-test/suite/encryption/t/innodb-rowlog.opt b/mysql-test/suite/encryption/t/innodb-rowlog.opt new file mode 100644 index 00000000000..6338ddbde35 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-rowlog.opt @@ -0,0 +1,7 @@ +--aria-encrypt-tables +--encrypt-tmp-disk-tables +--innodb-encrypt-tables +--innodb-encrypt-log +--innodb-encryption-rotate-key-age=15 +--innodb-encryption-threads=4 +--innodb-tablespaces-encryption diff --git a/mysql-test/suite/encryption/t/innodb-rowlog.test b/mysql-test/suite/encryption/t/innodb-rowlog.test new file mode 100644 index 00000000000..135293ef4b9 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-rowlog.test @@ -0,0 +1,90 @@ +-- source include/have_innodb.inc +-- source include/have_example_key_management_plugin.inc +# needs dbug_dbug +-- source include/have_debug.inc + +--disable_warnings +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $key_age = `SELECT @@innodb_encryption_rotate_key_age`; +--enable_query_log + +SET GLOBAL innodb_file_format = `Barracuda`; +--enable_warnings + +let $MYSQLD_DATADIR = `SELECT @@datadir`; +let $MYSQLD_TMPDIR = `SELECT @@tmpdir`; + +# +# Create a table that will be encrypted and put some sensitive data to it (credit card numbers) +# + +create table t1(id int not null primary key auto_increment, credit_card varchar(200), private varchar(50)) engine=innodb; + +let $rows = 15000; +--disable_query_log +begin; +while ($rows) +{ + eval insert into t1 values(NULL, '0000-0000-0000-0000','private_data'); + dec $rows; +} +commit; +--enable_query_log + +--let $wait_timeout= 600 +--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 +--source include/wait_condition.inc + +# +# Now we create secondary index for credit_card column in parallel we create new rows +# forcing alter table to wait so that row log is used. +# +SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; +send alter table t1 add index secret (credit_card), ALGORITHM=INPLACE, LOCK=NONE; +# +# Force key rotation and create second index for same table +# +connect (con2,localhost,root,,); +SET GLOBAL innodb_encryption_rotate_key_age = 1; +create table t2(id int) engine=innodb; +SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; +send alter table t1 add index secret2 (private), ALGORITHM=INPLACE, LOCK=NONE; +# +# Create new rows to row log +# +connect (con1,localhost,root,,); +insert into t1(credit_card) select credit_card from t1; +insert into t1(credit_card) select credit_card from t1; +insert into t1(credit_card) select credit_card from t1; + +connection default; +reap; +disconnect con1; + +connection con2; +reap; +drop table t2; + +connection default; +disconnect con2; + +let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED'; +--source include/wait_condition.inc + +# +# Verify that both merge blocks and row log blocks are encryted and decrypted +# + +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'; + +drop table t1; + +--disable_warnings +eval SET GLOBAL innodb_file_format=$innodb_file_format_orig; +eval SET GLOBAL innodb_encryption_rotate_key_age=$key_age; +--enable_warnings diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 4733c33a83f..afc320de517 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1087,6 +1087,18 @@ static SHOW_VAR innodb_status_variables[]= { {"encryption_key_rotation_list_length", (char*)&export_vars.innodb_key_rotation_list_length, SHOW_LONGLONG}, + {"encryption_n_merge_blocks_encrypted", + (char*)&export_vars.innodb_n_merge_blocks_encrypted, + SHOW_LONGLONG}, + {"encryption_n_merge_blocks_decrypted", + (char*)&export_vars.innodb_n_merge_blocks_decrypted, + SHOW_LONGLONG}, + {"encryption_n_rowlog_blocks_encrypted", + (char*)&export_vars.innodb_n_rowlog_blocks_encrypted, + SHOW_LONGLONG}, + {"encryption_n_rowlog_blocks_decrypted", + (char*)&export_vars.innodb_n_rowlog_blocks_decrypted, + SHOW_LONGLONG}, /* scrubing */ {"scrub_background_page_reorganizations", diff --git a/storage/innobase/include/log0crypt.h b/storage/innobase/include/log0crypt.h index 6b164e90d6e..0ad7e7d7037 100644 --- a/storage/innobase/include/log0crypt.h +++ b/storage/innobase/include/log0crypt.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -29,6 +29,7 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "univ.i" #include "ut0byte.h" #include "my_crypt.h" +#include "os0file.h" typedef int Crypt_result; @@ -125,4 +126,45 @@ log_crypt_print_checkpoint_keys( /*============================*/ const byte* log_block); +/** Encrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_encrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) + MY_ATTRIBUTE((warn_unused_result)); + +/** Decrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_decrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) + MY_ATTRIBUTE((warn_unused_result)); + +/** Find out is temporary log files encrypted. +@return true if temporary log file should be encrypted, false if not */ +UNIV_INTERN +bool +log_tmp_is_encrypted() MY_ATTRIBUTE((warn_unused_result)); #endif // log0crypt.h diff --git a/storage/innobase/include/row0ftsort.h b/storage/innobase/include/row0ftsort.h index 00bd3317de3..e784fae78b9 100644 --- a/storage/innobase/include/row0ftsort.h +++ b/storage/innobase/include/row0ftsort.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2010, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -72,7 +72,6 @@ struct fts_psort_common_t { store Doc ID during sort, if Doc ID will not be big enough to use 8 bytes value */ - fil_space_crypt_t* crypt_data; /*!< crypt data or NULL */ }; struct fts_psort_t { diff --git a/storage/innobase/include/row0merge.h b/storage/innobase/include/row0merge.h index 04d4010ad48..fea6bd99799 100644 --- a/storage/innobase/include/row0merge.h +++ b/storage/innobase/include/row0merge.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -41,9 +41,6 @@ Created 13/06/2005 Jan Lindstrom #include "lock0types.h" #include "srv0srv.h" -/* Reserve free space from every block for key_version */ -#define ROW_MERGE_RESERVE_SIZE 4 - /* Cluster index read task is mandatory */ #define COST_READ_CLUSTERED_INDEX 1.0 @@ -352,17 +349,16 @@ row_merge_buf_sort( Write a merge block to the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_write( /*============*/ int fd, /*!< in: file descriptor */ ulint offset, /*!< in: offset where to write, in number of row_merge_block_t elements */ const void* buf, /*!< in: data */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ void* crypt_buf, /*!< in: crypt buf or NULL */ - ulint space); /*!< in: space id */ - + ulint space) /*!< in: space id */ + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Empty a sort buffer. @return sort buffer */ @@ -400,10 +396,10 @@ row_merge_sort( const bool update_progress, /*!< in: update progress status variable or not */ const float pct_progress, /*!< in: total progress percent until now */ const float pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ - __attribute__((nonnull(1,2,3,4,5))); + MY_ATTRIBUTE((warn_unused_result)); + /*********************************************************************//** Allocate a sort buffer. @return own: sort buffer */ @@ -433,7 +429,7 @@ row_merge_file_destroy( Read a merge block from the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_read( /*===========*/ int fd, /*!< in: file descriptor */ @@ -441,9 +437,9 @@ row_merge_read( in number of row_merge_block_t elements */ row_merge_block_t* buf, /*!< out: data */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */ - ulint space); /*!< in: space id */ + ulint space) /*!< in: space id */ + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Read a merge record. @@ -462,8 +458,8 @@ row_merge_read_rec( or NULL on end of list (non-NULL on I/O error) */ ulint* offsets,/*!< out: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ - __attribute__((nonnull(1,2,3,4,6,7,8), warn_unused_result)); + MY_ATTRIBUTE((warn_unused_result)); + #endif /* row0merge.h */ diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 79cf9cdd2ab..09af0b2cdd2 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -137,6 +137,14 @@ struct srv_stats_t { ulint_ctr_64_t pages_encrypted; /* Number of pages decrypted */ ulint_ctr_64_t pages_decrypted; + /* Number of merge blocks encrypted */ + ulint_ctr_64_t n_merge_blocks_encrypted; + /* Number of merge blocks decrypted */ + ulint_ctr_64_t n_merge_blocks_decrypted; + /* Number of row log blocks encrypted */ + ulint_ctr_64_t n_rowlog_blocks_encrypted; + /* Number of row log blocks decrypted */ + ulint_ctr_64_t n_rowlog_blocks_decrypted; /** Number of data read in total (in bytes) */ ulint_ctr_1_t data_read; @@ -1033,6 +1041,15 @@ struct export_var_t{ ib_int64_t innodb_pages_decrypted; /*!< Number of pages decrypted */ + /*!< Number of merge blocks encrypted */ + ib_int64_t innodb_n_merge_blocks_encrypted; + /*!< Number of merge blocks decrypted */ + ib_int64_t innodb_n_merge_blocks_decrypted; + /*!< Number of row log blocks encrypted */ + ib_int64_t innodb_n_rowlog_blocks_encrypted; + /*!< Number of row log blocks decrypted */ + ib_int64_t innodb_n_rowlog_blocks_decrypted; + ulint innodb_sec_rec_cluster_reads; /*!< srv_sec_rec_cluster_reads */ ulint innodb_sec_rec_cluster_reads_avoided;/*!< srv_sec_rec_cluster_reads_avoided */ diff --git a/storage/innobase/log/log0crypt.cc b/storage/innobase/log/log0crypt.cc index af9b2349187..fe1b41202ea 100644 --- a/storage/innobase/log/log0crypt.cc +++ b/storage/innobase/log/log0crypt.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -237,6 +237,129 @@ next: return rc; } +/** Encrypt/decrypt temporary log blocks. + +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] what ENCRYPTION_FLAG_ENCRYPT or + ENCRYPTION_FLAG_DECRYPT +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +static +bool +log_tmp_blocks_crypt( + const byte* src_block, + ulint size, + byte* dst_block, + int what, + os_offset_t offs, + ulint space_id) +{ + Crypt_result rc = MY_AES_OK; + uint dst_len; + byte aes_ctr_counter[MY_AES_BLOCK_SIZE]; + byte is_encrypt= what == ENCRYPTION_FLAG_ENCRYPT; + const crypt_info_t* info = static_cast<const crypt_info_t*>(&crypt_info[0]); + + // AES_CTR_COUNTER = space_id + offs + + bzero(aes_ctr_counter, MY_AES_BLOCK_SIZE); + mach_write_to_8(aes_ctr_counter, space_id); + mach_write_to_8(aes_ctr_counter + 8, offs); + + rc = encryption_crypt(src_block, size, + dst_block, &dst_len, + (unsigned char*)(info->crypt_key), 16, + aes_ctr_counter, MY_AES_BLOCK_SIZE, + what | ENCRYPTION_FLAG_NOPAD, + LOG_DEFAULT_ENCRYPTION_KEY, + info->key_version); + + if (rc != MY_AES_OK) { + ib_logf(IB_LOG_LEVEL_ERROR, + "%s failed for temporary log file with rc = %d", + is_encrypt ? "Encryption" : "Decryption", + rc); + return false; + } + + return true; +} + +/** Get crypt info +@return pointer to log crypt info or NULL +*/ +inline +const crypt_info_t* +get_crypt_info() +{ + mutex_enter(&log_sys->mutex); + const crypt_info_t* info = get_crypt_info(log_sys->next_checkpoint_no); + mutex_exit(&log_sys->mutex); + + return info; +} + +/** Find out is temporary log files encrypted. +@return true if temporary log file should be encrypted, false if not */ +UNIV_INTERN +bool +log_tmp_is_encrypted() +{ + const crypt_info_t* info = get_crypt_info(); + + if (info == NULL || info->key_version == UNENCRYPTED_KEY_VER) { + return false; + } + + return true; +} + +/** Encrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_encrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) +{ + return (log_tmp_blocks_crypt(src_block, size, dst_block, + ENCRYPTION_FLAG_ENCRYPT, offs, space_id)); +} + +/** Decrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_decrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) +{ + return (log_tmp_blocks_crypt(src_block, size, dst_block, + ENCRYPTION_FLAG_DECRYPT, offs, space_id)); +} + /*********************************************************************//** Generate crypt key from crypt msg. @return true if successfull, false if not. */ diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index 62271e766cf..40ddb3a6a87 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -39,7 +39,7 @@ Created 10/13/2010 Jimmy Yang b[N] = row_merge_read_rec( \ block[N], buf[N], b[N], index, \ fd[N], &foffs[N], &mrec[N], offsets[N], \ - crypt_data, crypt_block[N], space); \ + crypt_block[N], space); \ if (UNIV_UNLIKELY(!b[N])) { \ if (mrec[N]) { \ goto exit; \ @@ -191,7 +191,6 @@ row_fts_psort_info_init( fts_psort_t* merge_info = NULL; ulint block_size; ibool ret = TRUE; - fil_space_crypt_t* crypt_data = NULL; bool encrypted = false; block_size = 3 * srv_sort_buf_size; @@ -222,21 +221,8 @@ row_fts_psort_info_init( common_info->merge_event = os_event_create(); common_info->opt_doc_id_size = opt_doc_id_size; - /* Theoretically the tablespace can be dropped straight away. - In practice, the DDL completion will wait for this thread to - finish. */ - if (fil_space_t* space = fil_space_acquire(new_table->space)) { - crypt_data = space->crypt_data; - fil_space_release(space); - } - - if (crypt_data && crypt_data->should_encrypt()) { - common_info->crypt_data = crypt_data; + if (log_tmp_is_encrypted()) { encrypted = true; - } else { - /* Not needed */ - common_info->crypt_data = NULL; - crypt_data = NULL; } ut_ad(trx->mysql_thd != NULL); @@ -574,11 +560,9 @@ row_merge_fts_doc_tokenize( cur_len += len; dfield_dup(field, buf->heap); - /* Reserve one byte for the end marker of row_merge_block_t - and we have reserved ROW_MERGE_RESERVE_SIZE (= 4) for - encryption key_version in the beginning of the buffer. */ + /* Reserve one byte for the end marker of row_merge_block_t */ if (buf->total_size + data_size[idx] + cur_len - >= (srv_sort_buf_size - 1 - ROW_MERGE_RESERVE_SIZE)) { + >= (srv_sort_buf_size - 1)) { buf_full = TRUE; break; @@ -672,7 +656,6 @@ fts_parallel_tokenization( fts_tokenize_ctx_t t_ctx; ulint retried = 0; dberr_t error = DB_SUCCESS; - fil_space_crypt_t* crypt_data = NULL; ut_ad(psort_info->psort_common->trx->mysql_thd != NULL); @@ -693,7 +676,6 @@ fts_parallel_tokenization( block = psort_info->merge_block; crypt_block = psort_info->crypt_block; - crypt_data = psort_info->psort_common->crypt_data; zip_size = dict_table_zip_size(table); row_merge_fts_get_next_doc_item(psort_info, &doc_item); @@ -788,7 +770,6 @@ loop: if (!row_merge_write(merge_file[t_ctx.buf_used]->fd, merge_file[t_ctx.buf_used]->offset++, block[t_ctx.buf_used], - crypt_data, crypt_block[t_ctx.buf_used], table->space)) { error = DB_TEMP_FILE_WRITE_FAILURE; @@ -884,7 +865,6 @@ exit: if (!row_merge_write(merge_file[i]->fd, merge_file[i]->offset++, block[i], - crypt_data, crypt_block[i], table->space)) { error = DB_TEMP_FILE_WRITE_FAILURE; @@ -924,7 +904,7 @@ exit: psort_info->psort_common->dup, merge_file[i], block[i], &tmpfd[i], false, 0.0/* pct_progress */, 0.0/* pct_cost */, - crypt_data, crypt_block[i], table->space); + crypt_block[i], table->space); if (error != DB_SUCCESS) { close(tmpfd[i]); @@ -1436,7 +1416,6 @@ row_fts_merge_insert( ulint start; fts_psort_insert_t ins_ctx; ulint count_diag = 0; - fil_space_crypt_t* crypt_data = NULL; ulint space; ut_ad(index); @@ -1450,7 +1429,6 @@ row_fts_merge_insert( ins_ctx.trx->op_info = "inserting index entries"; ins_ctx.opt_doc_id_size = psort_info[0].psort_common->opt_doc_id_size; - crypt_data = psort_info[0].psort_common->crypt_data; heap = mem_heap_create(500 + sizeof(mrec_buf_t)); @@ -1544,7 +1522,6 @@ row_fts_merge_insert( && (!row_merge_read( fd[i], foffs[i], (row_merge_block_t*) block[i], - crypt_data, (row_merge_block_t*) crypt_block[i], space))) { error = DB_CORRUPTION; diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc index 88398c20342..ec6b4a60680 100644 --- a/storage/innobase/row/row0log.cc +++ b/storage/innobase/row/row0log.cc @@ -182,6 +182,7 @@ struct row_log_t { dict_table_t* table; /*!< table that is being rebuilt, or NULL when this is a secondary index that is being created online */ + dict_index_t* index; /*!< index to be build */ bool same_pk;/*!< whether the definition of the PRIMARY KEY has remained the same */ const dtuple_t* add_cols; @@ -197,8 +198,14 @@ struct row_log_t { row_log_buf_t tail; /*!< writer context; protected by mutex and index->lock S-latch, or by index->lock X-latch only */ + byte* crypt_tail; /*!< writer context; + temporary buffer used in encryption, + decryption or NULL*/ row_log_buf_t head; /*!< reader context; protected by MDL only; modifiable by row_log_apply_ops() */ + byte* crypt_head; /*!< reader context; + temporary buffer used in encryption, + decryption or NULL */ const char* path; /*!< where to create temporary file during log operation */ }; @@ -349,6 +356,7 @@ row_log_online_op( = (os_offset_t) log->tail.blocks * srv_sort_buf_size; ibool ret; + byte * buf = log->tail.block; if (byte_offset + srv_sort_buf_size >= srv_online_max_size) { goto write_failed; @@ -368,11 +376,29 @@ row_log_online_op( goto err_exit; } + /* If encryption is enabled encrypt buffer before writing it + to file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt(log->tail.block, + srv_sort_buf_size, + log->crypt_tail, + byte_offset, + index->table->space)) { + log->error = DB_DECRYPTION_FAILED; + goto err_exit; + } + + srv_stats.n_rowlog_blocks_encrypted.inc(); + buf = log->crypt_tail; + } + ret = os_file_write_int_fd( "(modification log)", log->fd, - log->tail.block, byte_offset, srv_sort_buf_size); + buf, byte_offset, srv_sort_buf_size); + log->tail.blocks++; + if (!ret) { write_failed: /* We set the flag directly instead of invoking @@ -380,7 +406,9 @@ write_failed: because the index is not "public" yet. */ index->type |= DICT_CORRUPT; } + UNIV_MEM_INVALID(log->tail.block, srv_sort_buf_size); + memcpy(log->tail.block, log->tail.buf + avail_size, mrec_size - avail_size); log->tail.bytes = mrec_size - avail_size; @@ -465,6 +493,7 @@ row_log_table_close_func( = (os_offset_t) log->tail.blocks * srv_sort_buf_size; ibool ret; + byte * buf = log->tail.block; if (byte_offset + srv_sort_buf_size >= srv_online_max_size) { goto write_failed; @@ -484,11 +513,29 @@ row_log_table_close_func( goto err_exit; } + /* If encryption is enabled encrypt buffer before writing it + to file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt(log->tail.block, + srv_sort_buf_size, + log->crypt_tail, + byte_offset, + log->index->table->space)) { + log->error = DB_DECRYPTION_FAILED; + goto err_exit; + } + + srv_stats.n_rowlog_blocks_encrypted.inc(); + buf = log->crypt_tail; + } + ret = os_file_write_int_fd( "(modification log)", log->fd, - log->tail.block, byte_offset, srv_sort_buf_size); + buf, byte_offset, srv_sort_buf_size); + log->tail.blocks++; + if (!ret) { write_failed: log->error = DB_ONLINE_LOG_TOO_BIG; @@ -2622,10 +2669,29 @@ all_done: goto func_exit; } + byte * buf = index->online_log->head.block; + success = os_file_read_no_error_handling_int_fd( index->online_log->fd, - index->online_log->head.block, ofs, + buf, ofs, srv_sort_buf_size); + + /* If encryption is enabled decrypt buffer after reading it + from file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, + srv_sort_buf_size, + index->online_log->crypt_head, + ofs, + index->table->space)) { + error = DB_DECRYPTION_FAILED; + goto func_exit; + } + + srv_stats.n_rowlog_blocks_decrypted.inc(); + memcpy(buf, index->online_log->crypt_head, srv_sort_buf_size); + } + if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" " for table %s\n", index->table_name); @@ -2932,9 +2998,22 @@ row_log_allocate( log->head.blocks = log->head.bytes = 0; log->head.total = 0; log->path = path; + log->crypt_tail = log->crypt_head = NULL; + log->index = index; dict_index_set_online_status(index, ONLINE_INDEX_CREATION); index->online_log = log; + if (log_tmp_is_encrypted()) { + ulint size = srv_sort_buf_size; + log->crypt_head = static_cast<byte *>(os_mem_alloc_large(&size)); + log->crypt_tail = static_cast<byte *>(os_mem_alloc_large(&size)); + + if (!log->crypt_head || !log->crypt_tail) { + row_log_free(log); + DBUG_RETURN(false); + } + } + /* While we might be holding an exclusive data dictionary lock here, in row_log_abort_sec() we will not always be holding it. Use atomic operations in both cases. */ @@ -2957,6 +3036,15 @@ row_log_free( row_log_block_free(log->tail); row_log_block_free(log->head); row_merge_file_destroy_low(log->fd); + + if (log->crypt_head) { + os_mem_free_large(log->crypt_head, srv_sort_buf_size); + } + + if (log->crypt_tail) { + os_mem_free_large(log->crypt_tail, srv_sort_buf_size); + } + mutex_free(&log->mutex); ut_free(log); log = 0; @@ -3451,11 +3539,29 @@ all_done: goto func_exit; } + byte* buf = index->online_log->head.block; + success = os_file_read_no_error_handling_int_fd( index->online_log->fd, - index->online_log->head.block, ofs, + buf, ofs, srv_sort_buf_size); + /* If encryption is enabled decrypt buffer after reading it + from file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, + srv_sort_buf_size, + index->online_log->crypt_head, + ofs, + index->table->space)) { + error = DB_DECRYPTION_FAILED; + goto func_exit; + } + + srv_stats.n_rowlog_blocks_decrypted.inc(); + memcpy(buf, index->online_log->crypt_head, srv_sort_buf_size); + } + if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" " for index %s\n", index->name + 1); diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 65569eabf57..a91c0f67218 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -79,88 +79,6 @@ UNIV_INTERN char srv_disable_sort_file_cache; /* Maximum pending doc memory limit in bytes for a fts tokenization thread */ #define FTS_PENDING_DOC_MEMORY_LIMIT 1000000 - -/******************************************************//** -Encrypt a merge block. */ -static -void -row_merge_encrypt_buf( -/*==================*/ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ - ulint offset, /*!< in: offset where to - write */ - ulint space, /*!< in: tablespace id */ - const byte* input_buf, /*!< in: input buffer */ - byte* crypted_buf) /*!< out: crypted buffer */ -{ - uint key_version; - uint dstlen=0; - os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset; - - key_version = encryption_key_get_latest_version(crypt_data->key_id); - - /* Store key_version at the begining of the input buffer */ - mach_write_to_4((byte *)crypted_buf, key_version); - - int rc = encryption_scheme_encrypt(input_buf+ROW_MERGE_RESERVE_SIZE, - srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE, - crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen, - crypt_data, key_version, - space, ofs, 0); - - if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Unable to encrypt data-block " - " src: %p srclen: %lu buf: %p buflen: %d." - " return-code: %d. Can't continue!\n", - input_buf, srv_sort_buf_size, - crypted_buf, dstlen, rc); - } -} - -/******************************************************//** -Decrypt a merge block. */ -static -bool -row_merge_decrypt_buf( -/*==================*/ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ - ulint offset, /*!< in: offset where to - write */ - ulint space, /*!< in: tablespace id */ - const byte* input_buf, /*!< in: input buffer */ - byte* crypted_buf) /*!< out: crypted buffer */ -{ - uint key_version; - uint dstlen=0; - os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset; - - /* Read key_version from begining of the buffer */ - key_version = mach_read_from_4((byte *)input_buf); - - if (key_version == 0) { - /* block not encrypted */ - return false; - } - - int rc = encryption_scheme_decrypt(input_buf+ROW_MERGE_RESERVE_SIZE, - srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE, - crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen, - crypt_data, key_version, - space, ofs, 0); - - if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Unable to encrypt data-block " - " src: %p srclen: %lu buf: %p buflen: %d." - " return-code: %d. Can't continue!\n", - input_buf, srv_sort_buf_size, - crypted_buf, dstlen, rc); - } - - return true; -} - #ifdef UNIV_DEBUG /******************************************************//** Display a merge tuple. */ @@ -279,7 +197,7 @@ row_merge_buf_create( ulint buf_size; mem_heap_t* heap; - max_tuples = (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE) + max_tuples = (srv_sort_buf_size) / ut_max(1, dict_index_get_min_size(index)); buf_size = (sizeof *buf); @@ -695,7 +613,7 @@ row_merge_buf_add( ut_ad(data_size < srv_sort_buf_size); /* Reserve bytes for the end marker of row_merge_block_t. */ - if (buf->total_size + data_size >= (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE)) { + if (buf->total_size + data_size >= srv_sort_buf_size) { DBUG_RETURN(0); } @@ -816,7 +734,7 @@ UT_SORT_FUNCTION_BODY(). /**********************************************************************//** Merge sort the tuple buffer in main memory. */ -static MY_ATTRIBUTE((nonnull(4,5))) +static void row_merge_tuple_sort( /*=================*/ @@ -867,7 +785,7 @@ row_merge_buf_write( { const dict_index_t* index = buf->index; ulint n_fields= dict_index_get_n_fields(index); - byte* b = &block[ROW_MERGE_RESERVE_SIZE]; + byte* b = &block[0]; for (ulint i = 0; i < buf->n_tuples; i++) { const mtuple_t* entry = &buf->tuples[i]; @@ -886,7 +804,7 @@ row_merge_buf_write( /* Write an "end-of-chunk" marker. */ ut_a(b < &block[srv_sort_buf_size]); - ut_a(b == &block[0] + buf->total_size + ROW_MERGE_RESERVE_SIZE); + ut_a(b == &block[0] + buf->total_size); *b++ = 0; #ifdef UNIV_DEBUG_VALGRIND /* The rest of the block is uninitialized. Initialize it @@ -936,7 +854,7 @@ row_merge_heap_create( Read a merge block from the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_read( /*===========*/ int fd, /*!< in: file descriptor */ @@ -944,12 +862,11 @@ row_merge_read( in number of row_merge_block_t elements */ row_merge_block_t* buf, /*!< out: data */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { os_offset_t ofs = ((os_offset_t) offset) * srv_sort_buf_size; - ibool success; + bool success; DBUG_EXECUTE_IF("row_merge_read_failure", return(FALSE);); @@ -964,10 +881,14 @@ row_merge_read( ofs, srv_sort_buf_size); /* For encrypted tables, decrypt data after reading and copy data */ - if (crypt_data && crypt_buf) { - if (row_merge_decrypt_buf(crypt_data, offset, space, buf, crypt_buf)) { - memcpy(buf, crypt_buf, srv_sort_buf_size); + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, srv_sort_buf_size, + crypt_buf, ofs, space)) { + return (FALSE); } + + srv_stats.n_merge_blocks_decrypted.inc(); + memcpy(buf, crypt_buf, srv_sort_buf_size); } #ifdef POSIX_FADV_DONTNEED @@ -989,31 +910,32 @@ row_merge_read( Write a merge block to the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_write( /*============*/ int fd, /*!< in: file descriptor */ ulint offset, /*!< in: offset where to write, in number of row_merge_block_t elements */ const void* buf, /*!< in: data */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ void* crypt_buf, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ { size_t buf_len = srv_sort_buf_size; os_offset_t ofs = buf_len * (os_offset_t) offset; - ibool ret; + bool ret; void* out_buf = (void *)buf; DBUG_EXECUTE_IF("row_merge_write_failure", return(FALSE);); /* For encrypted tables, encrypt data before writing */ - if (crypt_data && crypt_buf) { - row_merge_encrypt_buf(crypt_data, offset, space, (const byte *)buf, (byte *)crypt_buf); + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt((const byte *)buf, buf_len, + (byte *)crypt_buf, ofs, space)) { + return (FALSE); + } + + srv_stats.n_merge_blocks_encrypted.inc(); out_buf = crypt_buf; - } else { - /* Mark block unencrypted */ - mach_write_to_4((byte *)out_buf, 0); } ret = os_file_write_int_fd("(merge)", fd, out_buf, ofs, buf_len); @@ -1051,9 +973,8 @@ row_merge_read_rec( or NULL on end of list (non-NULL on I/O error) */ ulint* offsets,/*!< out: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { ulint extra_size; ulint data_size; @@ -1065,10 +986,6 @@ row_merge_read_rec( ut_ad(*offsets == 1 + REC_OFFS_HEADER_SIZE + dict_index_get_n_fields(index)); - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - extra_size = *b++; if (UNIV_UNLIKELY(!extra_size)) { @@ -1090,7 +1007,8 @@ row_merge_read_rec( if (UNIV_UNLIKELY(b >= &block[srv_sort_buf_size])) { if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { err_exit: /* Signal I/O error. */ *mrec = b; @@ -1098,7 +1016,7 @@ err_exit: } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; } extra_size = (extra_size & 0x7f) << 8; @@ -1120,13 +1038,14 @@ err_exit: memcpy(*buf, b, avail_size); if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { goto err_exit; } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; /* Copy the record. */ memcpy(*buf + avail_size, b, extra_size - avail_size); @@ -1182,13 +1101,14 @@ err_exit: #endif /* UNIV_DEBUG */ if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { goto err_exit; } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; /* Copy the rest of the record. */ memcpy(*buf + avail_size, b, extra_size + data_size - avail_size); @@ -1265,7 +1185,6 @@ row_merge_write_rec( ulint* foffs, /*!< in/out: file offset */ const mrec_t* mrec, /*!< in: record to write */ const ulint* offsets,/*!< in: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ { @@ -1288,10 +1207,6 @@ row_merge_write_rec( size = extra_size + (extra_size >= 0x80) + rec_offs_data_size(offsets); - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - if (UNIV_UNLIKELY(b + size >= &block[srv_sort_buf_size])) { /* The record spans two blocks. Copy it to the temporary buffer first. */ @@ -1307,14 +1222,15 @@ row_merge_write_rec( memcpy(b, buf[0], avail_size); if (!row_merge_write(fd, (*foffs)++, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(NULL); } UNIV_MEM_INVALID(&block[0], srv_sort_buf_size); /* Copy the rest. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; memcpy(b, buf[0] + avail_size, size - avail_size); b += size - avail_size; } else { @@ -1337,7 +1253,6 @@ row_merge_write_eof( byte* b, /*!< in: pointer to end of block */ int fd, /*!< in: file descriptor */ ulint* foffs, /*!< in/out: file offset */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ { @@ -1352,10 +1267,6 @@ row_merge_write_eof( } #endif /* UNIV_DEBUG */ - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - *b++ = 0; UNIV_MEM_ASSERT_RW(&block[0], b - &block[0]); UNIV_MEM_ASSERT_W(&block[0], srv_sort_buf_size); @@ -1367,7 +1278,8 @@ row_merge_write_eof( #endif /* UNIV_DEBUG_VALGRIND */ if (!row_merge_write(fd, (*foffs)++, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(NULL); } @@ -1445,10 +1357,9 @@ containing the index entries for the indexes to be built. @param[in,out] block file buffer @param[in,out] tmpfd temporary file handle @param[in] pct_cost percent of task weight out of total alter job -@param[in] crypt_data crypt data or NULL @param[in,out] crypt_block crypted file buffer -return DB_SUCCESS or error */ -static MY_ATTRIBUTE((nonnull(1,2,3,4,6,9,10,16), warn_unused_result)) +@return DB_SUCCESS or error */ +static MY_ATTRIBUTE((warn_unused_result)) dberr_t row_merge_read_clustered_index( trx_t* trx, @@ -1469,7 +1380,6 @@ row_merge_read_clustered_index( row_merge_block_t* block, int* tmpfd, float pct_cost, - fil_space_crypt_t* crypt_data, row_merge_block_t* crypt_block) { dict_index_t* clust_index; /* Clustered index */ @@ -2004,8 +1914,8 @@ write_buffers: row_merge_buf_write(buf, file, block); if (!row_merge_write(file->fd, file->offset++, - block, crypt_data, crypt_block, - new_table->space)) { + block, crypt_block, + new_table->space)) { err = DB_TEMP_FILE_WRITE_FAILURE; trx->error_key_num = i; break; @@ -2054,7 +1964,7 @@ write_buffers: if(read_rows % 1000 == 0) { /* Update progress for each 1000 rows */ curr_progress = (read_rows >= table_total_rows) ? - pct_cost : + pct_cost : ((pct_cost * read_rows) / table_total_rows); /* presenting 10.12% as 1012 integer */ onlineddl_pct_progress = curr_progress * 100; @@ -2183,9 +2093,8 @@ wait_again: &buf[2], b2, \ of->fd, &of->offset, \ mrec##N, offsets##N, \ - crypt_data, \ crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL , \ - space); \ + space); \ if (UNIV_UNLIKELY(!b2 || ++of->n_rec > file->n_rec)) { \ goto corrupt; \ } \ @@ -2193,7 +2102,6 @@ wait_again: &buf[N], b##N, INDEX, \ file->fd, foffs##N, \ &mrec##N, offsets##N, \ - crypt_data, \ crypt_block ? &crypt_block[N * srv_sort_buf_size] : NULL, \ space); \ \ @@ -2208,7 +2116,7 @@ wait_again: /*************************************************************//** Merge two blocks of records on disk and write a bigger block. @return DB_SUCCESS or error code */ -static __attribute__((nonnull(1,2,3,4,5,6), warn_unused_result)) +static __attribute__((warn_unused_result)) dberr_t row_merge_blocks( /*=============*/ @@ -2222,7 +2130,6 @@ row_merge_blocks( ulint* foffs1, /*!< in/out: offset of second source list in the file */ merge_file_t* of, /*!< in/out: output file */ - fil_space_crypt_t* crypt_data,/*!< in: crypt data or NULL */ row_merge_block_t* crypt_block,/*!< in: in/out: crypted file buffer */ ulint space) /*!< in: space id */ @@ -2258,9 +2165,11 @@ row_merge_blocks( file in two halves, which can be merged on the following pass. */ if (!row_merge_read(file->fd, *foffs0, &block[0], - crypt_data, crypt_block ? &crypt_block[0] : NULL, space) - || !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size], - crypt_data, crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, space)) { + crypt_block ? &crypt_block[0] : NULL, + space) || + !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size], + crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, + space)) { corrupt: mem_heap_free(heap); return(DB_CORRUPTION); @@ -2273,13 +2182,15 @@ corrupt: b0 = row_merge_read_rec( &block[0], &buf[0], b0, dup->index, file->fd, foffs0, &mrec0, offsets0, - crypt_data, crypt_block ? &crypt_block[0] : NULL, space); + crypt_block ? &crypt_block[0] : NULL, + space); b1 = row_merge_read_rec( &block[srv_sort_buf_size], &buf[srv_sort_buf_size], b1, dup->index, file->fd, foffs1, &mrec1, offsets1, - crypt_data, crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, space); + crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, + space); if (UNIV_UNLIKELY(!b0 && mrec0) || UNIV_UNLIKELY(!b1 && mrec1)) { @@ -2325,7 +2236,8 @@ done1: b2 = row_merge_write_eof(&block[2 * srv_sort_buf_size], b2, of->fd, &of->offset, - crypt_data, crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, space); + crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, + space); return(b2 ? DB_SUCCESS : DB_CORRUPTION); } @@ -2333,8 +2245,8 @@ done1: /*************************************************************//** Copy a block of index entries. @return TRUE on success, FALSE on failure */ -static __attribute__((nonnull(1,2,3,4,5), warn_unused_result)) -ibool +static __attribute__((warn_unused_result)) +bool row_merge_blocks_copy( /*==================*/ const dict_index_t* index, /*!< in: index being created */ @@ -2342,9 +2254,8 @@ row_merge_blocks_copy( row_merge_block_t* block, /*!< in/out: 3 buffers */ ulint* foffs0, /*!< in/out: input file offset */ merge_file_t* of, /*!< in/out: output file */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { mem_heap_t* heap; /*!< memory heap for offsets0, offsets1 */ @@ -2372,7 +2283,8 @@ row_merge_blocks_copy( file in two halves, which can be merged on the following pass. */ if (!row_merge_read(file->fd, *foffs0, &block[0], - crypt_data, crypt_block ? &crypt_block[0] : NULL, space)) { + crypt_block ? &crypt_block[0] : NULL, + space)) { corrupt: mem_heap_free(heap); return(FALSE); @@ -2384,7 +2296,8 @@ corrupt: b0 = row_merge_read_rec(&block[0], &buf[0], b0, index, file->fd, foffs0, &mrec0, offsets0, - crypt_data, crypt_block ? &crypt_block[0] : NULL, space); + crypt_block ? &crypt_block[0] : NULL, + space); if (UNIV_UNLIKELY(!b0 && mrec0)) { @@ -2407,15 +2320,15 @@ done0: return(row_merge_write_eof(&block[2 * srv_sort_buf_size], b2, of->fd, &of->offset, - crypt_data, - crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, space) + crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, + space) != NULL); } /*************************************************************//** Merge disk files. @return DB_SUCCESS or error code */ -static __attribute__((nonnull(1,2,3,4,5,6,7))) +static dberr_t row_merge( /*======*/ @@ -2431,9 +2344,8 @@ row_merge( ulint* run_offset, /*!< in/out: Array contains the first offset number for each merge run */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { ulint foffs0; /*!< first input offset */ ulint foffs1; /*!< second input offset */ @@ -2481,7 +2393,8 @@ row_merge( error = row_merge_blocks(dup, file, block, &foffs0, &foffs1, &of, - crypt_data, crypt_block, space); + crypt_block, + space); if (error != DB_SUCCESS) { return(error); @@ -2502,7 +2415,8 @@ row_merge( if (!row_merge_blocks_copy(dup->index, file, block, &foffs0, &of, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(DB_CORRUPTION); } } @@ -2520,7 +2434,8 @@ row_merge( if (!row_merge_blocks_copy(dup->index, file, block, &foffs1, &of, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(DB_CORRUPTION); } } @@ -2576,9 +2491,8 @@ row_merge_sort( /*!< in: total progress percent until now */ const float pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { const ulint half = file->offset / 2; ulint num_runs; @@ -2646,7 +2560,8 @@ row_merge_sort( error = row_merge(trx, dup, file, block, tmpfd, &num_runs, run_offset, - crypt_data, crypt_block, space); + crypt_block, + space); if(update_progress) { merge_count++; @@ -2723,7 +2638,7 @@ row_merge_copy_blobs( Read sorted file containing index data tuples and insert these data tuples to the index @return DB_SUCCESS or error number */ -static __attribute__((nonnull(2,3,5), warn_unused_result)) +static __attribute__((warn_unused_result)) dberr_t row_merge_insert_index_tuples( /*==========================*/ @@ -2736,9 +2651,8 @@ row_merge_insert_index_tuples( const float pct_progress, /*!< in: total progress percent until now */ const float pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { const byte* b; mem_heap_t* heap; @@ -2772,7 +2686,8 @@ row_merge_insert_index_tuples( b = &block[0]; if (!row_merge_read(fd, foffs, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { error = DB_CORRUPTION; } else { buf = static_cast<mrec_buf_t*>( @@ -2789,7 +2704,9 @@ row_merge_insert_index_tuples( b = row_merge_read_rec(block, buf, b, index, fd, &foffs, &mrec, offsets, - crypt_data, crypt_block, space); + crypt_block, + space); + if (UNIV_UNLIKELY(!b)) { /* End of list, or I/O error */ if (mrec) { @@ -3974,7 +3891,6 @@ row_merge_build_indexes( fts_psort_t* merge_info = NULL; ib_int64_t sig_count = 0; bool fts_psort_initiated = false; - fil_space_crypt_t * crypt_data = NULL; float total_static_cost = 0; float total_dynamic_cost = 0; @@ -4003,25 +3919,20 @@ row_merge_build_indexes( from concurrent DDL (e.g. drop table) by MDL-locks. */ fil_space_t* space = fil_space_acquire(new_table->space); - if (space) { - crypt_data = space->crypt_data; - } else { + if (!space) { DBUG_RETURN(DB_TABLESPACE_NOT_FOUND); } /* If tablespace is encrypted, allocate additional buffer for encryption/decryption. */ - if (crypt_data && crypt_data->should_encrypt()) { + if (log_tmp_is_encrypted()) { crypt_block = static_cast<row_merge_block_t*>( - os_mem_alloc_large(&block_size)); + os_mem_alloc_large(&block_size)); if (crypt_block == NULL) { fil_space_release(space); DBUG_RETURN(DB_OUT_OF_MEMORY); } - } else { - /* Not needed */ - crypt_data = NULL; } trx_start_if_not_started_xa(trx); @@ -4103,7 +4014,7 @@ row_merge_build_indexes( fts_sort_idx, psort_info, merge_files, key_numbers, n_indexes, add_cols, col_map, add_autoinc, sequence, block, &tmpfd, - pct_cost, crypt_data, crypt_block); + pct_cost, crypt_block); pct_progress += pct_cost; @@ -4240,7 +4151,8 @@ wait_again: trx, &dup, &merge_files[i], block, &tmpfd, true, pct_progress, pct_cost, - crypt_data, crypt_block, new_table->space); + crypt_block, + new_table->space); pct_progress += pct_cost; @@ -4278,7 +4190,8 @@ wait_again: trx->id, sort_idx, old_table, merge_files[i].fd, block, merge_files[i].n_rec, pct_progress, pct_cost, - crypt_data, crypt_block, new_table->space); + crypt_block, new_table->space); + pct_progress += pct_cost; if (global_system_variables.log_warnings > 2) { diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 7ddd934a9a1..1431211279b 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -1619,6 +1619,10 @@ srv_export_innodb_status(void) export_vars.innodb_pages_page_compression_error = srv_stats.pages_page_compression_error; export_vars.innodb_pages_decrypted = srv_stats.pages_decrypted; export_vars.innodb_pages_encrypted = srv_stats.pages_encrypted; + export_vars.innodb_n_merge_blocks_encrypted = srv_stats.n_merge_blocks_encrypted; + export_vars.innodb_n_merge_blocks_decrypted = srv_stats.n_merge_blocks_decrypted; + export_vars.innodb_n_rowlog_blocks_encrypted = srv_stats.n_rowlog_blocks_encrypted; + export_vars.innodb_n_rowlog_blocks_decrypted = srv_stats.n_rowlog_blocks_decrypted; export_vars.innodb_defragment_compression_failures = btr_defragment_compression_failures; diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index caf7a3ef412..4af523b1018 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -1309,6 +1309,18 @@ static SHOW_VAR innodb_status_variables[]= { {"encryption_key_rotation_list_length", (char*)&export_vars.innodb_key_rotation_list_length, SHOW_LONGLONG}, + {"encryption_n_merge_blocks_encrypted", + (char*)&export_vars.innodb_n_merge_blocks_encrypted, + SHOW_LONGLONG}, + {"encryption_n_merge_blocks_decrypted", + (char*)&export_vars.innodb_n_merge_blocks_decrypted, + SHOW_LONGLONG}, + {"encryption_n_rowlog_blocks_encrypted", + (char*)&export_vars.innodb_n_rowlog_blocks_encrypted, + SHOW_LONGLONG}, + {"encryption_n_rowlog_blocks_decrypted", + (char*)&export_vars.innodb_n_rowlog_blocks_decrypted, + SHOW_LONGLONG}, /* Scrubing feature */ {"scrub_background_page_reorganizations", diff --git a/storage/xtradb/include/log0crypt.h b/storage/xtradb/include/log0crypt.h index 6b164e90d6e..0ad7e7d7037 100644 --- a/storage/xtradb/include/log0crypt.h +++ b/storage/xtradb/include/log0crypt.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -29,6 +29,7 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include "univ.i" #include "ut0byte.h" #include "my_crypt.h" +#include "os0file.h" typedef int Crypt_result; @@ -125,4 +126,45 @@ log_crypt_print_checkpoint_keys( /*============================*/ const byte* log_block); +/** Encrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_encrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) + MY_ATTRIBUTE((warn_unused_result)); + +/** Decrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_decrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) + MY_ATTRIBUTE((warn_unused_result)); + +/** Find out is temporary log files encrypted. +@return true if temporary log file should be encrypted, false if not */ +UNIV_INTERN +bool +log_tmp_is_encrypted() MY_ATTRIBUTE((warn_unused_result)); #endif // log0crypt.h diff --git a/storage/xtradb/include/row0ftsort.h b/storage/xtradb/include/row0ftsort.h index 7c9ed23645c..b2dd90e7e3b 100644 --- a/storage/xtradb/include/row0ftsort.h +++ b/storage/xtradb/include/row0ftsort.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2010, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, MariaDB Corporation. +Copyright (c) 2016, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -72,7 +72,6 @@ struct fts_psort_common_t { store Doc ID during sort, if Doc ID will not be big enough to use 8 bytes value */ - fil_space_crypt_t* crypt_data; /*!< crypt data or NULL */ }; struct fts_psort_t { diff --git a/storage/xtradb/include/row0merge.h b/storage/xtradb/include/row0merge.h index 04d4010ad48..152a51dafc6 100644 --- a/storage/xtradb/include/row0merge.h +++ b/storage/xtradb/include/row0merge.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -41,9 +41,6 @@ Created 13/06/2005 Jan Lindstrom #include "lock0types.h" #include "srv0srv.h" -/* Reserve free space from every block for key_version */ -#define ROW_MERGE_RESERVE_SIZE 4 - /* Cluster index read task is mandatory */ #define COST_READ_CLUSTERED_INDEX 1.0 @@ -352,17 +349,16 @@ row_merge_buf_sort( Write a merge block to the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_write( /*============*/ int fd, /*!< in: file descriptor */ ulint offset, /*!< in: offset where to write, in number of row_merge_block_t elements */ const void* buf, /*!< in: data */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ void* crypt_buf, /*!< in: crypt buf or NULL */ - ulint space); /*!< in: space id */ - + ulint space) /*!< in: space id */ + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Empty a sort buffer. @return sort buffer */ @@ -400,10 +396,9 @@ row_merge_sort( const bool update_progress, /*!< in: update progress status variable or not */ const float pct_progress, /*!< in: total progress percent until now */ const float pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ - __attribute__((nonnull(1,2,3,4,5))); + ulint space) /*!< in: space id */ + MY_ATTRIBUTE((warn_unused_result)); /*********************************************************************//** Allocate a sort buffer. @return own: sort buffer */ @@ -433,7 +428,7 @@ row_merge_file_destroy( Read a merge block from the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_read( /*===========*/ int fd, /*!< in: file descriptor */ @@ -441,10 +436,9 @@ row_merge_read( in number of row_merge_block_t elements */ row_merge_block_t* buf, /*!< out: data */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */ - ulint space); /*!< in: space id */ - + ulint space) /*!< in: space id */ + MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** Read a merge record. @return pointer to next record, or NULL on I/O error or end of list */ @@ -462,8 +456,8 @@ row_merge_read_rec( or NULL on end of list (non-NULL on I/O error) */ ulint* offsets,/*!< out: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ - __attribute__((nonnull(1,2,3,4,6,7,8), warn_unused_result)); + ulint space) /*!< in: space id */ + MY_ATTRIBUTE((warn_unused_result)); + #endif /* row0merge.h */ diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index 30c125ea269..e7f6350987e 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -137,6 +137,14 @@ struct srv_stats_t { ulint_ctr_64_t pages_encrypted; /* Number of pages decrypted */ ulint_ctr_64_t pages_decrypted; + /* Number of merge blocks encrypted */ + ulint_ctr_64_t n_merge_blocks_encrypted; + /* Number of merge blocks decrypted */ + ulint_ctr_64_t n_merge_blocks_decrypted; + /* Number of row log blocks encrypted */ + ulint_ctr_64_t n_rowlog_blocks_encrypted; + /* Number of row log blocks decrypted */ + ulint_ctr_64_t n_rowlog_blocks_decrypted; /** Number of data read in total (in bytes) */ ulint_ctr_1_t data_read; @@ -1276,6 +1284,15 @@ struct export_var_t{ ib_int64_t innodb_pages_decrypted; /*!< Number of pages decrypted */ + /*!< Number of merge blocks encrypted */ + ib_int64_t innodb_n_merge_blocks_encrypted; + /*!< Number of merge blocks decrypted */ + ib_int64_t innodb_n_merge_blocks_decrypted; + /*!< Number of row log blocks encrypted */ + ib_int64_t innodb_n_rowlog_blocks_encrypted; + /*!< Number of row log blocks decrypted */ + ib_int64_t innodb_n_rowlog_blocks_decrypted; + ulint innodb_sec_rec_cluster_reads; /*!< srv_sec_rec_cluster_reads */ ulint innodb_sec_rec_cluster_reads_avoided;/*!< srv_sec_rec_cluster_reads_avoided */ diff --git a/storage/xtradb/log/log0crypt.cc b/storage/xtradb/log/log0crypt.cc index f6c1416d81a..9509797dde3 100644 --- a/storage/xtradb/log/log0crypt.cc +++ b/storage/xtradb/log/log0crypt.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -236,6 +236,129 @@ next: return rc; } +/** Encrypt/decrypt temporary log blocks. + +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] what ENCRYPTION_FLAG_ENCRYPT or + ENCRYPTION_FLAG_DECRYPT +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +static +bool +log_tmp_blocks_crypt( + const byte* src_block, + ulint size, + byte* dst_block, + int what, + os_offset_t offs, + ulint space_id) +{ + Crypt_result rc = MY_AES_OK; + uint dst_len; + byte aes_ctr_counter[MY_AES_BLOCK_SIZE]; + byte is_encrypt= what == ENCRYPTION_FLAG_ENCRYPT; + const crypt_info_t* info = static_cast<const crypt_info_t*>(&crypt_info[0]); + + // AES_CTR_COUNTER = space_id + offs + + bzero(aes_ctr_counter, MY_AES_BLOCK_SIZE); + mach_write_to_8(aes_ctr_counter, space_id); + mach_write_to_8(aes_ctr_counter + 8, offs); + + rc = encryption_crypt(src_block, size, + dst_block, &dst_len, + (unsigned char*)(info->crypt_key), 16, + aes_ctr_counter, MY_AES_BLOCK_SIZE, + what | ENCRYPTION_FLAG_NOPAD, + LOG_DEFAULT_ENCRYPTION_KEY, + info->key_version); + + if (rc != MY_AES_OK) { + ib_logf(IB_LOG_LEVEL_ERROR, + "%s failed for temporary log file with rc = %d", + is_encrypt ? "Encryption" : "Decryption", + rc); + return false; + } + + return true; +} + +/** Get crypt info +@return pointer to log crypt info or NULL +*/ +inline +const crypt_info_t* +get_crypt_info() +{ + mutex_enter(&log_sys->mutex); + const crypt_info_t* info = get_crypt_info(log_sys->next_checkpoint_no); + mutex_exit(&log_sys->mutex); + + return info; +} + +/** Find out is temporary log files encrypted. +@return true if temporary log file should be encrypted, false if not */ +UNIV_INTERN +bool +log_tmp_is_encrypted() +{ + const crypt_info_t* info = get_crypt_info(); + + if (info == NULL || info->key_version == UNENCRYPTED_KEY_VER) { + return false; + } + + return true; +} + +/** Encrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_encrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) +{ + return (log_tmp_blocks_crypt(src_block, size, dst_block, + ENCRYPTION_FLAG_ENCRYPT, offs, space_id)); +} + +/** Decrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_decrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) +{ + return (log_tmp_blocks_crypt(src_block, size, dst_block, + ENCRYPTION_FLAG_DECRYPT, offs, space_id)); +} + /*********************************************************************//** Generate crypt key from crypt msg. @return true if successfull, false if not. */ diff --git a/storage/xtradb/row/row0ftsort.cc b/storage/xtradb/row/row0ftsort.cc index 7ffcc59dc5f..0f9c15f6a69 100644 --- a/storage/xtradb/row/row0ftsort.cc +++ b/storage/xtradb/row/row0ftsort.cc @@ -40,7 +40,7 @@ Created 10/13/2010 Jimmy Yang b[N] = row_merge_read_rec( \ block[N], buf[N], b[N], index, \ fd[N], &foffs[N], &mrec[N], offsets[N], \ - crypt_data, crypt_block[N], space); \ + crypt_block[N], space); \ if (UNIV_UNLIKELY(!b[N])) { \ if (mrec[N]) { \ goto exit; \ @@ -194,7 +194,6 @@ row_fts_psort_info_init( fts_psort_t* merge_info = NULL; ulint block_size; ibool ret = TRUE; - fil_space_crypt_t* crypt_data = NULL; bool encrypted = false; block_size = 3 * srv_sort_buf_size; @@ -225,21 +224,8 @@ row_fts_psort_info_init( common_info->merge_event = os_event_create(); common_info->opt_doc_id_size = opt_doc_id_size; - /* Theoretically the tablespace can be dropped straight away. - In practice, the DDL completion will wait for this thread to - finish. */ - if (fil_space_t* space = fil_space_acquire(new_table->space)) { - crypt_data = space->crypt_data; - fil_space_release(space); - } - - if (crypt_data && crypt_data->should_encrypt()) { - common_info->crypt_data = crypt_data; + if (log_tmp_is_encrypted()) { encrypted = true; - } else { - /* Not needed */ - common_info->crypt_data = NULL; - crypt_data = NULL; } ut_ad(trx->mysql_thd != NULL); @@ -577,11 +563,9 @@ row_merge_fts_doc_tokenize( cur_len += len; dfield_dup(field, buf->heap); - /* Reserve one byte for the end marker of row_merge_block_t - and we have reserved ROW_MERGE_RESERVE_SIZE (= 4) for - encryption key_version in the beginning of the buffer. */ + /* Reserve one byte for the end marker of row_merge_block_t */ if (buf->total_size + data_size[idx] + cur_len - >= (srv_sort_buf_size - 1 - ROW_MERGE_RESERVE_SIZE)) { + >= (srv_sort_buf_size - 1)) { buf_full = TRUE; break; @@ -675,7 +659,6 @@ fts_parallel_tokenization( fts_tokenize_ctx_t t_ctx; ulint retried = 0; dberr_t error = DB_SUCCESS; - fil_space_crypt_t* crypt_data = NULL; ut_ad(psort_info->psort_common->trx->mysql_thd != NULL); @@ -696,7 +679,6 @@ fts_parallel_tokenization( block = psort_info->merge_block; crypt_block = psort_info->crypt_block; - crypt_data = psort_info->psort_common->crypt_data; zip_size = dict_table_zip_size(table); row_merge_fts_get_next_doc_item(psort_info, &doc_item); @@ -791,7 +773,6 @@ loop: if (!row_merge_write(merge_file[t_ctx.buf_used]->fd, merge_file[t_ctx.buf_used]->offset++, block[t_ctx.buf_used], - crypt_data, crypt_block[t_ctx.buf_used], table->space)) { error = DB_TEMP_FILE_WRITE_FAILURE; @@ -887,7 +868,6 @@ exit: if (!row_merge_write(merge_file[i]->fd, merge_file[i]->offset++, block[i], - crypt_data, crypt_block[i], table->space)) { error = DB_TEMP_FILE_WRITE_FAILURE; @@ -927,7 +907,7 @@ exit: psort_info->psort_common->dup, merge_file[i], block[i], &tmpfd[i], false, 0.0/* pct_progress */, 0.0/* pct_cost */, - crypt_data, crypt_block[i], table->space); + crypt_block[i], table->space); if (error != DB_SUCCESS) { close(tmpfd[i]); @@ -1439,7 +1419,6 @@ row_fts_merge_insert( ulint start; fts_psort_insert_t ins_ctx; ulint count_diag = 0; - fil_space_crypt_t* crypt_data = NULL; ulint space; ut_ad(index); @@ -1453,7 +1432,6 @@ row_fts_merge_insert( ins_ctx.trx->op_info = "inserting index entries"; ins_ctx.opt_doc_id_size = psort_info[0].psort_common->opt_doc_id_size; - crypt_data = psort_info[0].psort_common->crypt_data; heap = mem_heap_create(500 + sizeof(mrec_buf_t)); @@ -1547,7 +1525,6 @@ row_fts_merge_insert( && (!row_merge_read( fd[i], foffs[i], (row_merge_block_t*) block[i], - crypt_data, (row_merge_block_t*) crypt_block[i], space))) { error = DB_CORRUPTION; diff --git a/storage/xtradb/row/row0log.cc b/storage/xtradb/row/row0log.cc index 2cd663fd600..f49ff2f1530 100644 --- a/storage/xtradb/row/row0log.cc +++ b/storage/xtradb/row/row0log.cc @@ -182,6 +182,7 @@ struct row_log_t { dict_table_t* table; /*!< table that is being rebuilt, or NULL when this is a secondary index that is being created online */ + dict_index_t* index; /*!< index to be build */ bool same_pk;/*!< whether the definition of the PRIMARY KEY has remained the same */ const dtuple_t* add_cols; @@ -197,8 +198,14 @@ struct row_log_t { row_log_buf_t tail; /*!< writer context; protected by mutex and index->lock S-latch, or by index->lock X-latch only */ + byte* crypt_tail; /*!< writer context; + temporary buffer used in encryption, + decryption or NULL*/ row_log_buf_t head; /*!< reader context; protected by MDL only; modifiable by row_log_apply_ops() */ + byte* crypt_head; /*!< reader context; + temporary buffer used in encryption, + decryption or NULL */ const char* path; /*!< where to create temporary file during log operation */ }; @@ -349,6 +356,7 @@ row_log_online_op( = (os_offset_t) log->tail.blocks * srv_sort_buf_size; ibool ret; + byte * buf = log->tail.block; if (byte_offset + srv_sort_buf_size >= srv_online_max_size) { goto write_failed; @@ -368,11 +376,29 @@ row_log_online_op( goto err_exit; } + /* If encryption is enabled encrypt buffer before writing it + to file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt(log->tail.block, + srv_sort_buf_size, + log->crypt_tail, + byte_offset, + index->table->space)) { + log->error = DB_DECRYPTION_FAILED; + goto err_exit; + } + + srv_stats.n_rowlog_blocks_encrypted.inc(); + buf = log->crypt_tail; + } + ret = os_file_write_int_fd( "(modification log)", log->fd, - log->tail.block, byte_offset, srv_sort_buf_size); + buf, byte_offset, srv_sort_buf_size); + log->tail.blocks++; + if (!ret) { write_failed: /* We set the flag directly instead of invoking @@ -380,7 +406,9 @@ write_failed: because the index is not "public" yet. */ index->type |= DICT_CORRUPT; } + UNIV_MEM_INVALID(log->tail.block, srv_sort_buf_size); + memcpy(log->tail.block, log->tail.buf + avail_size, mrec_size - avail_size); log->tail.bytes = mrec_size - avail_size; @@ -465,6 +493,7 @@ row_log_table_close_func( = (os_offset_t) log->tail.blocks * srv_sort_buf_size; ibool ret; + byte * buf = log->tail.block; if (byte_offset + srv_sort_buf_size >= srv_online_max_size) { goto write_failed; @@ -484,11 +513,29 @@ row_log_table_close_func( goto err_exit; } + /* If encryption is enabled encrypt buffer before writing it + to file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt(log->tail.block, + srv_sort_buf_size, + log->crypt_tail, + byte_offset, + log->index->table->space)) { + log->error = DB_DECRYPTION_FAILED; + goto err_exit; + } + + srv_stats.n_rowlog_blocks_encrypted.inc(); + buf = log->crypt_tail; + } + ret = os_file_write_int_fd( "(modification log)", log->fd, - log->tail.block, byte_offset, srv_sort_buf_size); + buf, byte_offset, srv_sort_buf_size); + log->tail.blocks++; + if (!ret) { write_failed: log->error = DB_ONLINE_LOG_TOO_BIG; @@ -2619,10 +2666,29 @@ all_done: goto func_exit; } + byte * buf = index->online_log->head.block; + success = os_file_read_no_error_handling_int_fd( index->online_log->fd, - index->online_log->head.block, ofs, + buf, ofs, srv_sort_buf_size); + + /* If encryption is enabled decrypt buffer after reading it + from file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, + srv_sort_buf_size, + index->online_log->crypt_head, + ofs, + index->table->space)) { + error = DB_DECRYPTION_FAILED; + goto func_exit; + } + + srv_stats.n_rowlog_blocks_decrypted.inc(); + memcpy(buf, index->online_log->crypt_head, srv_sort_buf_size); + } + if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" " for table %s\n", index->table_name); @@ -2929,9 +2995,22 @@ row_log_allocate( log->head.blocks = log->head.bytes = 0; log->head.total = 0; log->path = path; + log->crypt_tail = log->crypt_head = NULL; + log->index = index; dict_index_set_online_status(index, ONLINE_INDEX_CREATION); index->online_log = log; + if (log_tmp_is_encrypted()) { + ulint size = srv_sort_buf_size; + log->crypt_head = static_cast<byte *>(os_mem_alloc_large(&size)); + log->crypt_tail = static_cast<byte *>(os_mem_alloc_large(&size)); + + if (!log->crypt_head || !log->crypt_tail) { + row_log_free(log); + DBUG_RETURN(false); + } + } + /* While we might be holding an exclusive data dictionary lock here, in row_log_abort_sec() we will not always be holding it. Use atomic operations in both cases. */ @@ -2954,6 +3033,15 @@ row_log_free( row_log_block_free(log->tail); row_log_block_free(log->head); row_merge_file_destroy_low(log->fd); + + if (log->crypt_head) { + os_mem_free_large(log->crypt_head, srv_sort_buf_size); + } + + if (log->crypt_tail) { + os_mem_free_large(log->crypt_tail, srv_sort_buf_size); + } + mutex_free(&log->mutex); ut_free(log); log = 0; @@ -3445,11 +3533,29 @@ all_done: goto func_exit; } + byte* buf = index->online_log->head.block; + success = os_file_read_no_error_handling_int_fd( index->online_log->fd, - index->online_log->head.block, ofs, + buf, ofs, srv_sort_buf_size); + /* If encryption is enabled decrypt buffer after reading it + from file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, + srv_sort_buf_size, + index->online_log->crypt_head, + ofs, + index->table->space)) { + error = DB_DECRYPTION_FAILED; + goto func_exit; + } + + srv_stats.n_rowlog_blocks_decrypted.inc(); + memcpy(buf, index->online_log->crypt_head, srv_sort_buf_size); + } + if (!success) { fprintf(stderr, "InnoDB: unable to read temporary file" " for index %s\n", index->name + 1); diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index 408e94b3997..28b43d6abef 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -79,88 +79,6 @@ UNIV_INTERN char srv_disable_sort_file_cache; /* Maximum pending doc memory limit in bytes for a fts tokenization thread */ #define FTS_PENDING_DOC_MEMORY_LIMIT 1000000 - -/******************************************************//** -Encrypt a merge block. */ -static -void -row_merge_encrypt_buf( -/*==================*/ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ - ulint offset, /*!< in: offset where to - write */ - ulint space, /*!< in: tablespace id */ - const byte* input_buf, /*!< in: input buffer */ - byte* crypted_buf) /*!< out: crypted buffer */ -{ - uint key_version; - uint dstlen=0; - os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset; - - key_version = encryption_key_get_latest_version(crypt_data->key_id); - - /* Store key_version at the begining of the input buffer */ - mach_write_to_4((byte *)crypted_buf, key_version); - - int rc = encryption_scheme_encrypt(input_buf+ROW_MERGE_RESERVE_SIZE, - srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE, - crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen, - crypt_data, key_version, - space, ofs, 0); - - if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Unable to encrypt data-block " - " src: %p srclen: %lu buf: %p buflen: %d." - " return-code: %d. Can't continue!\n", - input_buf, srv_sort_buf_size, - crypted_buf, dstlen, rc); - } -} - -/******************************************************//** -Decrypt a merge block. */ -static -bool -row_merge_decrypt_buf( -/*==================*/ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ - ulint offset, /*!< in: offset where to - write */ - ulint space, /*!< in: tablespace id */ - const byte* input_buf, /*!< in: input buffer */ - byte* crypted_buf) /*!< out: crypted buffer */ -{ - uint key_version; - uint dstlen=0; - os_offset_t ofs = (os_offset_t)srv_sort_buf_size * (os_offset_t)offset; - - /* Read key_version from begining of the buffer */ - key_version = mach_read_from_4((byte *)input_buf); - - if (key_version == 0) { - /* block not encrypted */ - return false; - } - - int rc = encryption_scheme_decrypt(input_buf+ROW_MERGE_RESERVE_SIZE, - srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE, - crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen, - crypt_data, key_version, - space, ofs, 0); - - if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) { - ib_logf(IB_LOG_LEVEL_FATAL, - "Unable to encrypt data-block " - " src: %p srclen: %lu buf: %p buflen: %d." - " return-code: %d. Can't continue!\n", - input_buf, srv_sort_buf_size, - crypted_buf, dstlen, rc); - } - - return (true); -} - #ifdef UNIV_DEBUG /******************************************************//** Display a merge tuple. */ @@ -279,7 +197,7 @@ row_merge_buf_create( ulint buf_size; mem_heap_t* heap; - max_tuples = (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE) + max_tuples = (srv_sort_buf_size) / ut_max(1, dict_index_get_min_size(index)); buf_size = (sizeof *buf); @@ -702,7 +620,7 @@ row_merge_buf_add( ut_ad(data_size < srv_sort_buf_size); /* Reserve bytes for the end marker of row_merge_block_t. */ - if (buf->total_size + data_size >= (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE)) { + if (buf->total_size + data_size >= srv_sort_buf_size) { DBUG_RETURN(0); } @@ -823,7 +741,7 @@ UT_SORT_FUNCTION_BODY(). /**********************************************************************//** Merge sort the tuple buffer in main memory. */ -static MY_ATTRIBUTE((nonnull(4,5))) +static void row_merge_tuple_sort( /*=================*/ @@ -874,7 +792,7 @@ row_merge_buf_write( { const dict_index_t* index = buf->index; ulint n_fields= dict_index_get_n_fields(index); - byte* b = &block[ROW_MERGE_RESERVE_SIZE]; + byte* b = &block[0]; for (ulint i = 0; i < buf->n_tuples; i++) { const mtuple_t* entry = &buf->tuples[i]; @@ -893,7 +811,7 @@ row_merge_buf_write( /* Write an "end-of-chunk" marker. */ ut_a(b < &block[srv_sort_buf_size]); - ut_a(b == &block[0] + buf->total_size + ROW_MERGE_RESERVE_SIZE); + ut_a(b == &block[0] + buf->total_size); *b++ = 0; #ifdef UNIV_DEBUG_VALGRIND /* The rest of the block is uninitialized. Initialize it @@ -943,7 +861,7 @@ row_merge_heap_create( Read a merge block from the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_read( /*===========*/ int fd, /*!< in: file descriptor */ @@ -951,12 +869,11 @@ row_merge_read( in number of row_merge_block_t elements */ row_merge_block_t* buf, /*!< out: data */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { os_offset_t ofs = ((os_offset_t) offset) * srv_sort_buf_size; - ibool success; + bool success; DBUG_EXECUTE_IF("row_merge_read_failure", return(FALSE);); @@ -971,10 +888,14 @@ row_merge_read( ofs, srv_sort_buf_size); /* For encrypted tables, decrypt data after reading and copy data */ - if (crypt_data && crypt_buf) { - if( row_merge_decrypt_buf(crypt_data, offset, space, buf, crypt_buf)) { - memcpy(buf, crypt_buf, srv_sort_buf_size); + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, srv_sort_buf_size, + crypt_buf, ofs, space)) { + return (FALSE); } + + srv_stats.n_merge_blocks_decrypted.inc(); + memcpy(buf, crypt_buf, srv_sort_buf_size); } #ifdef POSIX_FADV_DONTNEED @@ -996,31 +917,32 @@ row_merge_read( Write a merge block to the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_write( /*============*/ int fd, /*!< in: file descriptor */ ulint offset, /*!< in: offset where to write, in number of row_merge_block_t elements */ const void* buf, /*!< in: data */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ void* crypt_buf, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ { size_t buf_len = srv_sort_buf_size; os_offset_t ofs = buf_len * (os_offset_t) offset; - ibool ret; + bool ret; void* out_buf = (void *)buf; DBUG_EXECUTE_IF("row_merge_write_failure", return(FALSE);); /* For encrypted tables, encrypt data before writing */ - if (crypt_data && crypt_buf) { - row_merge_encrypt_buf(crypt_data, offset, space, (const byte *)buf, (byte *)crypt_buf); + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt((const byte *)buf, buf_len, + (byte *)crypt_buf, ofs, space)) { + return (FALSE); + } + + srv_stats.n_merge_blocks_encrypted.inc(); out_buf = crypt_buf; - } else { - /* Mark block unencrypted */ - mach_write_to_4((byte *)out_buf, 0); } ret = os_file_write_int_fd("(merge)", fd, out_buf, ofs, buf_len); @@ -1058,9 +980,8 @@ row_merge_read_rec( or NULL on end of list (non-NULL on I/O error) */ ulint* offsets,/*!< out: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { ulint extra_size; ulint data_size; @@ -1072,10 +993,6 @@ row_merge_read_rec( ut_ad(*offsets == 1 + REC_OFFS_HEADER_SIZE + dict_index_get_n_fields(index)); - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - extra_size = *b++; if (UNIV_UNLIKELY(!extra_size)) { @@ -1097,7 +1014,8 @@ row_merge_read_rec( if (UNIV_UNLIKELY(b >= &block[srv_sort_buf_size])) { if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { err_exit: /* Signal I/O error. */ *mrec = b; @@ -1105,7 +1023,7 @@ err_exit: } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; } extra_size = (extra_size & 0x7f) << 8; @@ -1127,13 +1045,14 @@ err_exit: memcpy(*buf, b, avail_size); if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { goto err_exit; } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; /* Copy the record. */ memcpy(*buf + avail_size, b, extra_size - avail_size); @@ -1189,13 +1108,14 @@ err_exit: #endif /* UNIV_DEBUG */ if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { goto err_exit; } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; /* Copy the rest of the record. */ memcpy(*buf + avail_size, b, extra_size + data_size - avail_size); @@ -1272,9 +1192,8 @@ row_merge_write_rec( ulint* foffs, /*!< in/out: file offset */ const mrec_t* mrec, /*!< in: record to write */ const ulint* offsets,/*!< in: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { ulint extra_size; ulint size; @@ -1295,10 +1214,6 @@ row_merge_write_rec( size = extra_size + (extra_size >= 0x80) + rec_offs_data_size(offsets); - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - if (UNIV_UNLIKELY(b + size >= &block[srv_sort_buf_size])) { /* The record spans two blocks. Copy it to the temporary buffer first. */ @@ -1314,14 +1229,15 @@ row_merge_write_rec( memcpy(b, buf[0], avail_size); if (!row_merge_write(fd, (*foffs)++, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(NULL); } UNIV_MEM_INVALID(&block[0], srv_sort_buf_size); /* Copy the rest. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; memcpy(b, buf[0] + avail_size, size - avail_size); b += size - avail_size; } else { @@ -1344,7 +1260,6 @@ row_merge_write_eof( byte* b, /*!< in: pointer to end of block */ int fd, /*!< in: file descriptor */ ulint* foffs, /*!< in/out: file offset */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ { @@ -1359,10 +1274,6 @@ row_merge_write_eof( } #endif /* UNIV_DEBUG */ - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - *b++ = 0; UNIV_MEM_ASSERT_RW(&block[0], b - &block[0]); UNIV_MEM_ASSERT_W(&block[0], srv_sort_buf_size); @@ -1374,7 +1285,8 @@ row_merge_write_eof( #endif /* UNIV_DEBUG_VALGRIND */ if (!row_merge_write(fd, (*foffs)++, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(NULL); } @@ -1451,8 +1363,10 @@ containing the index entries for the indexes to be built. @param[in,out] sequence autoinc sequence @param[in,out] block file buffer @param[in,out] tmpfd temporary file handle -return DB_SUCCESS or error */ -static MY_ATTRIBUTE((nonnull(1,2,3,4,6,9,10,16), warn_unused_result)) +@param[in] pct_cost percent of task weight out of total alter job +@param[in,out] crypt_block crypted file buffer +@return DB_SUCCESS or error */ +static MY_ATTRIBUTE((warn_unused_result)) dberr_t row_merge_read_clustered_index( trx_t* trx, @@ -1472,11 +1386,8 @@ row_merge_read_clustered_index( ib_sequence_t& sequence, row_merge_block_t* block, int* tmpfd, - float pct_cost, /*!< in: percent of task weight - out of total alter job */ - fil_space_crypt_t* crypt_data,/*!< in: crypt data or NULL */ - row_merge_block_t* crypt_block)/*!< in: in/out: crypted file - buffer */ + float pct_cost, + row_merge_block_t* crypt_block) { dict_index_t* clust_index; /* Clustered index */ mem_heap_t* row_heap; /* Heap memory to create @@ -2014,7 +1925,7 @@ write_buffers: row_merge_buf_write(buf, file, block); if (!row_merge_write(file->fd, file->offset++, - block, crypt_data, crypt_block, + block, crypt_block, new_table->space)) { err = DB_TEMP_FILE_WRITE_FAILURE; trx->error_key_num = i; @@ -2066,7 +1977,7 @@ write_buffers: if(read_rows % 1000 == 0) { /* Update progress for each 1000 rows */ curr_progress = (read_rows >= table_total_rows) ? - pct_cost : + pct_cost : ((pct_cost * read_rows) / table_total_rows); /* presenting 10.12% as 1012 integer */ onlineddl_pct_progress = curr_progress * 100; @@ -2195,9 +2106,8 @@ wait_again: &buf[2], b2, \ of->fd, &of->offset, \ mrec##N, offsets##N, \ - crypt_data, \ crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL , \ - space); \ + space); \ if (UNIV_UNLIKELY(!b2 || ++of->n_rec > file->n_rec)) { \ goto corrupt; \ } \ @@ -2205,7 +2115,6 @@ wait_again: &buf[N], b##N, INDEX, \ file->fd, foffs##N, \ &mrec##N, offsets##N, \ - crypt_data, \ crypt_block ? &crypt_block[N * srv_sort_buf_size] : NULL, \ space); \ \ @@ -2220,7 +2129,7 @@ wait_again: /*************************************************************//** Merge two blocks of records on disk and write a bigger block. @return DB_SUCCESS or error code */ -static __attribute__((nonnull(1,2,3,4,5,6), warn_unused_result)) +static __attribute__((warn_unused_result)) dberr_t row_merge_blocks( /*=============*/ @@ -2234,7 +2143,6 @@ row_merge_blocks( ulint* foffs1, /*!< in/out: offset of second source list in the file */ merge_file_t* of, /*!< in/out: output file */ - fil_space_crypt_t* crypt_data,/*!< in: crypt data or NULL */ row_merge_block_t* crypt_block,/*!< in: in/out: crypted file buffer */ ulint space) /*!< in: space id */ @@ -2270,9 +2178,11 @@ row_merge_blocks( file in two halves, which can be merged on the following pass. */ if (!row_merge_read(file->fd, *foffs0, &block[0], - crypt_data, crypt_block ? &crypt_block[0] : NULL, space) - || !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size], - crypt_data, crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, space)) { + crypt_block ? &crypt_block[0] : NULL, + space) || + !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size], + crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, + space)) { corrupt: mem_heap_free(heap); return(DB_CORRUPTION); @@ -2285,13 +2195,15 @@ corrupt: b0 = row_merge_read_rec( &block[0], &buf[0], b0, dup->index, file->fd, foffs0, &mrec0, offsets0, - crypt_data, crypt_block ? &crypt_block[0] : NULL, space); + crypt_block ? &crypt_block[0] : NULL, + space); b1 = row_merge_read_rec( &block[srv_sort_buf_size], &buf[srv_sort_buf_size], b1, dup->index, file->fd, foffs1, &mrec1, offsets1, - crypt_data, crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, space); + crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, + space); if (UNIV_UNLIKELY(!b0 && mrec0) || UNIV_UNLIKELY(!b1 && mrec1)) { @@ -2337,7 +2249,8 @@ done1: b2 = row_merge_write_eof(&block[2 * srv_sort_buf_size], b2, of->fd, &of->offset, - crypt_data, crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, space); + crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, + space); return(b2 ? DB_SUCCESS : DB_CORRUPTION); } @@ -2345,8 +2258,8 @@ done1: /*************************************************************//** Copy a block of index entries. @return TRUE on success, FALSE on failure */ -static __attribute__((nonnull(1,2,3,4,5), warn_unused_result)) -ibool +static __attribute__((warn_unused_result)) +bool row_merge_blocks_copy( /*==================*/ const dict_index_t* index, /*!< in: index being created */ @@ -2354,9 +2267,8 @@ row_merge_blocks_copy( row_merge_block_t* block, /*!< in/out: 3 buffers */ ulint* foffs0, /*!< in/out: input file offset */ merge_file_t* of, /*!< in/out: output file */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { mem_heap_t* heap; /*!< memory heap for offsets0, offsets1 */ @@ -2384,7 +2296,8 @@ row_merge_blocks_copy( file in two halves, which can be merged on the following pass. */ if (!row_merge_read(file->fd, *foffs0, &block[0], - crypt_data, crypt_block ? &crypt_block[0] : NULL, space)) { + crypt_block ? &crypt_block[0] : NULL, + space)) { corrupt: mem_heap_free(heap); return(FALSE); @@ -2396,7 +2309,8 @@ corrupt: b0 = row_merge_read_rec(&block[0], &buf[0], b0, index, file->fd, foffs0, &mrec0, offsets0, - crypt_data, crypt_block ? &crypt_block[0] : NULL, space); + crypt_block ? &crypt_block[0] : NULL, + space); if (UNIV_UNLIKELY(!b0 && mrec0)) { @@ -2419,15 +2333,15 @@ done0: return(row_merge_write_eof(&block[2 * srv_sort_buf_size], b2, of->fd, &of->offset, - crypt_data, - crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, space) + crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, + space) != NULL); } /*************************************************************//** Merge disk files. @return DB_SUCCESS or error code */ -static __attribute__((nonnull(1,2,3,4,5,6,7))) +static dberr_t row_merge( /*======*/ @@ -2443,9 +2357,8 @@ row_merge( ulint* run_offset, /*!< in/out: Array contains the first offset number for each merge run */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { ulint foffs0; /*!< first input offset */ ulint foffs1; /*!< second input offset */ @@ -2493,7 +2406,8 @@ row_merge( error = row_merge_blocks(dup, file, block, &foffs0, &foffs1, &of, - crypt_data, crypt_block, space); + crypt_block, + space); if (error != DB_SUCCESS) { return(error); @@ -2514,7 +2428,8 @@ row_merge( if (!row_merge_blocks_copy(dup->index, file, block, &foffs0, &of, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(DB_CORRUPTION); } } @@ -2532,7 +2447,8 @@ row_merge( if (!row_merge_blocks_copy(dup->index, file, block, &foffs1, &of, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(DB_CORRUPTION); } } @@ -2588,9 +2504,8 @@ row_merge_sort( /*!< in: total progress percent until now */ const float pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { const ulint half = file->offset / 2; ulint num_runs; @@ -2650,7 +2565,8 @@ row_merge_sort( error = row_merge(trx, dup, file, block, tmpfd, &num_runs, run_offset, - crypt_data, crypt_block, space); + crypt_block, + space); if(update_progress) { merge_count++; @@ -2725,7 +2641,7 @@ row_merge_copy_blobs( Read sorted file containing index data tuples and insert these data tuples to the index @return DB_SUCCESS or error number */ -static __attribute__((nonnull(2,3,5), warn_unused_result)) +static __attribute__((warn_unused_result)) dberr_t row_merge_insert_index_tuples( /*==========================*/ @@ -2738,9 +2654,8 @@ row_merge_insert_index_tuples( const float pct_progress, /*!< in: total progress percent until now */ const float pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { const byte* b; mem_heap_t* heap; @@ -2774,7 +2689,8 @@ row_merge_insert_index_tuples( b = &block[0]; if (!row_merge_read(fd, foffs, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { error = DB_CORRUPTION; } else { buf = static_cast<mrec_buf_t*>( @@ -2791,7 +2707,9 @@ row_merge_insert_index_tuples( b = row_merge_read_rec(block, buf, b, index, fd, &foffs, &mrec, offsets, - crypt_data, crypt_block, space); + crypt_block, + space); + if (UNIV_UNLIKELY(!b)) { /* End of list, or I/O error */ if (mrec) { @@ -3976,7 +3894,6 @@ row_merge_build_indexes( fts_psort_t* merge_info = NULL; ib_int64_t sig_count = 0; bool fts_psort_initiated = false; - fil_space_crypt_t * crypt_data = NULL; float total_static_cost = 0; float total_dynamic_cost = 0; @@ -4005,25 +3922,20 @@ row_merge_build_indexes( from concurrent DDL (e.g. drop table) by MDL-locks. */ fil_space_t* space = fil_space_acquire(new_table->space); - if (space) { - crypt_data = space->crypt_data; - } else { + if (!space) { DBUG_RETURN(DB_TABLESPACE_NOT_FOUND); } - /* If tablespace is encrypted, allocate additional buffer for + /* If temporal log file is encrypted allocate memory for encryption/decryption. */ - if (crypt_data && crypt_data->should_encrypt()) { + if (log_tmp_is_encrypted()) { crypt_block = static_cast<row_merge_block_t*>( - os_mem_alloc_large(&block_size)); + os_mem_alloc_large(&block_size)); if (crypt_block == NULL) { fil_space_release(space); DBUG_RETURN(DB_OUT_OF_MEMORY); } - } else { - /* Not needed */ - crypt_data = NULL; } trx_start_if_not_started_xa(trx); @@ -4105,7 +4017,7 @@ row_merge_build_indexes( fts_sort_idx, psort_info, merge_files, key_numbers, n_indexes, add_cols, col_map, add_autoinc, sequence, block, &tmpfd, pct_cost, - crypt_data, crypt_block); + crypt_block); pct_progress += pct_cost; @@ -4242,7 +4154,8 @@ wait_again: trx, &dup, &merge_files[i], block, &tmpfd, true, pct_progress, pct_cost, - crypt_data, crypt_block, new_table->space); + crypt_block, + new_table->space); pct_progress += pct_cost; @@ -4280,7 +4193,8 @@ wait_again: trx->id, sort_idx, old_table, merge_files[i].fd, block, merge_files[i].n_rec, pct_progress, pct_cost, - crypt_data, crypt_block, new_table->space); + crypt_block, new_table->space); + pct_progress += pct_cost; if (global_system_variables.log_warnings > 2) { diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index 1f1fef6bea6..d6070107432 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -2060,6 +2060,10 @@ srv_export_innodb_status(void) export_vars.innodb_pages_page_compression_error = srv_stats.pages_page_compression_error; export_vars.innodb_pages_decrypted = srv_stats.pages_decrypted; export_vars.innodb_pages_encrypted = srv_stats.pages_encrypted; + export_vars.innodb_n_merge_blocks_encrypted = srv_stats.n_merge_blocks_encrypted; + export_vars.innodb_n_merge_blocks_decrypted = srv_stats.n_merge_blocks_decrypted; + export_vars.innodb_n_rowlog_blocks_encrypted = srv_stats.n_rowlog_blocks_encrypted; + export_vars.innodb_n_rowlog_blocks_decrypted = srv_stats.n_rowlog_blocks_decrypted; export_vars.innodb_defragment_compression_failures = btr_defragment_compression_failures; |