diff options
author | Jan Lindström <jan.lindstrom@mariadb.com> | 2017-09-16 15:32:58 +0300 |
---|---|---|
committer | Jan Lindström <jan.lindstrom@mariadb.com> | 2017-09-16 15:32:58 +0300 |
commit | c71b7b663d811ace044fcbe9140b8de26524eddb (patch) | |
tree | e0b425b2651bfa5c9492dd327dcf40b842d6fd76 | |
parent | c2815c7c2c7f9dcf4e9f241e10d154faf561103f (diff) | |
download | mariadb-git-bb-10.2-jplindst.tar.gz |
Merge 10.1 into 10.2bb-10.2-jplindst
22 files changed, 572 insertions, 230 deletions
diff --git a/mysql-test/r/ctype_utf8.result b/mysql-test/r/ctype_utf8.result index 0a4898ff0b9..a557371d3a4 100644 --- a/mysql-test/r/ctype_utf8.result +++ b/mysql-test/r/ctype_utf8.result @@ -1951,7 +1951,7 @@ Warnings: Warning 1300 Invalid utf8 character string: 'FD' select convert(char(0xff,0x8f) using utf8); convert(char(0xff,0x8f) using utf8) -NULL +?? Warnings: Warning 1300 Invalid utf8 character string: '\xFF\x8F' select hex(convert(char(2557 using latin1) using utf8)); @@ -2120,7 +2120,7 @@ Warnings: Warning 1300 Invalid utf8 character string: 'FF' select hex(convert(0xFF using utf8)); hex(convert(0xFF using utf8)) -NULL +3F Warnings: Warning 1300 Invalid utf8 character string: '\xFF' select hex(_utf8 0x616263FF); @@ -10479,6 +10479,31 @@ END DROP PROCEDURE p1; SET @@SQL_MODE=default; # +# MDEV-10191 non convertible chars convert() resulted in Null instead "?" on Windows +# +SET sql_mode='STRICT_TRANS_TABLES'; +SELECT CONVERT(_utf8 0xC499 USING latin1); +CONVERT(_utf8 0xC499 USING latin1) +? +Warnings: +Warning 1977 Cannot convert 'utf8' character 0xC499 to 'latin1' +SELECT CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1); +CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1) +? +Warnings: +Warning 1977 Cannot convert 'utf8' character 0xC499 to 'latin1' +SET sql_mode=default; +SELECT CONVERT(_utf8 0xC499 USING latin1); +CONVERT(_utf8 0xC499 USING latin1) +? +Warnings: +Warning 1977 Cannot convert 'utf8' character 0xC499 to 'latin1' +SELECT CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1); +CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1) +? +Warnings: +Warning 1977 Cannot convert 'utf8' character 0xC499 to 'latin1' +# # End of 10.1 tests # # diff --git a/mysql-test/r/ctype_utf8mb4.result b/mysql-test/r/ctype_utf8mb4.result index d74cfd4dd3f..39cc8da332c 100644 --- a/mysql-test/r/ctype_utf8mb4.result +++ b/mysql-test/r/ctype_utf8mb4.result @@ -1976,7 +1976,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FD' select convert(char(0xff,0x8f) using utf8mb4); convert(char(0xff,0x8f) using utf8mb4) -NULL +?? Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF\x8F' select hex(convert(char(2557 using latin1) using utf8mb4)); @@ -2145,7 +2145,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FF' select hex(convert(0xFF using utf8mb4)); hex(convert(0xFF using utf8mb4)) -NULL +3F Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF' select hex(_utf8mb4 0x616263FF); diff --git a/mysql-test/r/ctype_utf8mb4_heap.result b/mysql-test/r/ctype_utf8mb4_heap.result index 859ceff864e..8b054049d5b 100644 --- a/mysql-test/r/ctype_utf8mb4_heap.result +++ b/mysql-test/r/ctype_utf8mb4_heap.result @@ -1808,7 +1808,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FD' select convert(char(0xff,0x8f) using utf8mb4); convert(char(0xff,0x8f) using utf8mb4) -NULL +?? Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF\x8F' select hex(convert(char(2557 using latin1) using utf8mb4)); @@ -1977,7 +1977,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FF' select hex(convert(0xFF using utf8mb4)); hex(convert(0xFF using utf8mb4)) -NULL +3F Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF' select hex(_utf8mb4 0x616263FF); diff --git a/mysql-test/r/ctype_utf8mb4_innodb.result b/mysql-test/r/ctype_utf8mb4_innodb.result index 11d89154269..017a9d88c3e 100644 --- a/mysql-test/r/ctype_utf8mb4_innodb.result +++ b/mysql-test/r/ctype_utf8mb4_innodb.result @@ -1936,7 +1936,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FD' select convert(char(0xff,0x8f) using utf8mb4); convert(char(0xff,0x8f) using utf8mb4) -NULL +?? Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF\x8F' select hex(convert(char(2557 using latin1) using utf8mb4)); @@ -2105,7 +2105,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FF' select hex(convert(0xFF using utf8mb4)); hex(convert(0xFF using utf8mb4)) -NULL +3F Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF' select hex(_utf8mb4 0x616263FF); diff --git a/mysql-test/r/ctype_utf8mb4_myisam.result b/mysql-test/r/ctype_utf8mb4_myisam.result index ef7bda0a79b..bf4d05f2883 100644 --- a/mysql-test/r/ctype_utf8mb4_myisam.result +++ b/mysql-test/r/ctype_utf8mb4_myisam.result @@ -1936,7 +1936,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FD' select convert(char(0xff,0x8f) using utf8mb4); convert(char(0xff,0x8f) using utf8mb4) -NULL +?? Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF\x8F' select hex(convert(char(2557 using latin1) using utf8mb4)); @@ -2105,7 +2105,7 @@ Warnings: Warning 1300 Invalid utf8mb4 character string: 'FF' select hex(convert(0xFF using utf8mb4)); hex(convert(0xFF using utf8mb4)) -NULL +3F Warnings: Warning 1300 Invalid utf8mb4 character string: '\xFF' select hex(_utf8mb4 0x616263FF); diff --git a/mysql-test/suite/encryption/r/innodb-rowlog.result b/mysql-test/suite/encryption/r/innodb-rowlog.result new file mode 100644 index 00000000000..873041ab421 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb-rowlog.result @@ -0,0 +1,30 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +create table t1(id int not null primary key auto_increment, credit_card varchar(200), private varchar(50)) engine=innodb; +SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; +alter table t1 add index secret (credit_card), ALGORITHM=INPLACE, LOCK=NONE; +SET GLOBAL innodb_encryption_rotate_key_age = 1; +create table t2(id int) engine=innodb; +SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; +alter table t1 add index secret2 (private), ALGORITHM=INPLACE, LOCK=NONE; +insert into t1(credit_card) select credit_card from t1; +insert into t1(credit_card) select credit_card from t1; +insert into t1(credit_card) select credit_card from t1; +drop table t2; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; +variable_value > 0 +1 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'; +variable_value > 0 +1 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'; +variable_value > 0 +1 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'; +variable_value > 0 +1 +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'; +variable_value > 0 +1 +drop table t1; +SET GLOBAL innodb_file_format=Antelope; +SET GLOBAL innodb_encryption_rotate_key_age=15; diff --git a/mysql-test/suite/encryption/t/innodb-rowlog.opt b/mysql-test/suite/encryption/t/innodb-rowlog.opt new file mode 100644 index 00000000000..6338ddbde35 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-rowlog.opt @@ -0,0 +1,7 @@ +--aria-encrypt-tables +--encrypt-tmp-disk-tables +--innodb-encrypt-tables +--innodb-encrypt-log +--innodb-encryption-rotate-key-age=15 +--innodb-encryption-threads=4 +--innodb-tablespaces-encryption diff --git a/mysql-test/suite/encryption/t/innodb-rowlog.test b/mysql-test/suite/encryption/t/innodb-rowlog.test new file mode 100644 index 00000000000..135293ef4b9 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-rowlog.test @@ -0,0 +1,90 @@ +-- source include/have_innodb.inc +-- source include/have_example_key_management_plugin.inc +# needs dbug_dbug +-- source include/have_debug.inc + +--disable_warnings +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $key_age = `SELECT @@innodb_encryption_rotate_key_age`; +--enable_query_log + +SET GLOBAL innodb_file_format = `Barracuda`; +--enable_warnings + +let $MYSQLD_DATADIR = `SELECT @@datadir`; +let $MYSQLD_TMPDIR = `SELECT @@tmpdir`; + +# +# Create a table that will be encrypted and put some sensitive data to it (credit card numbers) +# + +create table t1(id int not null primary key auto_increment, credit_card varchar(200), private varchar(50)) engine=innodb; + +let $rows = 15000; +--disable_query_log +begin; +while ($rows) +{ + eval insert into t1 values(NULL, '0000-0000-0000-0000','private_data'); + dec $rows; +} +commit; +--enable_query_log + +--let $wait_timeout= 600 +--let $wait_condition=SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 +--source include/wait_condition.inc + +# +# Now we create secondary index for credit_card column in parallel we create new rows +# forcing alter table to wait so that row log is used. +# +SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; +send alter table t1 add index secret (credit_card), ALGORITHM=INPLACE, LOCK=NONE; +# +# Force key rotation and create second index for same table +# +connect (con2,localhost,root,,); +SET GLOBAL innodb_encryption_rotate_key_age = 1; +create table t2(id int) engine=innodb; +SET SESSION debug_dbug="+d,ib_merge_wait_after_read"; +send alter table t1 add index secret2 (private), ALGORITHM=INPLACE, LOCK=NONE; +# +# Create new rows to row log +# +connect (con1,localhost,root,,); +insert into t1(credit_card) select credit_card from t1; +insert into t1(credit_card) select credit_card from t1; +insert into t1(credit_card) select credit_card from t1; + +connection default; +reap; +disconnect con1; + +connection con2; +reap; +drop table t2; + +connection default; +disconnect con2; + +let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED'; +--source include/wait_condition.inc + +# +# Verify that both merge blocks and row log blocks are encryted and decrypted +# + +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_encrypted'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_merge_blocks_decrypted'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_encrypted'; +SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_encryption_n_rowlog_blocks_decrypted'; + +drop table t1; + +--disable_warnings +eval SET GLOBAL innodb_file_format=$innodb_file_format_orig; +eval SET GLOBAL innodb_encryption_rotate_key_age=$key_age; +--enable_warnings diff --git a/mysql-test/t/ctype_utf8.test b/mysql-test/t/ctype_utf8.test index d1fe93d9251..877db3f54f6 100644 --- a/mysql-test/t/ctype_utf8.test +++ b/mysql-test/t/ctype_utf8.test @@ -1990,6 +1990,18 @@ SET @@SQL_MODE=default; #DROP FUNCTION f1; #SET NAMES utf8; +--echo # +--echo # MDEV-10191 non convertible chars convert() resulted in Null instead "?" on Windows +--echo # + +SET sql_mode='STRICT_TRANS_TABLES'; +SELECT CONVERT(_utf8 0xC499 USING latin1); +SELECT CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1); + +SET sql_mode=default; +SELECT CONVERT(_utf8 0xC499 USING latin1); +SELECT CAST(_utf8 0xC499 AS CHAR CHARACTER SET latin1); + --echo # --echo # End of 10.1 tests diff --git a/sql/item.cc b/sql/item.cc index 6db1b355ed3..d4c860a009c 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -5982,7 +5982,7 @@ String_copier_for_item::copy_with_warn(CHARSET_INFO *dstcs, String *dst, srccs == &my_charset_bin ? dstcs->csname : srccs->csname, err.ptr()); - return m_thd->is_strict_mode(); + return false; } if (const char *pos= cannot_convert_error_pos()) { @@ -5994,7 +5994,7 @@ String_copier_for_item::copy_with_warn(CHARSET_INFO *dstcs, String *dst, ER_CANNOT_CONVERT_CHARACTER, ER_THD(m_thd, ER_CANNOT_CONVERT_CHARACTER), srccs->csname, buf, dstcs->csname); - return m_thd->is_strict_mode(); + return false; } return false; } diff --git a/sql/item.h b/sql/item.h index cee88756bbf..3bf1fd1113c 100644 --- a/sql/item.h +++ b/sql/item.h @@ -587,8 +587,7 @@ class Item_func_not; class Item_splocal; /** - String_copier that honors the current sql_mode (strict vs non strict) - and can send warnings. + String_copier that sends Item specific warnings. */ class String_copier_for_item: public String_copier { diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc index 70f160991d0..599d818cf3f 100644 --- a/sql/item_timefunc.cc +++ b/sql/item_timefunc.cc @@ -2422,7 +2422,7 @@ String *Item_char_typecast::copy(String *str, CHARSET_INFO *strcs) if (copier.copy_with_warn(cast_cs, &tmp_value, strcs, str->ptr(), str->length(), cast_length)) { - null_value= 1; // In strict mode: malformed data or could not convert + null_value= 1; // EOM return 0; } check_truncation_with_warn(str, copier.source_end_pos() - str->ptr()); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 5e7904b2b84..f6b065e96b1 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1200,6 +1200,18 @@ static SHOW_VAR innodb_status_variables[]= { {"encryption_key_rotation_list_length", (char*)&export_vars.innodb_key_rotation_list_length, SHOW_LONGLONG}, + {"encryption_n_merge_blocks_encrypted", + (char*)&export_vars.innodb_n_merge_blocks_encrypted, + SHOW_LONGLONG}, + {"encryption_n_merge_blocks_decrypted", + (char*)&export_vars.innodb_n_merge_blocks_decrypted, + SHOW_LONGLONG}, + {"encryption_n_rowlog_blocks_encrypted", + (char*)&export_vars.innodb_n_rowlog_blocks_encrypted, + SHOW_LONGLONG}, + {"encryption_n_rowlog_blocks_decrypted", + (char*)&export_vars.innodb_n_rowlog_blocks_decrypted, + SHOW_LONGLONG}, /* scrubing */ {"scrub_background_page_reorganizations", diff --git a/storage/innobase/include/log0crypt.h b/storage/innobase/include/log0crypt.h index a5f7c56cc3f..f5a694fac15 100644 --- a/storage/innobase/include/log0crypt.h +++ b/storage/innobase/include/log0crypt.h @@ -28,6 +28,7 @@ MDEV-11782: Rewritten for MariaDB 10.2 by Marko Mäkelä, MariaDB Corporation. #define log0crypt_h #include "log0log.h" +#include "os0file.h" /** innodb_encrypt_log: whether to encrypt the redo log */ extern my_bool srv_encrypt_log; @@ -82,4 +83,45 @@ UNIV_INTERN void log_crypt(byte* buf, lsn_t lsn, ulint size, bool decrypt = false); +/** Encrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_encrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) + MY_ATTRIBUTE((warn_unused_result)); + +/** Decrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_decrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) + MY_ATTRIBUTE((warn_unused_result)); + +/** Find out is temporary log files encrypted. +@return true if temporary log file should be encrypted, false if not */ +UNIV_INTERN +bool +log_tmp_is_encrypted() MY_ATTRIBUTE((warn_unused_result)); #endif // log0crypt.h diff --git a/storage/innobase/include/row0ftsort.h b/storage/innobase/include/row0ftsort.h index b1f3b8ab328..c8556cc4ca4 100644 --- a/storage/innobase/include/row0ftsort.h +++ b/storage/innobase/include/row0ftsort.h @@ -73,7 +73,6 @@ struct fts_psort_common_t { store Doc ID during sort, if Doc ID will not be big enough to use 8 bytes value */ - fil_space_crypt_t* crypt_data; /*!< crypt data or NULL */ }; struct fts_psort_t { diff --git a/storage/innobase/include/row0merge.h b/storage/innobase/include/row0merge.h index 50c3361a3f9..ea4bd92d3dd 100644 --- a/storage/innobase/include/row0merge.h +++ b/storage/innobase/include/row0merge.h @@ -44,9 +44,6 @@ Created 13/06/2005 Jan Lindstrom /* Reserve free space from every block for key_version */ #define ROW_MERGE_RESERVE_SIZE 4 -/* Reserve free space from every block for key_version */ -#define ROW_MERGE_RESERVE_SIZE 4 - /* Cluster index read task is mandatory */ #define COST_READ_CLUSTERED_INDEX 1.0 @@ -374,18 +371,16 @@ row_merge_buf_sort( /********************************************************************//** Write a merge block to the file system. @return TRUE if request was successful, FALSE if fail */ -ibool +bool row_merge_write( /*============*/ int fd, /*!< in: file descriptor */ ulint offset, /*!< in: offset where to write, in number of row_merge_block_t elements */ const void* buf, /*!< in: data */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ void* crypt_buf, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ MY_ATTRIBUTE((warn_unused_result)); - /********************************************************************//** Empty a sort buffer. @return sort buffer */ @@ -414,7 +409,6 @@ row_merge_file_create( @param[in] update_progress true, if we should update progress status @param[in] pct_progress total progress percent until now @param[in] pct_ocst current progress percent -@param[in] crypt_data tale crypt data @param[in] crypt_block crypt buf or NULL @param[in] space space_id @param[in,out] stage performance schema accounting object, used by @@ -432,7 +426,6 @@ row_merge_sort( const bool update_progress, const double pct_progress, const double pct_cost, - fil_space_crypt_t* crypt_data, row_merge_block_t* crypt_block, ulint space, ut_stage_alter_t* stage = NULL) @@ -473,7 +466,6 @@ row_merge_read( in number of row_merge_block_t elements */ row_merge_block_t* buf, /*!< out: data */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ MY_ATTRIBUTE((warn_unused_result)); @@ -494,8 +486,8 @@ row_merge_read_rec( or NULL on end of list (non-NULL on I/O error) */ ulint* offsets,/*!< out: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ MY_ATTRIBUTE((warn_unused_result)); + #endif /* row0merge.h */ diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 3eddd300acc..a51520e881c 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -129,6 +129,14 @@ struct srv_stats_t ulint_ctr_64_t pages_encrypted; /* Number of pages decrypted */ ulint_ctr_64_t pages_decrypted; + /* Number of merge blocks encrypted */ + ulint_ctr_64_t n_merge_blocks_encrypted; + /* Number of merge blocks decrypted */ + ulint_ctr_64_t n_merge_blocks_decrypted; + /* Number of row log blocks encrypted */ + ulint_ctr_64_t n_rowlog_blocks_encrypted; + /* Number of row log blocks decrypted */ + ulint_ctr_64_t n_rowlog_blocks_decrypted; /** Number of data read in total (in bytes) */ ulint_ctr_1_t data_read; @@ -1052,6 +1060,15 @@ struct export_var_t{ int64_t innodb_pages_decrypted; /*!< Number of pages decrypted */ + /*!< Number of merge blocks encrypted */ + ib_int64_t innodb_n_merge_blocks_encrypted; + /*!< Number of merge blocks decrypted */ + ib_int64_t innodb_n_merge_blocks_decrypted; + /*!< Number of row log blocks encrypted */ + ib_int64_t innodb_n_rowlog_blocks_encrypted; + /*!< Number of row log blocks decrypted */ + ib_int64_t innodb_n_rowlog_blocks_decrypted; + ulint innodb_sec_rec_cluster_reads; /*!< srv_sec_rec_cluster_reads */ ulint innodb_sec_rec_cluster_reads_avoided;/*!< srv_sec_rec_cluster_reads_avoided */ diff --git a/storage/innobase/log/log0crypt.cc b/storage/innobase/log/log0crypt.cc index c0a10c74073..c91633d4f31 100644 --- a/storage/innobase/log/log0crypt.cc +++ b/storage/innobase/log/log0crypt.cc @@ -161,6 +161,128 @@ log_crypt(byte* buf, lsn_t lsn, ulint size, bool decrypt) } } +/** Encrypt/decrypt temporary log blocks. + +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] what ENCRYPTION_FLAG_ENCRYPT or + ENCRYPTION_FLAG_DECRYPT +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +static +bool +log_tmp_blocks_crypt( + const byte* src_block, + ulint size, + byte* dst_block, + int what, + os_offset_t offs, + ulint space_id) +{ + uint dst_len; + byte aes_ctr_counter[MY_AES_BLOCK_SIZE]; + byte is_encrypt= what == ENCRYPTION_FLAG_ENCRYPT; + const crypt_info_t* info = static_cast<const crypt_info_t*>(&infos[0]); + + // AES_CTR_COUNTER = space_id + offs + + bzero(aes_ctr_counter, MY_AES_BLOCK_SIZE); + mach_write_to_8(aes_ctr_counter, space_id); + mach_write_to_8(aes_ctr_counter + 8, offs); + + int rc = encryption_crypt(src_block, size, + dst_block, &dst_len, + const_cast<byte*>(info->crypt_key.bytes), 16, + aes_ctr_counter, MY_AES_BLOCK_SIZE, + what | ENCRYPTION_FLAG_NOPAD, + LOG_DEFAULT_ENCRYPTION_KEY, + info->key_version); + + if (rc != MY_AES_OK) { + ib::error() << (is_encrypt ? "Encryption" : "Decryption") + << " failed for temporary log file with rc = " << rc; + return false; + } + + return true; +} + +/** Get crypt info +@return pointer to log crypt info or NULL +*/ +inline +const crypt_info_t* +get_crypt_info() +{ + mutex_enter(&log_sys->mutex); + const crypt_info_t* info = get_crypt_info(log_sys->next_checkpoint_no); + mutex_exit(&log_sys->mutex); + + return info; +} + +/** Find out is temporary log files encrypted. +@return true if temporary log file should be encrypted, false if not */ +UNIV_INTERN +bool +log_tmp_is_encrypted() +{ + const crypt_info_t* info = get_crypt_info(); + + if (info == NULL || + info->key_version == ENCRYPTION_KEY_VERSION_INVALID || + !info->key_version) { + return false; + } + + return true; +} + +/** Encrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_encrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) +{ + return (log_tmp_blocks_crypt(src_block, size, dst_block, + ENCRYPTION_FLAG_ENCRYPT, offs, space_id)); +} + +/** Decrypt temporary log block. +@param[in] src_block block to encrypt or decrypt +@param[in] size size of the block +@param[out] dst_block destination block +@param[in] offs offset to block +@param[in] space_id tablespace id +@return true if successfull, false in case of failure +*/ +UNIV_INTERN +bool +log_tmp_block_decrypt( + const byte* src_block, + ulint size, + byte* dst_block, + os_offset_t offs, + ulint space_id) +{ + return (log_tmp_blocks_crypt(src_block, size, dst_block, + ENCRYPTION_FLAG_DECRYPT, offs, space_id)); +} + /** Generate crypt key from crypt msg. @param[in,out] info encryption key @param[in] upgrade whether to use the key in MariaDB 10.1 format diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index da5d96ccc18..027d4503885 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -35,6 +35,7 @@ Created 10/13/2010 Jimmy Yang #include "btr0cur.h" #include "btr0bulk.h" #include "fts0plugin.h" +#include "log0crypt.h" /** Read the next record to buffer N. @param N index into array of merge info structure */ @@ -43,7 +44,7 @@ Created 10/13/2010 Jimmy Yang b[N] = row_merge_read_rec( \ block[N], buf[N], b[N], index, \ fd[N], &foffs[N], &mrec[N], offsets[N], \ - crypt_data, crypt_block[N], space); \ + crypt_block[N], space); \ if (UNIV_UNLIKELY(!b[N])) { \ if (mrec[N]) { \ goto exit; \ @@ -194,7 +195,6 @@ row_fts_psort_info_init( fts_psort_t* merge_info = NULL; ulint block_size; ibool ret = TRUE; - fil_space_crypt_t* crypt_data = NULL; bool encrypted = false; block_size = 3 * srv_sort_buf_size; @@ -225,21 +225,8 @@ row_fts_psort_info_init( common_info->merge_event = os_event_create(0); common_info->opt_doc_id_size = opt_doc_id_size; - /* Theoretically the tablespace can be dropped straight away. - In practice, the DDL completion will wait for this thread to - finish. */ - if (fil_space_t* space = fil_space_acquire(new_table->space)) { - crypt_data = space->crypt_data; - fil_space_release(space); - } - - if (crypt_data && crypt_data->should_encrypt()) { - common_info->crypt_data = crypt_data; + if (log_tmp_is_encrypted()) { encrypted = true; - } else { - /* Not needed */ - common_info->crypt_data = NULL; - crypt_data = NULL; } ut_ad(trx->mysql_thd != NULL); @@ -698,11 +685,9 @@ row_merge_fts_doc_tokenize( cur_len += len; dfield_dup(field, buf->heap); - /* Reserve one byte for the end marker of row_merge_block_t - and we have reserved ROW_MERGE_RESERVE_SIZE (= 4) for - encryption key_version in the beginning of the buffer. */ + /* Reserve one byte for the end marker of row_merge_block_t */ if (buf->total_size + data_size[idx] + cur_len - >= (srv_sort_buf_size - 1 - ROW_MERGE_RESERVE_SIZE)) { + >= (srv_sort_buf_size - 1)) { buf_full = TRUE; break; } @@ -798,7 +783,6 @@ fts_parallel_tokenization( fts_tokenize_ctx_t t_ctx; ulint retried = 0; dberr_t error = DB_SUCCESS; - fil_space_crypt_t* crypt_data = NULL; ut_ad(psort_info->psort_common->trx->mysql_thd != NULL); @@ -825,7 +809,6 @@ fts_parallel_tokenization( block = psort_info->merge_block; crypt_block = psort_info->crypt_block; - crypt_data = psort_info->psort_common->crypt_data; const page_size_t& page_size = dict_table_page_size(table); @@ -918,7 +901,6 @@ loop: if (!row_merge_write(merge_file[t_ctx.buf_used]->fd, merge_file[t_ctx.buf_used]->offset++, block[t_ctx.buf_used], - crypt_data, crypt_block[t_ctx.buf_used], table->space)) { error = DB_TEMP_FILE_WRITE_FAIL; @@ -1013,7 +995,6 @@ exit: if (!row_merge_write(merge_file[i]->fd, merge_file[i]->offset++, block[i], - crypt_data, crypt_block[i], table->space)) { error = DB_TEMP_FILE_WRITE_FAIL; @@ -1053,7 +1034,7 @@ exit: psort_info->psort_common->dup, merge_file[i], block[i], &tmpfd[i], false, 0.0/* pct_progress */, 0.0/* pct_cost */, - crypt_data, crypt_block[i], table->space); + crypt_block[i], table->space); if (error != DB_SUCCESS) { close(tmpfd[i]); @@ -1595,7 +1576,6 @@ row_fts_merge_insert( ulint start; fts_psort_insert_t ins_ctx; ulint count_diag = 0; - fil_space_crypt_t* crypt_data = NULL; ulint space=0; fts_table_t fts_table; char aux_table_name[MAX_FULL_NAME_LEN]; @@ -1618,7 +1598,6 @@ row_fts_merge_insert( trx->op_info = "inserting index entries"; ins_ctx.opt_doc_id_size = psort_info[0].psort_common->opt_doc_id_size; - crypt_data = psort_info[0].psort_common->crypt_data; heap = mem_heap_create(500 + sizeof(mrec_buf_t)); @@ -1736,7 +1715,6 @@ row_fts_merge_insert( && (!row_merge_read( fd[i], foffs[i], (row_merge_block_t*) block[i], - crypt_data, (row_merge_block_t*) crypt_block[i], space))) { error = DB_CORRUPTION; diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc index 9e946757df3..881563276dd 100644 --- a/storage/innobase/row/row0log.cc +++ b/storage/innobase/row/row0log.cc @@ -37,6 +37,7 @@ Created 2011-05-26 Marko Makela #include "ut0new.h" #include "ut0stage.h" #include "trx0rec.h" +#include "log0crypt.h" #include <algorithm> #include <map> @@ -179,6 +180,7 @@ struct row_log_t { dict_table_t* table; /*!< table that is being rebuilt, or NULL when this is a secondary index that is being created online */ + dict_index_t* index; /*!< index to be build */ bool same_pk;/*!< whether the definition of the PRIMARY KEY has remained the same */ const dtuple_t* add_cols; @@ -194,8 +196,14 @@ struct row_log_t { row_log_buf_t tail; /*!< writer context; protected by mutex and index->lock S-latch, or by index->lock X-latch only */ + byte* crypt_tail; /*!< writer context; + temporary buffer used in encryption, + decryption or NULL*/ row_log_buf_t head; /*!< reader context; protected by MDL only; modifiable by row_log_apply_ops() */ + byte* crypt_head; /*!< reader context; + temporary buffer used in encryption, + decryption or NULL */ ulint n_old_col; /*!< number of non-virtual column in old table */ @@ -358,6 +366,7 @@ row_log_online_op( = (os_offset_t) log->tail.blocks * srv_sort_buf_size; IORequest request(IORequest::WRITE); + byte * buf = log->tail.block; if (byte_offset + srv_sort_buf_size >= srv_online_max_size) { goto write_failed; @@ -378,19 +387,39 @@ row_log_online_op( goto err_exit; } + /* If encryption is enabled encrypt buffer before writing it + to file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt(log->tail.block, + srv_sort_buf_size, + log->crypt_tail, + byte_offset, + index->table->space)) { + log->error = DB_DECRYPTION_FAILED; + goto err_exit; + } + + srv_stats.n_rowlog_blocks_encrypted.inc(); + buf = log->crypt_tail; + } + log->tail.blocks++; + if (!os_file_write_int_fd( request, - "(modification log)", - log->fd, - log->tail.block, byte_offset, srv_sort_buf_size)) { + "(modification log)", + log->fd, + buf, byte_offset, srv_sort_buf_size)) { + write_failed: /* We set the flag directly instead of invoking dict_set_corrupted_index_cache_only(index) here, because the index is not "public" yet. */ index->type |= DICT_CORRUPT; } + UNIV_MEM_INVALID(log->tail.block, srv_sort_buf_size); + memcpy(log->tail.block, log->tail.buf + avail_size, mrec_size - avail_size); log->tail.bytes = mrec_size - avail_size; @@ -474,6 +503,7 @@ row_log_table_close_func( = (os_offset_t) log->tail.blocks * srv_sort_buf_size; IORequest request(IORequest::WRITE); + byte * buf = log->tail.block; if (byte_offset + srv_sort_buf_size >= srv_online_max_size) { goto write_failed; @@ -495,12 +525,32 @@ row_log_table_close_func( } log->tail.blocks++; + + /* If encryption is enabled encrypt buffer before writing it + to file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt(log->tail.block, + srv_sort_buf_size, + log->crypt_tail, + byte_offset, + log->index->table->space)) { + log->error = DB_DECRYPTION_FAILED; + goto err_exit; + } + + srv_stats.n_rowlog_blocks_encrypted.inc(); + buf = log->crypt_tail; + } + if (!os_file_write_int_fd( request, "(modification log)", log->fd, - log->tail.block, byte_offset, srv_sort_buf_size)) { + buf, byte_offset, srv_sort_buf_size)) { write_failed: + /* We set the flag directly instead of invoking + dict_set_corrupted_index_cache_only(index) here, + because the index is not "public" yet. */ log->error = DB_ONLINE_LOG_TOO_BIG; } UNIV_MEM_INVALID(log->tail.block, srv_sort_buf_size); @@ -2885,12 +2935,13 @@ all_done: } IORequest request(IORequest::READ); + byte * buf = index->online_log->head.block; if (!os_file_read_no_error_handling_int_fd( request, index->online_log->fd, - index->online_log->head.block, ofs, + buf, ofs, srv_sort_buf_size)) { ib::error() << "Unable to read temporary file" @@ -2898,6 +2949,22 @@ all_done: goto corruption; } + /* If encryption is enabled decrypt buffer after reading it + from file system. */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, + srv_sort_buf_size, + index->online_log->crypt_head, + ofs, + index->table->space)) { + error = DB_DECRYPTION_FAILED; + goto func_exit; + } + + srv_stats.n_rowlog_blocks_decrypted.inc(); + memcpy(buf, index->online_log->crypt_head, srv_sort_buf_size); + } + #ifdef POSIX_FADV_DONTNEED /* Each block is read exactly once. Free up the file cache. */ posix_fadvise(index->online_log->fd, @@ -3198,10 +3265,23 @@ row_log_allocate( log->path = path; log->n_old_col = index->table->n_cols; log->n_old_vcol = index->table->n_v_cols; + log->crypt_tail = log->crypt_head = NULL; + log->index = index; dict_index_set_online_status(index, ONLINE_INDEX_CREATION); index->online_log = log; + if (log_tmp_is_encrypted()) { + ulint size = srv_sort_buf_size; + log->crypt_head = static_cast<byte *>(os_mem_alloc_large(&size)); + log->crypt_tail = static_cast<byte *>(os_mem_alloc_large(&size)); + + if (!log->crypt_head || !log->crypt_tail) { + row_log_free(log); + DBUG_RETURN(false); + } + } + /* While we might be holding an exclusive data dictionary lock here, in row_log_abort_sec() we will not always be holding it. Use atomic operations in both cases. */ @@ -3223,6 +3303,15 @@ row_log_free( row_log_block_free(log->tail); row_log_block_free(log->head); row_merge_file_destroy_low(log->fd); + + if (log->crypt_head) { + os_mem_free_large(log->crypt_head, srv_sort_buf_size); + } + + if (log->crypt_tail) { + os_mem_free_large(log->crypt_tail, srv_sort_buf_size); + } + mutex_free(&log->mutex); ut_free(log); log = NULL; @@ -3717,10 +3806,12 @@ all_done: goto func_exit; } + byte* buf = index->online_log->head.block; + if (!os_file_read_no_error_handling_int_fd( request, index->online_log->fd, - index->online_log->head.block, ofs, + buf, ofs, srv_sort_buf_size)) { ib::error() << "Unable to read temporary file" @@ -3728,6 +3819,20 @@ all_done: goto corruption; } + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, + srv_sort_buf_size, + index->online_log->crypt_head, + ofs, + index->table->space)) { + error = DB_DECRYPTION_FAILED; + goto func_exit; + } + + srv_stats.n_rowlog_blocks_decrypted.inc(); + memcpy(buf, index->online_log->crypt_head, srv_sort_buf_size); + } + #ifdef POSIX_FADV_DONTNEED /* Each block is read exactly once. Free up the file cache. */ posix_fadvise(index->online_log->fd, diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 85fac6c0bba..f5c0b286883 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -50,6 +50,7 @@ Completed by Sunny Bains and Marko Makela #include "ut0new.h" #include "ut0stage.h" #include "fil0crypt.h" +#include "log0crypt.h" float my_log2f(float n) { @@ -270,89 +271,6 @@ private: /* Maximum pending doc memory limit in bytes for a fts tokenization thread */ #define FTS_PENDING_DOC_MEMORY_LIMIT 1000000 -/******************************************************//** -Encrypt a merge block. */ -static -void -row_merge_encrypt_buf( -/*==================*/ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ - ulint offset, /*!< in: offset where to - write */ - ulint space, /*!< in: tablespace id */ - const byte* input_buf, /*!< in: input buffer */ - byte* crypted_buf) /*!< out: crypted buffer */ -{ - uint key_version; - uint dstlen=0; - uint ofs = (uint)(srv_sort_buf_size * offset); - - key_version = encryption_key_get_latest_version(crypt_data->key_id); - - /* Store key_version at the beginning of the input buffer */ - mach_write_to_4((byte *)crypted_buf, key_version); - - int rc = encryption_scheme_encrypt(input_buf+ROW_MERGE_RESERVE_SIZE, - srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE, - crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen, - crypt_data, key_version, - space, ofs, 0); - - if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) { - ib::fatal() - << "Unable to encrypt data-block " - " src: " << static_cast<const void*>(input_buf) - << " srclen: " << srv_sort_buf_size - << " buf: " << static_cast<const void*>(crypted_buf) - << " buflen: " << dstlen - << ". return-code: " << rc << ". Can't continue!"; - } -} - -/******************************************************//** -Decrypt a merge block. */ -static -bool -row_merge_decrypt_buf( -/*==================*/ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ - ulint offset, /*!< in: offset where to - write */ - ulint space, /*!< in: tablespace id */ - const byte* input_buf, /*!< in: input buffer */ - byte* crypted_buf) /*!< out: crypted buffer */ -{ - uint key_version; - uint dstlen=0; - uint ofs = (uint)(srv_sort_buf_size * offset); - - /* Read key_version from beginning of the buffer */ - key_version = mach_read_from_4((byte *)input_buf); - - if (key_version == 0) { - /* block not encrypted */ - return false; - } - - int rc = encryption_scheme_decrypt(input_buf+ROW_MERGE_RESERVE_SIZE, - srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE, - crypted_buf+ROW_MERGE_RESERVE_SIZE, &dstlen, - crypt_data, key_version, - space, ofs, 0); - - if (! ((rc == MY_AES_OK) && ((ulint)dstlen == srv_sort_buf_size-ROW_MERGE_RESERVE_SIZE))) { - ib::fatal() - << "Unable to decrypt data-block " - << " src: " << static_cast<const void*>(input_buf) - << " srclen: " << srv_sort_buf_size - << " buf: " << static_cast<const void*>(crypted_buf) - << " buflen: " << dstlen - << ". return-code: " << rc << ". Can't continue!"; - } - - return true; -} - /* Maximum pending doc memory limit in bytes for a fts tokenization thread */ #define FTS_PENDING_DOC_MEMORY_LIMIT 1000000 @@ -384,7 +302,6 @@ row_merge_insert_index_tuples( percent until now */ const double pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space, /*!< in: space id */ ut_stage_alter_t* stage = NULL); @@ -469,7 +386,7 @@ row_merge_buf_create( ulint buf_size; mem_heap_t* heap; - max_tuples = (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE) + max_tuples = (srv_sort_buf_size) / ut_max(static_cast<ulint>(1), dict_index_get_min_size(index)); @@ -917,7 +834,7 @@ row_merge_buf_add( ut_ad(data_size < srv_sort_buf_size); /* Reserve bytes for the end marker of row_merge_block_t. */ - if (buf->total_size + data_size >= (srv_sort_buf_size - ROW_MERGE_RESERVE_SIZE)) { + if (buf->total_size + data_size >= srv_sort_buf_size) { DBUG_RETURN(0); } @@ -1039,7 +956,7 @@ respectively */ /**********************************************************************//** Merge sort the tuple buffer in main memory. */ -static MY_ATTRIBUTE((nonnull(4,5))) +static void row_merge_tuple_sort( /*=================*/ @@ -1090,7 +1007,7 @@ row_merge_buf_write( { const dict_index_t* index = buf->index; ulint n_fields= dict_index_get_n_fields(index); - byte* b = &block[ROW_MERGE_RESERVE_SIZE]; + byte* b = &block[0]; DBUG_ENTER("row_merge_buf_write"); @@ -1109,7 +1026,7 @@ row_merge_buf_write( /* Write an "end-of-chunk" marker. */ ut_a(b < &block[srv_sort_buf_size]); - ut_a(b == &block[0] + buf->total_size + ROW_MERGE_RESERVE_SIZE); + ut_a(b == &block[0] + buf->total_size); *b++ = 0; #ifdef UNIV_DEBUG_VALGRIND /* The rest of the block is uninitialized. Initialize it @@ -1163,9 +1080,8 @@ row_merge_read( in number of row_merge_block_t elements */ row_merge_block_t* buf, /*!< out: data */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { os_offset_t ofs = ((os_offset_t) offset) * srv_sort_buf_size; @@ -1178,10 +1094,14 @@ row_merge_read( request, fd, buf, ofs, srv_sort_buf_size); /* For encrypted tables, decrypt data after reading and copy data */ - if (crypt_data && crypt_buf) { - if (row_merge_decrypt_buf(crypt_data, offset, space, buf, crypt_buf)) { - memcpy(buf, crypt_buf, srv_sort_buf_size); + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, srv_sort_buf_size, + crypt_buf, ofs, space)) { + return (FALSE); } + + srv_stats.n_merge_blocks_decrypted.inc(); + memcpy(buf, crypt_buf, srv_sort_buf_size); } #ifdef POSIX_FADV_DONTNEED @@ -1199,14 +1119,13 @@ row_merge_read( /********************************************************************//** Write a merge block to the file system. @return TRUE if request was successful, FALSE if fail */ -ibool +bool row_merge_write( /*============*/ int fd, /*!< in: file descriptor */ ulint offset, /*!< in: offset where to write, in number of row_merge_block_t elements */ const void* buf, /*!< in: data */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ void* crypt_buf, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ { @@ -1218,12 +1137,14 @@ row_merge_write( DBUG_LOG("ib_merge_sort", "fd=" << fd << " ofs=" << ofs); DBUG_EXECUTE_IF("row_merge_write_failure", DBUG_RETURN(FALSE);); - if (crypt_data && crypt_buf) { - row_merge_encrypt_buf(crypt_data, offset, space, (const byte *)buf, (byte *)crypt_buf); + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt((const byte *)buf, buf_len, + (byte *)crypt_buf, ofs, space)) { + return (FALSE); + } + + srv_stats.n_merge_blocks_encrypted.inc(); out_buf = crypt_buf; - } else { - /* Mark block unencrypted */ - mach_write_to_4((byte *)out_buf, 0); } IORequest request(IORequest::WRITE); @@ -1255,9 +1176,8 @@ row_merge_read_rec( or NULL on end of list (non-NULL on I/O error) */ ulint* offsets,/*!< out: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { ulint extra_size; ulint data_size; @@ -1271,10 +1191,6 @@ row_merge_read_rec( DBUG_ENTER("row_merge_read_rec"); - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - extra_size = *b++; if (UNIV_UNLIKELY(!extra_size)) { @@ -1292,7 +1208,8 @@ row_merge_read_rec( if (UNIV_UNLIKELY(b >= &block[srv_sort_buf_size])) { if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { err_exit: /* Signal I/O error. */ *mrec = b; @@ -1300,7 +1217,7 @@ err_exit: } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; } extra_size = (extra_size & 0x7f) << 8; @@ -1322,13 +1239,14 @@ err_exit: memcpy(*buf, b, avail_size); if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { goto err_exit; } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; /* Copy the record. */ memcpy(*buf + avail_size, b, extra_size - avail_size); @@ -1383,13 +1301,14 @@ err_exit: ut_d(offsets[3] = (ulint) index); if (!row_merge_read(fd, ++(*foffs), block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { goto err_exit; } /* Wrap around to the beginning of the buffer. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; /* Copy the rest of the record. */ memcpy(*buf + avail_size, b, extra_size + data_size - avail_size); @@ -1461,7 +1380,6 @@ row_merge_write_rec( ulint* foffs, /*!< in/out: file offset */ const mrec_t* mrec, /*!< in: record to write */ const ulint* offsets,/*!< in: offsets of mrec */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ { @@ -1484,10 +1402,6 @@ row_merge_write_rec( size = extra_size + (extra_size >= 0x80) + rec_offs_data_size(offsets); - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - if (UNIV_UNLIKELY(b + size >= &block[srv_sort_buf_size])) { /* The record spans two blocks. Copy it to the temporary buffer first. */ @@ -1503,14 +1417,15 @@ row_merge_write_rec( memcpy(b, buf[0], avail_size); if (!row_merge_write(fd, (*foffs)++, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(NULL); } UNIV_MEM_INVALID(&block[0], srv_sort_buf_size); /* Copy the rest. */ - b = &block[ROW_MERGE_RESERVE_SIZE]; + b = &block[0]; memcpy(b, buf[0] + avail_size, size - avail_size); b += size - avail_size; } else { @@ -1533,7 +1448,6 @@ row_merge_write_eof( byte* b, /*!< in: pointer to end of block */ int fd, /*!< in: file descriptor */ ulint* foffs, /*!< in/out: file offset */ - fil_space_crypt_t* crypt_data, /*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ { @@ -1548,10 +1462,6 @@ row_merge_write_eof( reinterpret_cast<const void*>(block) << ",fd=" << fd << ',' << *foffs); - if (b == &block[0]) { - b+= ROW_MERGE_RESERVE_SIZE; - } - *b++ = 0; UNIV_MEM_ASSERT_RW(&block[0], b - &block[0]); UNIV_MEM_ASSERT_W(&block[0], srv_sort_buf_size); @@ -1563,7 +1473,7 @@ row_merge_write_eof( #endif /* UNIV_DEBUG_VALGRIND */ if (!row_merge_write(fd, (*foffs)++, block, - crypt_data, crypt_block, space)) { + crypt_block, space)) { DBUG_RETURN(NULL); } @@ -1790,7 +1700,6 @@ row_merge_read_clustered_index( int* tmpfd, ut_stage_alter_t* stage, double pct_cost, - fil_space_crypt_t* crypt_data, row_merge_block_t* crypt_block, struct TABLE* eval_table) @@ -2491,7 +2400,6 @@ write_buffers: curr_progress, pct_cost, NULL, - NULL, new_table->space); if (row == NULL) { @@ -2603,7 +2511,6 @@ write_buffers: curr_progress, pct_cost, NULL, - NULL, new_table->space); err = btr_bulk.finish(err); @@ -2636,7 +2543,7 @@ write_buffers: row_merge_buf_write(buf, file, block); if (!row_merge_write(file->fd, file->offset++, block, - crypt_data, crypt_block, new_table->space)) { + crypt_block, new_table->space)) { err = DB_TEMP_FILE_WRITE_FAIL; trx->error_key_num = i; break; @@ -2689,7 +2596,7 @@ write_buffers: if(read_rows % 1000 == 0) { /* Update progress for each 1000 rows */ curr_progress = (read_rows >= table_total_rows) ? - pct_cost : + pct_cost : ((pct_cost * read_rows) / table_total_rows); /* presenting 10.12% as 1012 integer */ onlineddl_pct_progress = (ulint) (curr_progress * 100); @@ -2845,9 +2752,8 @@ wait_again: &buf[2], b2, \ of->fd, &of->offset, \ mrec##N, offsets##N, \ - crypt_data, \ crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL , \ - space); \ + space); \ if (UNIV_UNLIKELY(!b2 || ++of->n_rec > file->n_rec)) { \ goto corrupt; \ } \ @@ -2855,7 +2761,6 @@ wait_again: &buf[N], b##N, INDEX, \ file->fd, foffs##N, \ &mrec##N, offsets##N, \ - crypt_data, \ crypt_block ? &crypt_block[N * srv_sort_buf_size] : NULL, \ space); \ \ @@ -2901,7 +2806,6 @@ row_merge_blocks( ulint* foffs1, merge_file_t* of, ut_stage_alter_t* stage, - fil_space_crypt_t* crypt_data,/*!< in: crypt data or NULL */ row_merge_block_t* crypt_block,/*!< in: in/out: crypted file buffer */ ulint space) /*!< in: space id */ @@ -2930,9 +2834,11 @@ row_merge_blocks( file in two halves, which can be merged on the following pass. */ if (!row_merge_read(file->fd, *foffs0, &block[0], - crypt_data, crypt_block ? &crypt_block[0] : NULL, space) - || !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size], - crypt_data, crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, space)) { + crypt_block ? &crypt_block[0] : NULL, + space) || + !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size], + crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, + space)) { corrupt: mem_heap_free(heap); DBUG_RETURN(DB_CORRUPTION); @@ -2945,13 +2851,15 @@ corrupt: b0 = row_merge_read_rec( &block[0], &buf[0], b0, dup->index, file->fd, foffs0, &mrec0, offsets0, - crypt_data, crypt_block ? &crypt_block[0] : NULL, space); + crypt_block ? &crypt_block[0] : NULL, + space); b1 = row_merge_read_rec( &block[srv_sort_buf_size], &buf[srv_sort_buf_size], b1, dup->index, file->fd, foffs1, &mrec1, offsets1, - crypt_data, crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, space); + crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, + space); if (UNIV_UNLIKELY(!b0 && mrec0) || UNIV_UNLIKELY(!b1 && mrec1)) { @@ -2993,9 +2901,10 @@ done1: b2 = row_merge_write_eof(&block[2 * srv_sort_buf_size], b2, of->fd, &of->offset, - crypt_data, crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, space); - DBUG_RETURN(b2 ? DB_SUCCESS : DB_CORRUPTION); + crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, + space); + DBUG_RETURN(b2 ? DB_SUCCESS : DB_CORRUPTION); } /** Copy a block of index entries. @@ -3009,7 +2918,7 @@ ALTER TABLE. If not NULL stage->inc() will be called for each record processed. @return TRUE on success, FALSE on failure */ static MY_ATTRIBUTE((warn_unused_result)) -ibool +bool row_merge_blocks_copy( const dict_index_t* index, const merge_file_t* file, @@ -3017,9 +2926,8 @@ row_merge_blocks_copy( ulint* foffs0, merge_file_t* of, ut_stage_alter_t* stage, - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { mem_heap_t* heap; /*!< memory heap for offsets0, offsets1 */ @@ -3042,7 +2950,8 @@ row_merge_blocks_copy( file in two halves, which can be merged on the following pass. */ if (!row_merge_read(file->fd, *foffs0, &block[0], - crypt_data, crypt_block ? &crypt_block[0] : NULL, space)) { + crypt_block ? &crypt_block[0] : NULL, + space)) { corrupt: mem_heap_free(heap); DBUG_RETURN(FALSE); @@ -3054,7 +2963,8 @@ corrupt: b0 = row_merge_read_rec(&block[0], &buf[0], b0, index, file->fd, foffs0, &mrec0, offsets0, - crypt_data, crypt_block ? &crypt_block[0] : NULL, space); + crypt_block ? &crypt_block[0] : NULL, + space); if (UNIV_UNLIKELY(!b0 && mrec0)) { @@ -3077,8 +2987,8 @@ done0: DBUG_RETURN(row_merge_write_eof(&block[2 * srv_sort_buf_size], b2, of->fd, &of->offset, - crypt_data, - crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, space) + crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, + space) != NULL); } @@ -3106,9 +3016,8 @@ row_merge( ulint* num_run, ulint* run_offset, ut_stage_alter_t* stage, - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space) /*!< in: space id */ + ulint space) /*!< in: space id */ { ulint foffs0; /*!< first input offset */ ulint foffs1; /*!< second input offset */ @@ -3156,7 +3065,8 @@ row_merge( error = row_merge_blocks(dup, file, block, &foffs0, &foffs1, &of, stage, - crypt_data, crypt_block, space); + crypt_block, + space); if (error != DB_SUCCESS) { return(error); @@ -3177,7 +3087,8 @@ row_merge( if (!row_merge_blocks_copy(dup->index, file, block, &foffs0, &of, stage, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(DB_CORRUPTION); } } @@ -3195,7 +3106,8 @@ row_merge( if (!row_merge_blocks_copy(dup->index, file, block, &foffs1, &of, stage, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { return(DB_CORRUPTION); } } @@ -3253,7 +3165,6 @@ row_merge_sort( /*!< in: total progress percent until now */ const double pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space, /*!< in: space id */ ut_stage_alter_t* stage) @@ -3328,7 +3239,8 @@ row_merge_sort( error = row_merge(trx, dup, file, block, tmpfd, &num_runs, run_offset, stage, - crypt_data, crypt_block, space); + crypt_block, + space); if(update_progress) { merge_count++; @@ -3471,7 +3383,6 @@ row_merge_insert_index_tuples( percent until now */ const double pct_cost, /*!< in: current progress percent */ - fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space, /*!< in: space id */ ut_stage_alter_t* stage) @@ -3533,7 +3444,8 @@ row_merge_insert_index_tuples( dtuple = NULL; if (!row_merge_read(fd, foffs, block, - crypt_data, crypt_block, space)) { + crypt_block, + space)) { error = DB_CORRUPTION; goto err_exit; } else { @@ -3566,7 +3478,9 @@ row_merge_insert_index_tuples( } else { b = row_merge_read_rec(block, buf, b, index, fd, &foffs, &mrec, offsets, - crypt_data, crypt_block, space); + crypt_block, + space); + if (UNIV_UNLIKELY(!b)) { /* End of list, or I/O error */ if (mrec) { @@ -4665,7 +4579,6 @@ row_merge_build_indexes( fts_psort_t* merge_info = NULL; int64_t sig_count = 0; bool fts_psort_initiated = false; - fil_space_crypt_t * crypt_data = NULL; double total_static_cost = 0; double total_dynamic_cost = 0; @@ -4700,24 +4613,19 @@ row_merge_build_indexes( from concurrent DDL (e.g. drop table) by MDL-locks. */ FilSpace space(new_table->space); - if (const fil_space_t* fs = space()) { - crypt_data = fs->crypt_data; - } else { + if (!space()) { DBUG_RETURN(DB_TABLESPACE_NOT_FOUND); } /* If tablespace is encrypted, allocate additional buffer for encryption/decryption. */ - if (crypt_data && crypt_data->should_encrypt()) { + if (log_tmp_is_encrypted()) { crypt_block = static_cast<row_merge_block_t*>( alloc.allocate_large(3 * srv_sort_buf_size, &crypt_pfx)); if (crypt_block == NULL) { DBUG_RETURN(DB_OUT_OF_MEMORY); } - } else { - /* Not needed */ - crypt_data = NULL; } trx_start_if_not_started_xa(trx, true); @@ -4828,7 +4736,7 @@ row_merge_build_indexes( fts_sort_idx, psort_info, merge_files, key_numbers, n_indexes, add_cols, add_v, col_map, add_autoinc, sequence, block, skip_pk_sort, &tmpfd, stage, - pct_cost, crypt_data, crypt_block, eval_table); + pct_cost, crypt_block, eval_table); stage->end_phase_read_pk(); @@ -4969,7 +4877,7 @@ wait_again: trx, &dup, &merge_files[i], block, &tmpfd, true, pct_progress, pct_cost, - crypt_data, crypt_block, new_table->space, stage); + crypt_block, new_table->space, stage); pct_progress += pct_cost; @@ -5012,7 +4920,7 @@ wait_again: merge_files[i].fd, block, NULL, &btr_bulk, merge_files[i].n_rec, pct_progress, pct_cost, - crypt_data, crypt_block, new_table->space, stage); + crypt_block, new_table->space, stage); error = btr_bulk.finish(error); diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 663487fc3a6..ce1a30659ed 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -1623,6 +1623,10 @@ srv_export_innodb_status(void) export_vars.innodb_pages_page_compression_error = srv_stats.pages_page_compression_error; export_vars.innodb_pages_decrypted = srv_stats.pages_decrypted; export_vars.innodb_pages_encrypted = srv_stats.pages_encrypted; + export_vars.innodb_n_merge_blocks_encrypted = srv_stats.n_merge_blocks_encrypted; + export_vars.innodb_n_merge_blocks_decrypted = srv_stats.n_merge_blocks_decrypted; + export_vars.innodb_n_rowlog_blocks_encrypted = srv_stats.n_rowlog_blocks_encrypted; + export_vars.innodb_n_rowlog_blocks_decrypted = srv_stats.n_rowlog_blocks_decrypted; export_vars.innodb_defragment_compression_failures = btr_defragment_compression_failures; |