diff options
| -rw-r--r-- | mysql-test/suite/encryption/r/innodb-discard-import.result | 144 | ||||
| -rw-r--r-- | mysql-test/suite/encryption/t/innodb-discard-import.test | 167 | ||||
| -rw-r--r-- | storage/innobase/buf/buf0buf.cc | 1 | ||||
| -rw-r--r-- | storage/innobase/fil/fil0crypt.cc | 3 | ||||
| -rw-r--r-- | storage/innobase/fil/fil0fil.cc | 49 | ||||
| -rw-r--r-- | storage/innobase/fil/fil0pagecompress.cc | 70 | ||||
| -rw-r--r-- | storage/innobase/include/fsp0pagecompress.ic | 9 | ||||
| -rw-r--r-- | storage/xtradb/buf/buf0buf.cc | 1 | ||||
| -rw-r--r-- | storage/xtradb/fil/fil0crypt.cc | 3 | ||||
| -rw-r--r-- | storage/xtradb/fil/fil0fil.cc | 49 | ||||
| -rw-r--r-- | storage/xtradb/fil/fil0pagecompress.cc | 72 | ||||
| -rw-r--r-- | storage/xtradb/include/fsp0pagecompress.ic | 9 |
12 files changed, 519 insertions, 58 deletions
diff --git a/mysql-test/suite/encryption/r/innodb-discard-import.result b/mysql-test/suite/encryption/r/innodb-discard-import.result new file mode 100644 index 00000000000..195b82f7488 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb-discard-import.result @@ -0,0 +1,144 @@ +call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded"); +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +SET GLOBAL innodb_compression_algorithm = 1; +create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=4; +show warnings; +Level Code Message +create table t2(c1 bigint not null, b char(200)) engine=innodb page_compressed=1 encrypted=yes encryption_key_id=4; +show warnings; +Level Code Message +create table t3(c1 bigint not null, b char(200)) engine=innodb row_format=compressed encrypted=yes encryption_key_id=4; +show warnings; +Level Code Message +create table t4(c1 bigint not null, b char(200)) engine=innodb page_compressed=1; +show warnings; +Level Code Message +create procedure innodb_insert_proc (repeat_count int) +begin +declare current_num int; +set current_num = 0; +while current_num < repeat_count do +insert into t1 values(current_num, repeat('foobar',30)); +insert into t2 values(current_num, repeat('barfoo',30)); +insert into t3 values(current_num, repeat('tmpres',30)); +insert into t4 values(current_num, repeat('mysql',30)); +set current_num = current_num + 1; +end while; +end// +commit; +set autocommit=0; +call innodb_insert_proc(2000); +commit; +set autocommit=1; +select count(*) from t1; +count(*) +2000 +select count(*) from t2; +count(*) +2000 +select count(*) from t3; +count(*) +2000 +select count(*) from t4; +count(*) +2000 +FLUSH TABLE t1,t2,t3,t4 FOR EXPORT; +# List before copying files +t1.cfg +t1.frm +t1.ibd +t2.cfg +t2.frm +t2.ibd +t3.cfg +t3.frm +t3.ibd +t4.cfg +t4.frm +t4.ibd +UNLOCK TABLES; +# tables should be either encrypted and/or compressed +# t1 yes on expecting NOT FOUND +NOT FOUND /foobar/ in t1.ibd +# t2 yes on expecting NOT FOUND +NOT FOUND /barfoo/ in t2.ibd +# t3 yes on expecting NOT FOUND +NOT FOUND /tmpres/ in t3.ibd +# t4 yes on expecting NOT FOUND +NOT FOUND /mysql/ in t4.ibd +ALTER TABLE t1 DISCARD TABLESPACE; +ALTER TABLE t2 DISCARD TABLESPACE; +ALTER TABLE t3 DISCARD TABLESPACE; +ALTER TABLE t4 DISCARD TABLESPACE; +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +SET GLOBAL innodb_compression_algorithm = 1; +# List after t1 DISCARD +t1.frm +t2.frm +t3.frm +t4.frm +ALTER TABLE t1 IMPORT TABLESPACE; +Warnings: +Warning 1814 Tablespace has been discarded for table 't1' +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c1` bigint(20) NOT NULL, + `b` char(200) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 `encrypted`=yes `encryption_key_id`=4 +SELECT COUNT(*) FROM t1; +COUNT(*) +2000 +ALTER TABLE t2 IMPORT TABLESPACE; +Warnings: +Warning 1814 Tablespace has been discarded for table 't2' +SHOW CREATE TABLE t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `c1` bigint(20) NOT NULL, + `b` char(200) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 `page_compressed`=1 `encrypted`=yes `encryption_key_id`=4 +SELECT COUNT(*) FROM t2; +COUNT(*) +2000 +ALTER TABLE t3 IMPORT TABLESPACE; +Warnings: +Warning 1814 Tablespace has been discarded for table 't3' +SHOW CREATE TABLE t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `c1` bigint(20) NOT NULL, + `b` char(200) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 ROW_FORMAT=COMPRESSED `encrypted`=yes `encryption_key_id`=4 +SELECT COUNT(*) FROM t3; +COUNT(*) +2000 +ALTER TABLE t4 IMPORT TABLESPACE; +Warnings: +Warning 1814 Tablespace has been discarded for table 't4' +SHOW CREATE TABLE t4; +Table Create Table +t4 CREATE TABLE `t4` ( + `c1` bigint(20) NOT NULL, + `b` char(200) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 `page_compressed`=1 +SELECT COUNT(*) FROM t4; +COUNT(*) +2000 +flush data to disk +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +SET GLOBAL innodb_compression_algorithm = 1; +# tables should be still either encrypted and/or compressed +# t1 yes on expecting NOT FOUND +NOT FOUND /foobar/ in t1.ibd +# t2 yes on expecting NOT FOUND +NOT FOUND /barfoo/ in t2.ibd +# t3 yes on expecting NOT FOUND +NOT FOUND /tmpres/ in t3.ibd +# t4 yes on expecting NOT FOUND +NOT FOUND /mysql/ in t4.ibd +DROP PROCEDURE innodb_insert_proc; +DROP TABLE t1,t2,t3,t4; diff --git a/mysql-test/suite/encryption/t/innodb-discard-import.test b/mysql-test/suite/encryption/t/innodb-discard-import.test new file mode 100644 index 00000000000..6d9f6c5dbb3 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-discard-import.test @@ -0,0 +1,167 @@ +-- source include/have_innodb.inc +-- source include/have_file_key_management_plugin.inc +# embedded does not support restart +-- source include/not_embedded.inc +-- source include/not_valgrind.inc +# Avoid CrashReporter popup on Mac +-- source include/not_crashrep.inc + +# +# MDEV-8770: Incorrect error message when importing page compressed tablespace +# + +call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded"); + +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; +let $innodb_compression_algo = `SELECT @@innodb_compression_algorithm`; +--enable_query_log + +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +SET GLOBAL innodb_compression_algorithm = 1; + +--let $MYSQLD_TMPDIR = `SELECT @@tmpdir` +--let $MYSQLD_DATADIR = `SELECT @@datadir` +--let SEARCH_RANGE = 10000000 +--let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd +--let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd +--let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd +--let t4_IBD = $MYSQLD_DATADIR/test/t4.ibd + +create table t1(c1 bigint not null, b char(200)) engine=innodb encrypted=yes encryption_key_id=4; +show warnings; +create table t2(c1 bigint not null, b char(200)) engine=innodb page_compressed=1 encrypted=yes encryption_key_id=4; +show warnings; +create table t3(c1 bigint not null, b char(200)) engine=innodb row_format=compressed encrypted=yes encryption_key_id=4; +show warnings; +create table t4(c1 bigint not null, b char(200)) engine=innodb page_compressed=1; +show warnings; + +delimiter //; +create procedure innodb_insert_proc (repeat_count int) +begin + declare current_num int; + set current_num = 0; + while current_num < repeat_count do + insert into t1 values(current_num, repeat('foobar',30)); + insert into t2 values(current_num, repeat('barfoo',30)); + insert into t3 values(current_num, repeat('tmpres',30)); + insert into t4 values(current_num, repeat('mysql',30)); + set current_num = current_num + 1; + end while; +end// +delimiter ;// +commit; + +set autocommit=0; +call innodb_insert_proc(2000); +commit; +set autocommit=1; + +select count(*) from t1; +select count(*) from t2; +select count(*) from t3; +select count(*) from t4; + +FLUSH TABLE t1,t2,t3,t4 FOR EXPORT; +--echo # List before copying files +--list_files $MYSQLD_DATADIR/test +--copy_file $MYSQLD_DATADIR/test/t1.cfg $MYSQLD_TMPDIR/t1.cfg +--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_TMPDIR/t1.ibd +--copy_file $MYSQLD_DATADIR/test/t2.cfg $MYSQLD_TMPDIR/t2.cfg +--copy_file $MYSQLD_DATADIR/test/t2.ibd $MYSQLD_TMPDIR/t2.ibd +--copy_file $MYSQLD_DATADIR/test/t3.cfg $MYSQLD_TMPDIR/t3.cfg +--copy_file $MYSQLD_DATADIR/test/t3.ibd $MYSQLD_TMPDIR/t3.ibd +--copy_file $MYSQLD_DATADIR/test/t4.cfg $MYSQLD_TMPDIR/t4.cfg +--copy_file $MYSQLD_DATADIR/test/t4.ibd $MYSQLD_TMPDIR/t4.ibd +UNLOCK TABLES; + +--echo # tables should be either encrypted and/or compressed +--let SEARCH_PATTERN=foobar +--echo # t1 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t1_IBD +-- source include/search_pattern_in_file.inc +--let SEARCH_PATTERN=barfoo +--echo # t2 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t2_IBD +-- source include/search_pattern_in_file.inc +--let SEARCH_PATTERN=tmpres +--echo # t3 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t3_IBD +-- source include/search_pattern_in_file.inc +--let SEARCH_PATTERN=mysql +--echo # t4 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t4_IBD +-- source include/search_pattern_in_file.inc + +ALTER TABLE t1 DISCARD TABLESPACE; +ALTER TABLE t2 DISCARD TABLESPACE; +ALTER TABLE t3 DISCARD TABLESPACE; +ALTER TABLE t4 DISCARD TABLESPACE; + +--source include/restart_mysqld.inc + +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +SET GLOBAL innodb_compression_algorithm = 1; + +--echo # List after t1 DISCARD +--list_files $MYSQLD_DATADIR/test +--copy_file $MYSQLD_TMPDIR/t1.cfg $MYSQLD_DATADIR/test/t1.cfg +--copy_file $MYSQLD_TMPDIR/t1.ibd $MYSQLD_DATADIR/test/t1.ibd +--copy_file $MYSQLD_TMPDIR/t2.cfg $MYSQLD_DATADIR/test/t2.cfg +--copy_file $MYSQLD_TMPDIR/t2.ibd $MYSQLD_DATADIR/test/t2.ibd +--copy_file $MYSQLD_TMPDIR/t3.cfg $MYSQLD_DATADIR/test/t3.cfg +--copy_file $MYSQLD_TMPDIR/t3.ibd $MYSQLD_DATADIR/test/t3.ibd +--copy_file $MYSQLD_TMPDIR/t4.cfg $MYSQLD_DATADIR/test/t4.cfg +--copy_file $MYSQLD_TMPDIR/t4.ibd $MYSQLD_DATADIR/test/t4.ibd + +ALTER TABLE t1 IMPORT TABLESPACE; +SHOW CREATE TABLE t1; +SELECT COUNT(*) FROM t1; +ALTER TABLE t2 IMPORT TABLESPACE; +SHOW CREATE TABLE t2; +SELECT COUNT(*) FROM t2; +ALTER TABLE t3 IMPORT TABLESPACE; +SHOW CREATE TABLE t3; +SELECT COUNT(*) FROM t3; +ALTER TABLE t4 IMPORT TABLESPACE; +SHOW CREATE TABLE t4; +SELECT COUNT(*) FROM t4; + +--echo flush data to disk +--source include/restart_mysqld.inc + +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +SET GLOBAL innodb_compression_algorithm = 1; + +--echo # tables should be still either encrypted and/or compressed +--let SEARCH_PATTERN=foobar +--echo # t1 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t1_IBD +-- source include/search_pattern_in_file.inc +--let SEARCH_PATTERN=barfoo +--echo # t2 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t2_IBD +-- source include/search_pattern_in_file.inc +--let SEARCH_PATTERN=tmpres +--echo # t3 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t3_IBD +-- source include/search_pattern_in_file.inc +--let SEARCH_PATTERN=mysql +--echo # t4 yes on expecting NOT FOUND +-- let SEARCH_FILE=$t4_IBD +-- source include/search_pattern_in_file.inc + +DROP PROCEDURE innodb_insert_proc; +DROP TABLE t1,t2,t3,t4; + +# reset system +--disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; +EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algo; +--enable_query_log diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 2cadaa18b9c..4f393d8ffec 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -5892,6 +5892,7 @@ buf_page_init_for_backup_restore( Reserve unused slot from temporary memory array and allocate necessary temporary memory if not yet allocated. @return reserved slot */ +UNIV_INTERN buf_tmp_buffer_t* buf_pool_reserve_tmp_slot( /*======================*/ diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 8df23ff0215..bb0ca88d06d 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -613,6 +613,9 @@ fil_encrypt_buf( memcpy(dst_frame + page_size - FIL_PAGE_DATA_END, src_frame + page_size - FIL_PAGE_DATA_END, FIL_PAGE_DATA_END); + } else { + /* Clean up rest of buffer */ + memset(dst_frame+header_len+srclen, 0, page_size - (header_len+srclen)); } /* handle post encryption checksum */ diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index b3dae5d24d7..32708580477 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -6444,17 +6444,24 @@ fil_iterate( bool decrypted = false; for (ulint i = 0; i < n_pages_read; ++i) { - ulint size = iter.page_size; + ulint size = iter.page_size; dberr_t err = DB_SUCCESS; + byte* src = (readptr + (i * size)); + byte* dst = (io_buffer + (i * size)); + + ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); + + bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED || + page_type == FIL_PAGE_PAGE_COMPRESSED); /* If tablespace is encrypted, we need to decrypt the page. */ if (encrypted) { decrypted = fil_space_decrypt( iter.crypt_data, - io_buffer + i * size, //dst + dst, //dst iter.page_size, - readptr + i * size, // src + src, // src &err); // src if (err != DB_SUCCESS) { @@ -6465,12 +6472,18 @@ fil_iterate( updated = true; } else { /* TODO: remove unnecessary memcpy's */ - memcpy(io_buffer + i * size, readptr + i * size, size); + memcpy(dst, src, size); } } - buf_block_set_file_page(block, space_id, page_no++); + /* If the original page is page_compressed, we need + to decompress page before we can update it. */ + if (page_compressed) { + fil_decompress_page(NULL, dst, size, NULL); + updated = true; + } + buf_block_set_file_page(block, space_id, page_no++); if ((err = callback(page_off, block)) != DB_SUCCESS) { @@ -6484,12 +6497,28 @@ fil_iterate( buf_block_set_state(block, BUF_BLOCK_NOT_USED); buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); + src = (io_buffer + (i * size)); + + if (page_compressed) { + ulint len = 0; + byte* res = fil_compress_page(space_id, + src, + NULL, + size, + fil_space_get_page_compression_level(space_id), + fil_space_get_block_size(space_id, offset, size), + encrypted, + &len, + NULL); + + updated = true; + } + /* If tablespace is encrypted, encrypt page before we write it back. Note that we should not encrypt the buffer that is in buffer pool. */ if (decrypted && encrypted) { - unsigned char *src = io_buffer + (i * size); - unsigned char *dst = writeptr + (i * size); + unsigned char *dest = (writeptr + (i * size)); ulint space = mach_read_from_4( src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET); @@ -6502,12 +6531,14 @@ fil_iterate( lsn, src, iter.page_size == UNIV_PAGE_SIZE ? 0 : iter.page_size, - dst); + dest); if (tmp == src) { /* TODO: remove unnecessary memcpy's */ - memcpy(writeptr, src, size); + memcpy(dest, src, size); } + + updated = true; } page_off += iter.page_size; diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index ff8541a40a5..52a63c4c32f 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -111,11 +111,23 @@ fil_compress_page( /* Cache to avoid change during function execution */ ulint comp_method = innodb_compression_algorithm; ulint orig_page_type; + bool allocated=false; if (encrypted) { header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE; } + if (!out_buf) { + allocated = true; + out_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE)); +#ifdef HAVE_LZO + if (comp_method == PAGE_LZO_ALGORITHM) { + lzo_mem = static_cast<byte *>(ut_malloc(LZO1X_1_15_MEM_COMPRESS)); + memset(lzo_mem, 0, LZO1X_1_15_MEM_COMPRESS); + } +#endif + } + ut_ad(buf); ut_ad(out_buf); ut_ad(len); @@ -124,6 +136,10 @@ fil_compress_page( /* read original page type */ orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); + fil_system_enter(); + fil_space_t* space = fil_space_get_by_id(space_id); + fil_system_exit(); + /* Let's not compress file space header or extent descriptor */ if (orig_page_type == 0 || @@ -131,12 +147,9 @@ fil_compress_page( orig_page_type == FIL_PAGE_TYPE_XDES || orig_page_type == FIL_PAGE_PAGE_COMPRESSED) { *out_len = len; - return (buf); - } - fil_system_enter(); - fil_space_t* space = fil_space_get_by_id(space_id); - fil_system_exit(); + goto err_exit; + } /* If no compression level was provided to this table, use system default level */ @@ -174,7 +187,7 @@ fil_compress_page( #endif srv_stats.pages_page_compression_error.inc(); *out_len = len; - return (buf); + goto err_exit; } break; #endif /* HAVE_LZ4 */ @@ -190,9 +203,10 @@ fil_compress_page( space_id, fil_space_name(space), len, err, write_size); space->printed_compression_failure = true; } + srv_stats.pages_page_compression_error.inc(); *out_len = len; - return (buf); + goto err_exit; } break; @@ -218,9 +232,10 @@ fil_compress_page( space_id, fil_space_name(space), len, err, out_pos); space->printed_compression_failure = true; } + srv_stats.pages_page_compression_error.inc(); *out_len = len; - return (buf); + goto err_exit; } write_size = out_pos; @@ -248,9 +263,10 @@ fil_compress_page( space_id, fil_space_name(space), len, err, write_size); space->printed_compression_failure = true; } + srv_stats.pages_page_compression_error.inc(); *out_len = len; - return (buf); + goto err_exit; } break; } @@ -270,9 +286,10 @@ fil_compress_page( space_id, fil_space_name(space), len, (int)cstatus, write_size); space->printed_compression_failure = true; } + srv_stats.pages_page_compression_error.inc(); *out_len = len; - return (buf); + goto err_exit; } break; } @@ -293,7 +310,7 @@ fil_compress_page( srv_stats.pages_page_compression_error.inc(); *out_len = len; - return (buf); + goto err_exit; } break; @@ -339,15 +356,17 @@ fil_compress_page( byte *comp_page; byte *uncomp_page; - comp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*3)); - uncomp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*3)); + comp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE)); + uncomp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE)); memcpy(comp_page, out_buf, UNIV_PAGE_SIZE); fil_decompress_page(uncomp_page, comp_page, len, NULL); + if(buf_page_is_corrupted(false, uncomp_page, 0)) { buf_page_print(uncomp_page, 0, BUF_PAGE_PRINT_NO_CRASH); ut_error; } + ut_free(comp_page); ut_free(uncomp_page); } @@ -366,6 +385,8 @@ fil_compress_page( size_t tmp = write_size; write_size = (size_t)ut_uint64_align_up((ib_uint64_t)write_size, block_size); + /* Clean up the end of buffer */ + memset(out_buf+tmp, 0, write_size - tmp); #ifdef UNIV_DEBUG ut_a(write_size > 0 && ((write_size % block_size) == 0)); ut_a(write_size >= tmp); @@ -384,12 +405,30 @@ fil_compress_page( /* If we do not persistently trim rest of page, we need to write it all */ if (!srv_use_trim) { + memset(out_buf+write_size,0,len-write_size); write_size = len; } *out_len = write_size; - return(out_buf); + if (allocated) { + /* TODO: reduce number of memcpy's */ + memcpy(buf, out_buf, len); + } else { + return(out_buf); + } + +err_exit: + if (allocated) { + ut_free(out_buf); +#ifdef HAVE_LZO + if (comp_method == PAGE_LZO_ALGORITHM) { + ut_free(lzo_mem); + } +#endif + } + + return (buf); } @@ -432,7 +471,8 @@ fil_decompress_page( // If no buffer was given, we need to allocate temporal buffer if (page_buf == NULL) { - in_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*3)); + in_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE)); + memset(in_buf, 0, UNIV_PAGE_SIZE); } else { in_buf = page_buf; } diff --git a/storage/innobase/include/fsp0pagecompress.ic b/storage/innobase/include/fsp0pagecompress.ic index 03c0ddaee28..e879aa2c16e 100644 --- a/storage/innobase/include/fsp0pagecompress.ic +++ b/storage/innobase/include/fsp0pagecompress.ic @@ -21,7 +21,8 @@ this program; if not, write to the Free Software Foundation, Inc., Implementation for helper functions for extracting/storing page compression and atomic writes information to file space. -Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com +Created 11/12/2013 Jan Lindström jan.lindstrom@mariadb.com + ***********************************************************************/ /********************************************************************//** @@ -100,7 +101,7 @@ fil_page_is_compressed_encrypted( /*******************************************************************//** Returns the page compression level of the space, or 0 if the space is not compressed. The tablespace must be cached in the memory cache. -@return page compression level, ULINT_UNDEFINED if space not found */ +@return page compression level, 0 if space not found */ UNIV_INLINE ulint fil_space_get_page_compression_level( @@ -116,7 +117,7 @@ fil_space_get_page_compression_level( return(fsp_flags_get_page_compression_level(flags)); } - return(flags); + return(0); } /*******************************************************************//** @@ -138,7 +139,7 @@ fil_space_is_page_compressed( return(fsp_flags_is_page_compressed(flags)); } - return(flags); + return(0); } #endif /* UNIV_INNOCHECKSUM */ diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 85023be9402..1d348b0d108 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -6070,6 +6070,7 @@ buf_pool_mutex_exit( Reserve unused slot from temporary memory array and allocate necessary temporary memory if not yet allocated. @return reserved slot */ +UNIV_INTERN buf_tmp_buffer_t* buf_pool_reserve_tmp_slot( /*======================*/ diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 8df23ff0215..bb0ca88d06d 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -613,6 +613,9 @@ fil_encrypt_buf( memcpy(dst_frame + page_size - FIL_PAGE_DATA_END, src_frame + page_size - FIL_PAGE_DATA_END, FIL_PAGE_DATA_END); + } else { + /* Clean up rest of buffer */ + memset(dst_frame+header_len+srclen, 0, page_size - (header_len+srclen)); } /* handle post encryption checksum */ diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index dde309f4234..3e5cf4ec288 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -6501,17 +6501,24 @@ fil_iterate( bool decrypted = false; for (ulint i = 0; i < n_pages_read; ++i) { - ulint size = iter.page_size; + ulint size = iter.page_size; dberr_t err = DB_SUCCESS; + byte* src = (readptr + (i * size)); + byte* dst = (io_buffer + (i * size)); + + ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); + + bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED || + page_type == FIL_PAGE_PAGE_COMPRESSED); /* If tablespace is encrypted, we need to decrypt the page. */ if (encrypted) { decrypted = fil_space_decrypt( iter.crypt_data, - io_buffer + i * size, //dst + dst, //dst iter.page_size, - readptr + i * size, // src + src, // src &err); // src if (err != DB_SUCCESS) { @@ -6522,12 +6529,18 @@ fil_iterate( updated = true; } else { /* TODO: remove unnecessary memcpy's */ - memcpy(io_buffer + i * size, readptr + i * size, size); + memcpy(dst, src, size); } } - buf_block_set_file_page(block, space_id, page_no++); + /* If the original page is page_compressed, we need + to decompress page before we can update it. */ + if (page_compressed) { + fil_decompress_page(NULL, dst, size, NULL); + updated = true; + } + buf_block_set_file_page(block, space_id, page_no++); if ((err = callback(page_off, block)) != DB_SUCCESS) { @@ -6541,12 +6554,28 @@ fil_iterate( buf_block_set_state(block, BUF_BLOCK_NOT_USED); buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); + src = (io_buffer + (i * size)); + + if (page_compressed) { + ulint len = 0; + byte* res = fil_compress_page(space_id, + src, + NULL, + size, + fil_space_get_page_compression_level(space_id), + fil_space_get_block_size(space_id, offset, size), + encrypted, + &len, + NULL); + + updated = true; + } + /* If tablespace is encrypted, encrypt page before we write it back. Note that we should not encrypt the buffer that is in buffer pool. */ if (decrypted && encrypted) { - unsigned char *src = io_buffer + (i * size); - unsigned char *dst = writeptr + (i * size); + unsigned char *dest = (writeptr + (i * size)); ulint space = mach_read_from_4( src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET); @@ -6559,12 +6588,14 @@ fil_iterate( lsn, src, iter.page_size == UNIV_PAGE_SIZE ? 0 : iter.page_size, - dst); + dest); if (tmp == src) { /* TODO: remove unnecessary memcpy's */ - memcpy(writeptr, src, size); + memcpy(dest, src, size); } + + updated = true; } page_off += iter.page_size; diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc index e544981fa2e..1cfbe2ff48c 100644 --- a/storage/xtradb/fil/fil0pagecompress.cc +++ b/storage/xtradb/fil/fil0pagecompress.cc @@ -111,11 +111,23 @@ fil_compress_page( /* Cache to avoid change during function execution */ ulint comp_method = innodb_compression_algorithm; ulint orig_page_type; + bool allocated=false; if (encrypted) { header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE; } + if (!out_buf) { + allocated = true; + out_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE)); +#ifdef HAVE_LZO + if (comp_method == PAGE_LZO_ALGORITHM) { + lzo_mem = static_cast<byte *>(ut_malloc(LZO1X_1_15_MEM_COMPRESS)); + memset(lzo_mem, 0, LZO1X_1_15_MEM_COMPRESS); + } +#endif + } + ut_ad(buf); ut_ad(out_buf); ut_ad(len); @@ -124,6 +136,10 @@ fil_compress_page( /* read original page type */ orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); + fil_system_enter(); + fil_space_t* space = fil_space_get_by_id(space_id); + fil_system_exit(); + /* Let's not compress file space header or extent descriptor */ if (orig_page_type == 0 || @@ -131,12 +147,9 @@ fil_compress_page( orig_page_type == FIL_PAGE_TYPE_XDES || orig_page_type == FIL_PAGE_PAGE_COMPRESSED) { *out_len = len; - return (buf); - } - fil_system_enter(); - fil_space_t* space = fil_space_get_by_id(space_id); - fil_system_exit(); + goto err_exit; + } /* If no compression level was provided to this table, use system default level */ @@ -174,7 +187,7 @@ fil_compress_page( #endif srv_stats.pages_page_compression_error.inc(); *out_len = len; - return (buf); + goto err_exit; } break; #endif /* HAVE_LZ4 */ @@ -190,9 +203,10 @@ fil_compress_page( space_id, fil_space_name(space), len, err, write_size); space->printed_compression_failure = true; } + srv_stats.pages_page_compression_error.inc(); *out_len = len; - return (buf); + goto err_exit; } break; @@ -218,9 +232,10 @@ fil_compress_page( space_id, fil_space_name(space), len, err, out_pos); space->printed_compression_failure = true; } + srv_stats.pages_page_compression_error.inc(); *out_len = len; - return (buf); + goto err_exit; } write_size = out_pos; @@ -248,9 +263,10 @@ fil_compress_page( space_id, fil_space_name(space), len, err, write_size); space->printed_compression_failure = true; } + srv_stats.pages_page_compression_error.inc(); *out_len = len; - return (buf); + goto err_exit; } break; } @@ -270,9 +286,10 @@ fil_compress_page( space_id, fil_space_name(space), len, (int)cstatus, write_size); space->printed_compression_failure = true; } + srv_stats.pages_page_compression_error.inc(); *out_len = len; - return (buf); + goto err_exit; } break; } @@ -293,7 +310,7 @@ fil_compress_page( srv_stats.pages_page_compression_error.inc(); *out_len = len; - return (buf); + goto err_exit; } break; @@ -339,15 +356,17 @@ fil_compress_page( byte *comp_page; byte *uncomp_page; - comp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*3)); - uncomp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*3)); + comp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE)); + uncomp_page = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE)); memcpy(comp_page, out_buf, UNIV_PAGE_SIZE); fil_decompress_page(uncomp_page, comp_page, len, NULL); + if(buf_page_is_corrupted(false, uncomp_page, 0)) { buf_page_print(uncomp_page, 0, BUF_PAGE_PRINT_NO_CRASH); ut_error; } + ut_free(comp_page); ut_free(uncomp_page); } @@ -363,10 +382,10 @@ fil_compress_page( /* Actual write needs to be alligned on block size */ if (write_size % block_size) { -#ifdef UNIV_DEBUG size_t tmp = write_size; -#endif write_size = (size_t)ut_uint64_align_up((ib_uint64_t)write_size, block_size); + /* Clean up the end of buffer */ + memset(out_buf+tmp, 0, write_size - tmp); #ifdef UNIV_DEBUG ut_a(write_size > 0 && ((write_size % block_size) == 0)); ut_a(write_size >= tmp); @@ -385,12 +404,30 @@ fil_compress_page( /* If we do not persistently trim rest of page, we need to write it all */ if (!srv_use_trim) { + memset(out_buf+write_size,0,len-write_size); write_size = len; } *out_len = write_size; - return(out_buf); + if (allocated) { + /* TODO: reduce number of memcpy's */ + memcpy(buf, out_buf, len); + } else { + return(out_buf); + } + +err_exit: + if (allocated) { + ut_free(out_buf); +#ifdef HAVE_LZO + if (comp_method == PAGE_LZO_ALGORITHM) { + ut_free(lzo_mem); + } +#endif + } + + return (buf); } @@ -433,7 +470,8 @@ fil_decompress_page( // If no buffer was given, we need to allocate temporal buffer if (page_buf == NULL) { - in_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE*3)); + in_buf = static_cast<byte *>(ut_malloc(UNIV_PAGE_SIZE)); + memset(in_buf, 0, UNIV_PAGE_SIZE); } else { in_buf = page_buf; } diff --git a/storage/xtradb/include/fsp0pagecompress.ic b/storage/xtradb/include/fsp0pagecompress.ic index 03c0ddaee28..e879aa2c16e 100644 --- a/storage/xtradb/include/fsp0pagecompress.ic +++ b/storage/xtradb/include/fsp0pagecompress.ic @@ -21,7 +21,8 @@ this program; if not, write to the Free Software Foundation, Inc., Implementation for helper functions for extracting/storing page compression and atomic writes information to file space. -Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com +Created 11/12/2013 Jan Lindström jan.lindstrom@mariadb.com + ***********************************************************************/ /********************************************************************//** @@ -100,7 +101,7 @@ fil_page_is_compressed_encrypted( /*******************************************************************//** Returns the page compression level of the space, or 0 if the space is not compressed. The tablespace must be cached in the memory cache. -@return page compression level, ULINT_UNDEFINED if space not found */ +@return page compression level, 0 if space not found */ UNIV_INLINE ulint fil_space_get_page_compression_level( @@ -116,7 +117,7 @@ fil_space_get_page_compression_level( return(fsp_flags_get_page_compression_level(flags)); } - return(flags); + return(0); } /*******************************************************************//** @@ -138,7 +139,7 @@ fil_space_is_page_compressed( return(fsp_flags_is_page_compressed(flags)); } - return(flags); + return(0); } #endif /* UNIV_INNOCHECKSUM */ |