diff options
Diffstat (limited to 'storage')
28 files changed, 425 insertions, 249 deletions
diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index c0859bf82b6..59cbe633dec 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1006,6 +1006,7 @@ buf_block_init( block->page.comp_buf = NULL; block->page.comp_buf_free = NULL; block->page.key_version = 0; + block->page.encrypt_later = false; block->modify_clock = 0; @@ -3498,6 +3499,7 @@ buf_page_init_low( bpage->comp_buf = NULL; bpage->comp_buf_free = NULL; bpage->key_version = 0; + bpage->encrypt_later = false; HASH_INVALIDATE(bpage, hash); #if defined UNIV_DEBUG_FILE_ACCESSES || defined UNIV_DEBUG @@ -5609,6 +5611,8 @@ buf_page_encrypt_before_write( buf_page_t* bpage, /*!< in/out: buffer page to be flushed */ const byte* src_frame) /*!< in: src frame */ { + bpage->encrypt_later = false; + if (srv_encrypt_tables == FALSE) { /* Encryption is disabled */ return const_cast<byte*>(src_frame); @@ -5669,7 +5673,8 @@ buf_page_encrypt_before_write( ut_ad(key_version == 0 || key_version >= bpage->key_version); bpage->key_version = key_version; } else { - // We do compression and encryption later on os0file.cc + /** Compression and encryption is done later at os0file.cc */ + bpage->encrypt_later = true; dst_frame = (byte *)src_frame; } diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 37b8152f1fa..477bcfe0ee3 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -521,10 +521,18 @@ buf_dblwr_process() ulint zip_size = fil_space_get_zip_size(space_id); /* Read in the actual page from the file */ - fil_io(OS_FILE_READ, true, space_id, zip_size, - page_no, 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - read_buf, NULL, 0, 0); + fil_io(OS_FILE_READ, + true, + space_id, + zip_size, + page_no, + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + read_buf, + NULL, + 0, + 0, + false); if (fil_space_verify_crypt_checksum(read_buf, zip_size)) { /* page is encrypted and checksum is OK */ @@ -576,10 +584,18 @@ buf_dblwr_process() doublewrite buffer to the intended position */ - fil_io(OS_FILE_WRITE, true, space_id, - zip_size, page_no, 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - page, NULL, 0, 0); + fil_io(OS_FILE_WRITE, + true, + space_id, + zip_size, + page_no, + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + page, + NULL, + 0, + 0, + false); ib_logf(IB_LOG_LEVEL_INFO, "Recovered the page from" @@ -595,11 +611,18 @@ buf_dblwr_process() zeroes, while a valid copy is available in dblwr buffer. */ - fil_io(OS_FILE_WRITE, true, space_id, - zip_size, page_no, 0, - zip_size ? zip_size - : UNIV_PAGE_SIZE, - page, NULL, 0, 0); + fil_io(OS_FILE_WRITE, + true, + space_id, + zip_size, + page_no, + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + page, + NULL, + 0, + 0, + false); } } } @@ -621,9 +644,9 @@ buf_dblwr_process() memset(buf, 0, bytes); fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block1, 0, bytes, buf, NULL, NULL, 0); + buf_dblwr->block1, 0, bytes, buf, NULL, NULL, 0, false); fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block2, 0, bytes, buf, NULL, NULL, 0); + buf_dblwr->block2, 0, bytes, buf, NULL, NULL, 0, false); ut_free(unaligned_buf); } @@ -828,12 +851,18 @@ buf_dblwr_write_block_to_datafile( void * frame = buf_page_get_frame(bpage); if (bpage->zip.data) { - fil_io(flags, sync, buf_page_get_space(bpage), - buf_page_get_zip_size(bpage), - buf_page_get_page_no(bpage), 0, - buf_page_get_zip_size(bpage), - frame, - (void*) bpage, 0, bpage->newest_modification); + fil_io(flags, + sync, + buf_page_get_space(bpage), + buf_page_get_zip_size(bpage), + buf_page_get_page_no(bpage), + 0, + buf_page_get_zip_size(bpage), + frame, + (void*) bpage, + 0, + bpage->newest_modification, + bpage->encrypt_later); return; } @@ -843,12 +872,18 @@ buf_dblwr_write_block_to_datafile( ut_a(buf_block_get_state(block) == BUF_BLOCK_FILE_PAGE); buf_dblwr_check_page_lsn(block->frame); - fprintf(stderr, "JAN: space %lu page_type %lu\n", buf_block_get_space(block), - mach_read_from_2((byte *)frame+FIL_PAGE_TYPE)); - - fil_io(flags, sync, buf_block_get_space(block), 0, - buf_block_get_page_no(block), 0, UNIV_PAGE_SIZE, - frame, (void*) block, (ulint *)&bpage->write_size, bpage->newest_modification ); + fil_io(flags, + sync, + buf_block_get_space(block), + 0, + buf_block_get_page_no(block), + 0, + UNIV_PAGE_SIZE, + frame, + (void*) block, + (ulint *)&bpage->write_size, + bpage->newest_modification, + bpage->encrypt_later); } /********************************************************************//** @@ -942,7 +977,7 @@ try_again: fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, buf_dblwr->block1, 0, len, - (void*) write_buf, NULL, 0, 0); + (void*) write_buf, NULL, 0, 0, false); if (buf_dblwr->first_free <= TRX_SYS_DOUBLEWRITE_BLOCK_SIZE) { /* No unwritten pages in the second block. */ @@ -958,7 +993,7 @@ try_again: fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, buf_dblwr->block2, 0, len, - (void*) write_buf, NULL, 0, 0); + (void*) write_buf, NULL, 0, 0, false); flush: /* increment the doublewrite flushed pages counter */ @@ -1187,17 +1222,31 @@ retry: memset(buf_dblwr->write_buf + UNIV_PAGE_SIZE * i + zip_size, 0, UNIV_PAGE_SIZE - zip_size); - fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - offset, 0, UNIV_PAGE_SIZE, - (void*) (buf_dblwr->write_buf - + UNIV_PAGE_SIZE * i), NULL, 0, bpage->newest_modification); + fil_io(OS_FILE_WRITE, + true, + TRX_SYS_SPACE, 0, + offset, + 0, + UNIV_PAGE_SIZE, + (void*) (buf_dblwr->write_buf + UNIV_PAGE_SIZE * i), + NULL, + 0, + bpage->newest_modification, + bpage->encrypt_later); } else { /* It is a regular page. Write it directly to the doublewrite buffer */ - fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - offset, 0, UNIV_PAGE_SIZE, - frame, - NULL, 0, bpage->newest_modification); + fil_io(OS_FILE_WRITE, + true, + TRX_SYS_SPACE, 0, + offset, + 0, + UNIV_PAGE_SIZE, + frame, + NULL, + 0, + bpage->newest_modification, + bpage->encrypt_later); } /* Now flush the doublewrite buffer data to disk */ diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 9c11ae2b43e..9bd7a7c007d 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -913,16 +913,17 @@ buf_flush_write_block_low( if (!srv_use_doublewrite_buf || !buf_dblwr) { fil_io(OS_FILE_WRITE | OS_AIO_SIMULATED_WAKE_LATER, - sync, - buf_page_get_space(bpage), - zip_size, - buf_page_get_page_no(bpage), - 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - frame, - bpage, - &bpage->write_size, - bpage->newest_modification); + sync, + buf_page_get_space(bpage), + zip_size, + buf_page_get_page_no(bpage), + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + frame, + bpage, + &bpage->write_size, + bpage->newest_modification, + bpage->encrypt_later); } else { /* InnoDB uses doublewrite buffer and doublewrite buffer @@ -943,7 +944,8 @@ buf_flush_write_block_low( frame, bpage, &bpage->write_size, - bpage->newest_modification); + bpage->newest_modification, + bpage->encrypt_later); } else if (flush_type == BUF_FLUSH_SINGLE_PAGE) { buf_dblwr_write_single_page(bpage, sync); } else { diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 19d18dcd870..e91a5da6621 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -184,17 +184,17 @@ buf_read_page_low( if (zip_size) { *err = fil_io(OS_FILE_READ | wake_later - | ignore_nonexistent_pages, - sync, space, zip_size, offset, 0, zip_size, - frame, bpage, &bpage->write_size, 0); + | ignore_nonexistent_pages, + sync, space, zip_size, offset, 0, zip_size, + frame, bpage, &bpage->write_size, 0, false); } else { ut_a(buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); *err = fil_io(OS_FILE_READ | wake_later - | ignore_nonexistent_pages, - sync, space, 0, offset, 0, UNIV_PAGE_SIZE, - frame, bpage, - &bpage->write_size, 0); + | ignore_nonexistent_pages, + sync, space, 0, offset, 0, UNIV_PAGE_SIZE, + frame, bpage, + &bpage->write_size, 0, false); } if (sync) { diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 9d4c66d4343..d7b26eed49f 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -621,13 +621,25 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, byte key[MY_AES_MAX_KEY_LENGTH]; uint key_length; + ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); + if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR + || orig_page_type==FIL_PAGE_TYPE_XDES + || orig_page_type== FIL_PAGE_PAGE_ENCRYPTED + || orig_page_type== FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + //TODO: is this really needed ? + memcpy(dst_frame, src_frame, page_size); + return; + } + if (srv_encrypt_tables) { crypt_data = fil_space_get_crypt_data(space); + if (crypt_data == NULL) { //TODO: Is this really needed ? memcpy(dst_frame, src_frame, page_size); return; } + fil_crypt_get_latest_key(key, &key_length, crypt_data, &key_version); } else { key_version = encryption_key; @@ -646,9 +658,7 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, src_frame + FIL_PAGE_OFFSET); mach_write_to_4(iv + 4, space_offset); mach_write_to_8(iv + 8, lsn); - } - else - { + } else { // take the iv from the key provider int load_iv_rc = get_encryption_iv(key_version, (uchar *) iv, sizeof(iv)); @@ -666,21 +676,10 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, } } - ibool page_compressed = (mach_read_from_2(src_frame+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED); ibool page_encrypted = fil_space_is_page_encrypted(space); - ulint compression_alg = mach_read_from_8(src_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); - if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR - || orig_page_type==FIL_PAGE_TYPE_XDES - || orig_page_type== FIL_PAGE_PAGE_ENCRYPTED - || orig_page_type== FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - memcpy(dst_frame, src_frame, page_size); - return; - } - // copy page header memcpy(dst_frame, src_frame, FIL_PAGE_DATA); @@ -711,7 +710,6 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, srclen = page_size - FIL_PAGE_DATA;; } - int rc = (* my_aes_encrypt_dynamic)(src, srclen, dst, &dstlen, (unsigned char*)key, key_length, @@ -790,6 +788,7 @@ fil_space_check_encryption_read( ulint space) /*!< in: tablespace id */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); + if (crypt_data == NULL) return false; @@ -816,6 +815,7 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, || page_type == FIL_PAGE_PAGE_COMPRESSED); ulint orig_page_type=0; + if (page_type == FIL_PAGE_PAGE_ENCRYPTED) { key_version = mach_read_from_2( src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); @@ -847,7 +847,6 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, mach_write_to_2(dst_frame+FIL_PAGE_TYPE, orig_page_type); } - // get key byte key[MY_AES_MAX_KEY_LENGTH]; uint key_length; @@ -863,9 +862,7 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, mach_write_to_4(iv + 0, space); mach_write_to_4(iv + 4, offset); mach_write_to_8(iv + 8, lsn); - } - else - { + } else { // take the iv from the key provider int load_iv_rc = get_encryption_iv(key_version, (uchar *) iv, sizeof(iv)); @@ -887,7 +884,6 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, byte* dst = dst_frame + FIL_PAGE_DATA; uint32 dstlen; ulint srclen = page_size - (FIL_PAGE_DATA + FIL_PAGE_DATA_END); - ulint compressed_len; ulint compression_method; diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 7eb585609d2..7c46bdbca6b 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -284,7 +284,7 @@ fil_read( actual page size does not decrease. */ { return(fil_io(OS_FILE_READ, sync, space_id, zip_size, block_offset, - byte_offset, len, buf, message, write_size, 0)); + byte_offset, len, buf, message, write_size, 0, false)); } /********************************************************************//** @@ -316,12 +316,13 @@ fil_write( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn) /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later) /*!< in: encrypt later ? */ { ut_ad(!srv_read_only_mode); return(fil_io(OS_FILE_WRITE, sync, space_id, zip_size, block_offset, - byte_offset, len, buf, message, write_size, lsn)); + byte_offset, len, buf, message, write_size, lsn, encrypt_later)); } /*******************************************************************//** @@ -614,6 +615,8 @@ fil_node_open_file( ut_a(size_bytes != (os_offset_t) -1); node->file_block_size = os_file_get_block_size(node->handle, node->name); + space->file_block_size = node->file_block_size; + #ifdef UNIV_HOTBACKUP if (space->id == 0) { node->size = (ulint) (size_bytes / UNIV_PAGE_SIZE); @@ -780,6 +783,7 @@ add_size: if (node->file_block_size == 0) { node->file_block_size = os_file_get_block_size(node->handle, node->name); + space->file_block_size = node->file_block_size; } ut_a(ret); @@ -1831,7 +1835,7 @@ fil_write_lsn_and_arch_no_to_file( lsn); err = fil_write(TRUE, space, 0, sum_of_sizes, 0, - UNIV_PAGE_SIZE, buf, NULL, 0, lsn); + UNIV_PAGE_SIZE, buf, NULL, 0, lsn, false); } mem_free(buf1); @@ -5151,6 +5155,7 @@ retry: /* Determine correct file block size */ if (node->file_block_size == 0) { node->file_block_size = os_file_get_block_size(node->handle, node->name); + space->file_block_size = node->file_block_size; } #ifdef HAVE_POSIX_FALLOCATE @@ -5211,7 +5216,7 @@ retry: success = os_aio(OS_FILE_WRITE, OS_AIO_SYNC, node->name, node->handle, buf, offset, page_size * n_pages, - node, NULL, 0, FALSE, 0, 0, 0, 0); + node, NULL, 0, FALSE, 0, 0, 0, 0, false); #endif /* UNIV_HOTBACKUP */ if (success) { os_has_said_disk_full = FALSE; @@ -5592,7 +5597,8 @@ fil_io( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn) /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later) /*!< in: encrypt later ? */ { ulint mode; fil_space_t* space; @@ -5820,7 +5826,8 @@ fil_io( page_compression_level, page_encrypted, page_encryption_key, - lsn); + lsn, + encrypt_later); #endif /* UNIV_HOTBACKUP */ diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index 53bf4ee42ca..670af29e7b5 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -281,11 +281,10 @@ fil_compress_page( /* read original page type */ orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); - ut_ad(orig_page_type != 0); - /* Let's not compress file space header or extent descriptor */ - if (orig_page_type == FIL_PAGE_TYPE_FSP_HDR || + if (orig_page_type == 0 || + orig_page_type == FIL_PAGE_TYPE_FSP_HDR || orig_page_type == FIL_PAGE_TYPE_XDES || orig_page_type == FIL_PAGE_PAGE_COMPRESSED || orig_page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 2863ab01ff9..0f2f9083675 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1566,6 +1566,8 @@ struct buf_page_t{ can be read while it's being flushed */ byte* comp_buf_free; /*!< for compression, allocated buffer that is then alligned */ + bool encrypt_later; /*!< should we encrypt the page + at os0file.cc ? */ #ifndef UNIV_HOTBACKUP buf_page_t* hash; /*!< node used in chaining to diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 1ed8cbf3293..bd7f39939a3 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2015, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -346,6 +346,7 @@ struct fil_space_t { UT_LIST_NODE_T(fil_space_t) space_list; /*!< list of all spaces */ fil_space_crypt_t* crypt_data; + ulint file_block_size;/*!< file system block size */ ulint magic_n;/*!< FIL_SPACE_MAGIC_N */ }; @@ -998,7 +999,8 @@ fil_io( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn) /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later) /*!< in: should we encrypt the page */ __attribute__((nonnull(8))); /**********************************************************************//** Waits for an aio operation to complete. This function is used to write the diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h index e2d0cf26682..73daea32d72 100644 --- a/storage/innobase/include/os0file.h +++ b/storage/innobase/include/os0file.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2015, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -314,12 +314,12 @@ The wrapper functions have the prefix of "innodb_". */ # define os_aio(type, mode, name, file, buf, offset, \ n, message1, message2, write_size, \ page_compression, page_compression_level, \ - page_encryption, page_encryption_key, lsn) \ + page_encryption, page_encryption_key, lsn, encrypt) \ pfs_os_aio_func(type, mode, name, file, buf, offset, \ n, message1, message2, write_size, \ page_compression, page_compression_level, \ page_encryption, page_encryption_key, \ - lsn, __FILE__, __LINE__) + lsn, encrypt, __FILE__, __LINE__) # define os_file_read(file, buf, offset, n, compressed) \ @@ -363,11 +363,11 @@ to original un-instrumented file I/O APIs */ # define os_aio(type, mode, name, file, buf, offset, n, message1, \ message2, write_size, page_compression, page_compression_level, \ - page_encryption, page_encryption_key, lsn) \ + page_encryption, page_encryption_key, lsn, encrypt) \ os_aio_func(type, mode, name, file, buf, offset, n, \ message1, message2, write_size, \ page_compression, page_compression_level, \ - page_encryption, page_encryption_key, lsn) + page_encryption, page_encryption_key, lsn, encrypt) # define os_file_read(file, buf, offset, n, compressed) \ os_file_read_func(file, buf, offset, n, compressed) @@ -787,9 +787,10 @@ pfs_os_aio_func( level to be used */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key, /*!< page encryption + ulint page_encryption_key, /*!< in: page encryption key to be used */ - lsn_t lsn, /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later, /*!< in: should we encrypt ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line);/*!< in: line where the func invoked */ /*******************************************************************//** @@ -1170,9 +1171,10 @@ os_aio_func( level to be used */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key, /*!< page encryption key + ulint page_encryption_key, /*!< in: page encryption key to be used */ - lsn_t lsn); /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later); /*!< in: should we encrypt ? */ /************************************************************************//** Wakes up all async i/o threads so that they know to exit themselves in shutdown. */ diff --git a/storage/innobase/include/os0file.ic b/storage/innobase/include/os0file.ic index bbd3826d50b..9839a841188 100644 --- a/storage/innobase/include/os0file.ic +++ b/storage/innobase/include/os0file.ic @@ -222,13 +222,15 @@ pfs_os_aio_func( actual page size does not decrease. */ ibool page_compression, /*!< in: is page compression used on this file space */ - ulint page_compression_level, /*!< page compression + ulint page_compression_level, /*!< in: page compression level to be used */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key, /*!< page encryption + ulint page_encryption_key, /*!< in: page encryption key to be used */ - lsn_t lsn, /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest + modification */ + bool encrypt_later, /*!< in: encrypt later ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line)/*!< in: line where the func invoked */ { @@ -246,7 +248,7 @@ pfs_os_aio_func( result = os_aio_func(type, mode, name, file, buf, offset, n, message1, message2, write_size, page_compression, page_compression_level, - page_encryption, page_encryption_key, lsn); + page_encryption, page_encryption_key, lsn, encrypt_later); register_pfs_file_io_end(locker, n); diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index ba05987dfbe..779ad112b05 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -1276,7 +1276,7 @@ log_group_file_header_flush( (ulint) (dest_offset / UNIV_PAGE_SIZE), (ulint) (dest_offset % UNIV_PAGE_SIZE), OS_FILE_LOG_BLOCK_SIZE, - buf, group, 0, 0); + buf, group, 0, 0, false); srv_stats.os_log_pending_writes.dec(); } @@ -1443,7 +1443,7 @@ loop: fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, group->space_id, 0, (ulint) (next_offset / UNIV_PAGE_SIZE), (ulint) (next_offset % UNIV_PAGE_SIZE), write_len, buf, - group, 0, 0); + group, 0, 0, false); srv_stats.os_log_pending_writes.dec(); @@ -2011,7 +2011,7 @@ log_group_checkpoint( write_offset / UNIV_PAGE_SIZE, write_offset % UNIV_PAGE_SIZE, OS_FILE_LOG_BLOCK_SIZE, - buf, ((byte*) group + 1), 0, 0); + buf, ((byte*) group + 1), 0, 0, false); ut_ad(((ulint) group & 0x1UL) == 0); } @@ -2093,7 +2093,7 @@ log_group_read_checkpoint_info( fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->space_id, 0, field / UNIV_PAGE_SIZE, field % UNIV_PAGE_SIZE, - OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0, 0); + OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0, 0, false); } /******************************************************//** @@ -2417,7 +2417,7 @@ loop: fil_io(OS_FILE_READ | OS_FILE_LOG, sync, group->space_id, 0, (ulint) (source_offset / UNIV_PAGE_SIZE), (ulint) (source_offset % UNIV_PAGE_SIZE), - len, buf, NULL, 0, 0); + len, buf, NULL, 0, 0, false); if (recv_sys->recv_log_crypt_ver != UNENCRYPTED_KEY_VER && !log_group_decrypt_after_read(group, buf, len)) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index fbed6137cd7..adea9a0ce27 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -3101,7 +3101,7 @@ recv_recovery_from_checkpoint_start_func( fil_io(OS_FILE_READ | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, LOG_FILE_HDR_SIZE, - log_hdr_buf, max_cp_group, 0, 0); + log_hdr_buf, max_cp_group, 0, 0, false); if (0 == ut_memcmp(log_hdr_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, (byte*)"ibbackup", (sizeof "ibbackup") - 1)) { @@ -3132,7 +3132,7 @@ recv_recovery_from_checkpoint_start_func( fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, OS_FILE_LOG_BLOCK_SIZE, - log_hdr_buf, max_cp_group, 0, 0); + log_hdr_buf, max_cp_group, 0, 0, false); } #ifdef UNIV_LOG_ARCHIVE diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 0b623448e03..7495e0d570c 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -260,6 +260,9 @@ struct os_aio_slot_t{ ulint file_block_size;/*!< file block size */ + bool encrypt_later; /*!< should the page be encrypted + before write */ + #ifdef WIN_ASYNC_IO HANDLE handle; /*!< handle object we need in the OVERLAPPED struct */ @@ -4546,9 +4549,11 @@ os_aio_array_reserve_slot( level to be used */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key, /*!< page encryption key + ulint page_encryption_key, /*!< in: page encryption key to be used */ - lsn_t lsn) /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later) /*!< in: should we encrypt before + writing the page */ { os_aio_slot_t* slot = NULL; #ifdef WIN_ASYNC_IO @@ -4635,7 +4640,6 @@ found: slot->name = name; slot->len = len; slot->type = type; - slot->buf = static_cast<byte*>(buf); slot->offset = offset; slot->lsn = lsn; slot->io_already_done = FALSE; @@ -4646,6 +4650,7 @@ found: slot->page_compression = page_compression; slot->page_encryption_key = page_encryption_key; slot->page_encryption = page_encryption; + slot->encrypt_later = encrypt_later; if (message1) { slot->file_block_size = fil_node_get_block_size(message1); @@ -4700,7 +4705,8 @@ found: /* If the space is page encryption and this is write operation then we encrypt the page */ - if (message1 && type == OS_FILE_WRITE && page_encryption ) { + if (message1 && type == OS_FILE_WRITE && (page_encryption == 1 || encrypt_later)) { + ut_a(page_encryption == 1 || srv_encrypt_tables == 1); /* Release the array mutex while encrypting */ os_mutex_exit(array->mutex); @@ -4724,6 +4730,8 @@ found: os_mutex_enter(array->mutex); } + slot->buf = static_cast<byte*>(buf); + #ifdef WIN_ASYNC_IO control = &slot->control; control->Offset = (DWORD) offset & 0xFFFFFFFF; @@ -5012,9 +5020,11 @@ os_aio_func( level to be used */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key, /*!< page encryption key + ulint page_encryption_key, /*!< in: page encryption key to be used */ - lsn_t lsn) /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later) /*!< in: should we encrypt page + before write */ { os_aio_array_t* array; os_aio_slot_t* slot; @@ -5131,7 +5141,7 @@ try_again: slot = os_aio_array_reserve_slot(type, array, message1, message2, file, name, buf, offset, n, write_size, page_compression, page_compression_level, - page_encryption, page_encryption_key, lsn); + page_encryption, page_encryption_key, lsn, encrypt_later); if (type == OS_FILE_READ) { if (srv_use_native_aio) { @@ -5169,6 +5179,7 @@ try_again: buffer = buf; } } + ret = WriteFile(file, buffer, (DWORD) n, &len, &(slot->control)); diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 37b1c65dfe1..71008455f9b 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -1084,6 +1084,7 @@ buf_block_init( block->page.comp_buf = NULL; block->page.comp_buf_free = NULL; block->page.key_version = 0; + block->page.encrypt_later = false; block->modify_clock = 0; @@ -3587,6 +3588,7 @@ buf_page_init_low( bpage->comp_buf = NULL; bpage->comp_buf_free = NULL; bpage->key_version = 0; + bpage->encrypt_later = false; HASH_INVALIDATE(bpage, hash); bpage->is_corrupt = FALSE; @@ -5793,6 +5795,8 @@ buf_page_encrypt_before_write( buf_page_t* bpage, /*!< in/out: buffer page to be flushed */ const byte* src_frame) /*!< in: src frame */ { + bpage->encrypt_later = false; + if (srv_encrypt_tables == FALSE) { /* Encryption is disabled */ return const_cast<byte*>(src_frame); @@ -5853,7 +5857,8 @@ buf_page_encrypt_before_write( ut_ad(key_version == 0 || key_version >= bpage->key_version); bpage->key_version = key_version; } else { - // We do compression and encryption later on os0file.cc + /** Compression and encryption is done later at os0file.cc */ + bpage->encrypt_later = true; dst_frame = (byte *)src_frame; } diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index 75e86ac4326..871f723549b 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -521,10 +521,18 @@ buf_dblwr_process() ulint zip_size = fil_space_get_zip_size(space_id); /* Read in the actual page from the file */ - fil_io(OS_FILE_READ, true, space_id, zip_size, - page_no, 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - read_buf, NULL, 0, 0); + fil_io(OS_FILE_READ, + true, + space_id, + zip_size, + page_no, + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + read_buf, + NULL, + 0, + 0, + false); if (fil_space_verify_crypt_checksum(read_buf, zip_size)) { /* page is encrypted and checksum is OK */ @@ -576,10 +584,18 @@ buf_dblwr_process() doublewrite buffer to the intended position */ - fil_io(OS_FILE_WRITE, true, space_id, - zip_size, page_no, 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - page, NULL, 0, 0); + fil_io(OS_FILE_WRITE, + true, + space_id, + zip_size, + page_no, + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + page, + NULL, + 0, + 0, + false); ib_logf(IB_LOG_LEVEL_INFO, "Recovered the page from" @@ -595,11 +611,17 @@ buf_dblwr_process() zeroes, while a valid copy is available in dblwr buffer. */ - fil_io(OS_FILE_WRITE, true, space_id, - zip_size, page_no, 0, - zip_size ? zip_size - : UNIV_PAGE_SIZE, - page, NULL, 0, 0); + fil_io(OS_FILE_WRITE, + true, + space_id, + zip_size, + page_no, 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + page, + NULL, + 0, + 0, + false); } } } @@ -621,9 +643,9 @@ buf_dblwr_process() memset(buf, 0, bytes); fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block1, 0, bytes, buf, NULL, NULL, 0); + buf_dblwr->block1, 0, bytes, buf, NULL, NULL, 0, false); fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block2, 0, bytes, buf, NULL, NULL, 0); + buf_dblwr->block2, 0, bytes, buf, NULL, NULL, 0, false); ut_free(unaligned_buf); } @@ -828,13 +850,18 @@ buf_dblwr_write_block_to_datafile( void * frame = buf_page_get_frame(bpage); if (bpage->zip.data) { - fil_io(flags, sync, buf_page_get_space(bpage), - buf_page_get_zip_size(bpage), - buf_page_get_page_no(bpage), 0, - buf_page_get_zip_size(bpage), - frame, - (void*) bpage, 0, - bpage->newest_modification); + fil_io(flags, + sync, + buf_page_get_space(bpage), + buf_page_get_zip_size(bpage), + buf_page_get_page_no(bpage), + 0, + buf_page_get_zip_size(bpage), + frame, + (void*) bpage, + 0, + bpage->newest_modification, + bpage->encrypt_later); return; } @@ -844,10 +871,18 @@ buf_dblwr_write_block_to_datafile( ut_a(buf_block_get_state(block) == BUF_BLOCK_FILE_PAGE); buf_dblwr_check_page_lsn(block->frame); - fil_io(flags, sync, buf_block_get_space(block), 0, - buf_block_get_page_no(block), 0, UNIV_PAGE_SIZE, - frame, (void*) block, - (ulint *)&bpage->write_size, bpage->newest_modification); + fil_io(flags, + sync, + buf_block_get_space(block), + 0, + buf_block_get_page_no(block), + 0, + UNIV_PAGE_SIZE, + frame, + (void*) block, + (ulint *)&bpage->write_size, + bpage->newest_modification, + bpage->encrypt_later); } /********************************************************************//** @@ -939,9 +974,19 @@ try_again: len = ut_min(TRX_SYS_DOUBLEWRITE_BLOCK_SIZE, buf_dblwr->first_free) * UNIV_PAGE_SIZE; - fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block1, 0, len, - (void*) write_buf, NULL, 0, 0); + fil_io(OS_FILE_WRITE, + true, + TRX_SYS_SPACE, + 0, + buf_dblwr->block1, + 0, + len, + (void*) + write_buf, + NULL, + 0, + 0, + false); if (buf_dblwr->first_free <= TRX_SYS_DOUBLEWRITE_BLOCK_SIZE) { /* No unwritten pages in the second block. */ @@ -955,9 +1000,18 @@ try_again: write_buf = buf_dblwr->write_buf + TRX_SYS_DOUBLEWRITE_BLOCK_SIZE * UNIV_PAGE_SIZE; - fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block2, 0, len, - (void*) write_buf, NULL, 0, 0); + fil_io(OS_FILE_WRITE, + true, + TRX_SYS_SPACE, + 0, + buf_dblwr->block2, + 0, + len, + (void*) write_buf, + NULL, + 0, + 0, + false); flush: /* increment the doublewrite flushed pages counter */ @@ -1187,17 +1241,32 @@ retry: memset(buf_dblwr->write_buf + UNIV_PAGE_SIZE * i + zip_size, 0, UNIV_PAGE_SIZE - zip_size); - fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - offset, 0, UNIV_PAGE_SIZE, - (void*) (buf_dblwr->write_buf - + UNIV_PAGE_SIZE * i), NULL, 0,bpage->newest_modification); + fil_io(OS_FILE_WRITE, + true, + TRX_SYS_SPACE, + 0, + offset, + 0, + UNIV_PAGE_SIZE, + (void*) (buf_dblwr->write_buf + UNIV_PAGE_SIZE * i), + NULL, + 0, + bpage->newest_modification, + bpage->encrypt_later); } else { /* It is a regular page. Write it directly to the doublewrite buffer */ - fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - offset, 0, UNIV_PAGE_SIZE, - frame, - NULL, 0, bpage->newest_modification); + fil_io(OS_FILE_WRITE, + true, + TRX_SYS_SPACE, + 0, + offset, + 0, UNIV_PAGE_SIZE, + frame, + NULL, + 0, + bpage->newest_modification, + bpage->encrypt_later); } /* Now flush the doublewrite buffer data to disk */ diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index c7356015788..3411dc27d2c 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -955,16 +955,17 @@ buf_flush_write_block_low( if (!srv_use_doublewrite_buf || !buf_dblwr) { fil_io(OS_FILE_WRITE | OS_AIO_SIMULATED_WAKE_LATER, - sync, - buf_page_get_space(bpage), - zip_size, - buf_page_get_page_no(bpage), - 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - frame, - bpage, - &bpage->write_size, - bpage->newest_modification); + sync, + buf_page_get_space(bpage), + zip_size, + buf_page_get_page_no(bpage), + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + frame, + bpage, + &bpage->write_size, + bpage->newest_modification, + bpage->encrypt_later); } else { /* InnoDB uses doublewrite buffer and doublewrite buffer is initialized. User can define do we use atomic writes @@ -975,16 +976,17 @@ buf_flush_write_block_low( if (awrites == ATOMIC_WRITES_ON) { fil_io(OS_FILE_WRITE | OS_AIO_SIMULATED_WAKE_LATER, - FALSE, - buf_page_get_space(bpage), - zip_size, - buf_page_get_page_no(bpage), - 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - frame, - bpage, - &bpage->write_size, - bpage->newest_modification); + FALSE, + buf_page_get_space(bpage), + zip_size, + buf_page_get_page_no(bpage), + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + frame, + bpage, + &bpage->write_size, + bpage->newest_modification, + bpage->encrypt_later); } else if (flush_type == BUF_FLUSH_SINGLE_PAGE) { buf_dblwr_write_single_page(bpage, sync); } else { diff --git a/storage/xtradb/buf/buf0rea.cc b/storage/xtradb/buf/buf0rea.cc index 271fcdc12aa..afa14f0df04 100644 --- a/storage/xtradb/buf/buf0rea.cc +++ b/storage/xtradb/buf/buf0rea.cc @@ -232,14 +232,14 @@ not_to_recover: *err = _fil_io(OS_FILE_READ | wake_later | ignore_nonexistent_pages, sync, space, zip_size, offset, 0, zip_size, - frame, bpage, 0, trx, 0); + frame, bpage, 0, trx, 0, false); } else { ut_a(buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); *err = _fil_io(OS_FILE_READ | wake_later | ignore_nonexistent_pages, sync, space, 0, offset, 0, UNIV_PAGE_SIZE, - frame, bpage, &bpage->write_size, trx, 0); + frame, bpage, &bpage->write_size, trx, 0, false); } if (sync) { diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 4719a37fab2..d7b26eed49f 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -303,9 +303,8 @@ fil_crypt_get_latest_key(byte *dst, uint* key_length, } return fil_crypt_get_key(dst, key_length, crypt_data, *version, false); - } else { - return fil_crypt_get_key(dst, key_length, NULL, *version, true); } + return fil_crypt_get_key(dst, key_length, NULL, *version, true); } /****************************************************************** @@ -622,13 +621,25 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, byte key[MY_AES_MAX_KEY_LENGTH]; uint key_length; + ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); + if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR + || orig_page_type==FIL_PAGE_TYPE_XDES + || orig_page_type== FIL_PAGE_PAGE_ENCRYPTED + || orig_page_type== FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + //TODO: is this really needed ? + memcpy(dst_frame, src_frame, page_size); + return; + } + if (srv_encrypt_tables) { crypt_data = fil_space_get_crypt_data(space); + if (crypt_data == NULL) { //TODO: Is this really needed ? memcpy(dst_frame, src_frame, page_size); return; } + fil_crypt_get_latest_key(key, &key_length, crypt_data, &key_version); } else { key_version = encryption_key; @@ -647,9 +658,7 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, src_frame + FIL_PAGE_OFFSET); mach_write_to_4(iv + 4, space_offset); mach_write_to_8(iv + 8, lsn); - } - else - { + } else { // take the iv from the key provider int load_iv_rc = get_encryption_iv(key_version, (uchar *) iv, sizeof(iv)); @@ -667,19 +676,9 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, } } - ibool page_compressed = (mach_read_from_2(src_frame+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED); ibool page_encrypted = fil_space_is_page_encrypted(space); ulint compression_alg = mach_read_from_8(src_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); - - if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR - || orig_page_type==FIL_PAGE_TYPE_XDES - || orig_page_type== FIL_PAGE_PAGE_ENCRYPTED - || orig_page_type== FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - memcpy(dst_frame, src_frame, page_size); - return; - } // copy page header memcpy(dst_frame, src_frame, FIL_PAGE_DATA); @@ -708,7 +707,7 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, uint32 dstlen; if (page_compressed) { - srclen = page_size - FIL_PAGE_DATA; + srclen = page_size - FIL_PAGE_DATA;; } int rc = (* my_aes_encrypt_dynamic)(src, srclen, @@ -789,6 +788,7 @@ fil_space_check_encryption_read( ulint space) /*!< in: tablespace id */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); + if (crypt_data == NULL) return false; @@ -815,6 +815,7 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, || page_type == FIL_PAGE_PAGE_COMPRESSED); ulint orig_page_type=0; + if (page_type == FIL_PAGE_PAGE_ENCRYPTED) { key_version = mach_read_from_2( src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); @@ -846,7 +847,6 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, mach_write_to_2(dst_frame+FIL_PAGE_TYPE, orig_page_type); } - // get key byte key[MY_AES_MAX_KEY_LENGTH]; uint key_length; @@ -862,9 +862,7 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, mach_write_to_4(iv + 0, space); mach_write_to_4(iv + 4, offset); mach_write_to_8(iv + 8, lsn); - } - else - { + } else { // take the iv from the key provider int load_iv_rc = get_encryption_iv(key_version, (uchar *) iv, sizeof(iv)); @@ -886,7 +884,6 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, byte* dst = dst_frame + FIL_PAGE_DATA; uint32 dstlen; ulint srclen = page_size - (FIL_PAGE_DATA + FIL_PAGE_DATA_END); - ulint compressed_len; ulint compression_method; diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 530326c0f66..8a01dc7d7e6 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -287,7 +287,7 @@ fil_read( actual page size does not decrease. */ { return(fil_io(OS_FILE_READ, sync, space_id, zip_size, block_offset, - byte_offset, len, buf, message, write_size, 0)); + byte_offset, len, buf, message, write_size, 0, false)); } /********************************************************************//** @@ -319,12 +319,13 @@ fil_write( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn) /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later) /*!< in: encrypt later ? */ { ut_ad(!srv_read_only_mode); return(fil_io(OS_FILE_WRITE, sync, space_id, zip_size, block_offset, - byte_offset, len, buf, message, write_size, lsn)); + byte_offset, len, buf, message, write_size, lsn, encrypt_later)); } /*******************************************************************//** @@ -617,6 +618,8 @@ fil_node_open_file( ut_a(size_bytes != (os_offset_t) -1); node->file_block_size = os_file_get_block_size(node->handle, node->name); + space->file_block_size = node->file_block_size; + #ifdef UNIV_HOTBACKUP if (space->id == 0) { node->size = (ulint) (size_bytes / UNIV_PAGE_SIZE); @@ -782,6 +785,7 @@ add_size: if (node->file_block_size == 0) { node->file_block_size = os_file_get_block_size(node->handle, node->name); + space->file_block_size = node->file_block_size; } ut_a(ret); @@ -1874,7 +1878,7 @@ fil_write_lsn_and_arch_no_to_file( lsn); err = fil_write(TRUE, space, 0, sum_of_sizes, 0, - UNIV_PAGE_SIZE, buf, NULL, 0, 0); + UNIV_PAGE_SIZE, buf, NULL, 0, 0, false); } mem_free(buf1); @@ -5189,6 +5193,7 @@ retry: /* Determine correct file block size */ if (node->file_block_size == 0) { node->file_block_size = os_file_get_block_size(node->handle, node->name); + space->file_block_size = node->file_block_size; } #ifdef HAVE_POSIX_FALLOCATE @@ -5248,7 +5253,7 @@ retry: success = os_aio(OS_FILE_WRITE, OS_AIO_SYNC, node->name, node->handle, buf, offset, page_size * n_pages, - node, NULL, space_id, NULL, 0, 0, 0, 0, 0, 0); + node, NULL, space_id, NULL, 0, 0, 0, 0, 0, 0, false); #endif /* UNIV_HOTBACKUP */ if (success) { os_has_said_disk_full = FALSE; @@ -5630,8 +5635,8 @@ _fil_io( initialized we do not trim again if actual page size does not decrease. */ trx_t* trx, - lsn_t lsn) /* lsn of the newest modification */ - + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later) /*!< in: encrypt later ? */ { ulint mode; fil_space_t* space; @@ -5880,7 +5885,8 @@ _fil_io( write_size, page_encrypted, page_encryption_key, - lsn); + lsn, + encrypt_later); #else /* In mysqlbackup do normal i/o, not aio */ diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc index 2223701f444..987493c8d26 100644 --- a/storage/xtradb/fil/fil0pagecompress.cc +++ b/storage/xtradb/fil/fil0pagecompress.cc @@ -283,7 +283,8 @@ fil_compress_page( /* Let's not compress file space header or extent descriptor */ - if (orig_page_type == FIL_PAGE_TYPE_FSP_HDR || + if (orig_page_type == 0 || + orig_page_type == FIL_PAGE_TYPE_FSP_HDR || orig_page_type == FIL_PAGE_TYPE_XDES || orig_page_type == FIL_PAGE_PAGE_COMPRESSED || orig_page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { diff --git a/storage/xtradb/include/buf0buf.h b/storage/xtradb/include/buf0buf.h index a2097b5c3b9..7eb58216698 100644 --- a/storage/xtradb/include/buf0buf.h +++ b/storage/xtradb/include/buf0buf.h @@ -1598,6 +1598,8 @@ struct buf_page_t{ can be read while it's being flushed */ byte* comp_buf_free; /*!< for compression, allocated buffer that is then alligned */ + bool encrypt_later; /*!< should we encrypt the page + at os0file.cc ? */ #ifndef UNIV_HOTBACKUP buf_page_t* hash; /*!< node used in chaining to buf_pool->page_hash or diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 360a26df776..1847a314d06 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2015, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -340,6 +340,7 @@ struct fil_space_t { UT_LIST_NODE_T(fil_space_t) space_list; /*!< list of all spaces */ fil_space_crypt_t* crypt_data; + ulint file_block_size;/*!< file system block size */ ulint magic_n;/*!< FIL_SPACE_MAGIC_N */ }; @@ -959,8 +960,8 @@ fil_space_get_n_reserved_extents( Reads or writes data. This operation is asynchronous (aio). @return DB_SUCCESS, or DB_TABLESPACE_DELETED if we are trying to do i/o on a tablespace which does not exist */ -#define fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size, lsn) \ - _fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size, NULL, lsn) +#define fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size, lsn, encrypt) \ + _fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size, NULL, lsn, encrypt) UNIV_INTERN dberr_t @@ -997,7 +998,8 @@ _fil_io( initialized we do not trim again if actual page size does not decrease. */ trx_t* trx, - lsn_t lsn) /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later) /*!< in: encrypt later ? */ __attribute__((nonnull(8))); /**********************************************************************//** diff --git a/storage/xtradb/include/os0file.h b/storage/xtradb/include/os0file.h index af3ad734f48..f93bf026903 100644 --- a/storage/xtradb/include/os0file.h +++ b/storage/xtradb/include/os0file.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2015, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -324,11 +324,11 @@ The wrapper functions have the prefix of "innodb_". */ # define os_aio(type, mode, name, file, buf, offset, \ n, message1, message2, space_id, \ trx, page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key, lsn) \ + page_encryption, page_encryption_key, lsn, encrypt) \ pfs_os_aio_func(type, mode, name, file, buf, offset, \ n, message1, message2, space_id, trx, \ page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key, lsn, \ + page_encryption, page_encryption_key, lsn, encrypt, \ __FILE__, __LINE__) # define os_file_read(file, buf, offset, n, compressed) \ @@ -378,11 +378,11 @@ to original un-instrumented file I/O APIs */ # define os_aio(type, mode, name, file, buf, offset, n, message1, \ message2, space_id, trx, \ page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key, lsn) \ + page_encryption, page_encryption_key, lsn, encrypt) \ os_aio_func(type, mode, name, file, buf, offset, n, \ message1, message2, space_id, trx, \ page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key, lsn) + page_encryption, page_encryption_key, lsn, encrypt) # define os_file_read(file, buf, offset, n, compressed) \ os_file_read_func(file, buf, offset, n, NULL, compressed) @@ -812,9 +812,10 @@ pfs_os_aio_func( actual page size does not decrease. */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key, /*!< page encryption + ulint page_encryption_key, /*!< in: page encryption key to be used */ - lsn_t lsn, /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later, /*!< in: should we encrypt ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line);/*!< in: line where the func invoked */ /*******************************************************************//** @@ -1204,10 +1205,10 @@ os_aio_func( actual page size does not decrease. */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key, /*!< page encryption key + ulint page_encryption_key, /*!< in: page encryption key to be used */ - lsn_t lsn); /* lsn of the newest modification */ - + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later); /*!< in: should we encrypt ? */ /************************************************************************//** Wakes up all async i/o threads so that they know to exit themselves in shutdown. */ diff --git a/storage/xtradb/include/os0file.ic b/storage/xtradb/include/os0file.ic index 9ab191bd460..39599ba2047 100644 --- a/storage/xtradb/include/os0file.ic +++ b/storage/xtradb/include/os0file.ic @@ -222,7 +222,7 @@ pfs_os_aio_func( trx_t* trx, ibool page_compression, /*!< in: is page compression used on this file space */ - ulint page_compression_level, /*!< page compression + ulint page_compression_level, /*!< in: page compression level to be used */ ulint* write_size,/*!< in/out: Actual write size initialized after fist successfull trim @@ -231,10 +231,12 @@ pfs_os_aio_func( actual page size does not decrease. */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key, /*!< page encryption + ulint page_encryption_key, /*!< in: page encryption key to be used */ - lsn_t lsn, /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest + modification */ + bool encrypt_later, /*!< in: encrypt later ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line)/*!< in: line where the func invoked */ { @@ -252,7 +254,7 @@ pfs_os_aio_func( result = os_aio_func(type, mode, name, file, buf, offset, n, message1, message2, space_id, trx, page_compression, page_compression_level, write_size , - page_encryption, page_encryption_key, lsn); + page_encryption, page_encryption_key, lsn, encrypt_later); register_pfs_file_io_end(locker, n); diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index e007bcc3507..37cf18c1a0d 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -1384,7 +1384,7 @@ log_group_file_header_flush( (ulint) (dest_offset / UNIV_PAGE_SIZE), (ulint) (dest_offset % UNIV_PAGE_SIZE), OS_FILE_LOG_BLOCK_SIZE, - buf, group, 0, 0); + buf, group, 0, 0, false); srv_stats.os_log_pending_writes.dec(); } @@ -1551,7 +1551,7 @@ loop: fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, group->space_id, 0, (ulint) (next_offset / UNIV_PAGE_SIZE), (ulint) (next_offset % UNIV_PAGE_SIZE), write_len, buf, - group, 0, 0); + group, 0, 0, false); srv_stats.os_log_pending_writes.dec(); @@ -2144,7 +2144,7 @@ log_group_checkpoint( write_offset / UNIV_PAGE_SIZE, write_offset % UNIV_PAGE_SIZE, OS_FILE_LOG_BLOCK_SIZE, - buf, ((byte*) group + 1), 0, 0); + buf, ((byte*) group + 1), 0, 0, false); ut_ad(((ulint) group & 0x1UL) == 0); } @@ -2226,7 +2226,7 @@ log_group_read_checkpoint_info( fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->space_id, 0, field / UNIV_PAGE_SIZE, field % UNIV_PAGE_SIZE, - OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0, 0); + OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0, 0, false); } /******************************************************//** @@ -2639,7 +2639,7 @@ loop: fil_io(OS_FILE_READ | OS_FILE_LOG, sync, group->space_id, 0, (ulint) (source_offset / UNIV_PAGE_SIZE), (ulint) (source_offset % UNIV_PAGE_SIZE), - len, buf, (type == LOG_ARCHIVE) ? &log_archive_io : NULL, 0, 0); + len, buf, (type == LOG_ARCHIVE) ? &log_archive_io : NULL, 0, 0, false); if (recv_sys->recv_log_crypt_ver != UNENCRYPTED_KEY_VER && !log_group_decrypt_after_read(group, buf, len)) @@ -2771,7 +2771,7 @@ log_group_archive_file_header_write( dest_offset / UNIV_PAGE_SIZE, dest_offset % UNIV_PAGE_SIZE, 2 * OS_FILE_LOG_BLOCK_SIZE, - buf, &log_archive_io, 0, 0); + buf, &log_archive_io, 0, 0, false); } /******************************************************//** @@ -2808,7 +2808,7 @@ log_group_archive_completed_header_write( dest_offset % UNIV_PAGE_SIZE, OS_FILE_LOG_BLOCK_SIZE, buf + LOG_FILE_ARCH_COMPLETED, - &log_archive_io, 0, 0); + &log_archive_io, 0, 0, false); } /******************************************************//** @@ -2953,7 +2953,7 @@ loop: (ulint) (next_offset / UNIV_PAGE_SIZE), (ulint) (next_offset % UNIV_PAGE_SIZE), ut_calc_align(len, OS_FILE_LOG_BLOCK_SIZE), buf, - &log_archive_io, 0, 0); + &log_archive_io, 0, 0, false); start_lsn += len; next_offset += len; diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index bb0888d3db7..c7fb3be9625 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -2156,7 +2156,7 @@ recv_apply_log_recs_for_backup(void) error = fil_io(OS_FILE_READ, true, recv_addr->space, zip_size, recv_addr->page_no, 0, zip_size, - block->page.zip.data, NULL, 0, 0); + block->page.zip.data, NULL, 0, 0, false); if (error == DB_SUCCESS && !buf_zip_decompress(block, TRUE)) { exit(1); @@ -2166,7 +2166,7 @@ recv_apply_log_recs_for_backup(void) recv_addr->space, 0, recv_addr->page_no, 0, UNIV_PAGE_SIZE, - block->frame, NULL, 0, 0); + block->frame, NULL, 0, 0, false); } if (error != DB_SUCCESS) { @@ -2195,13 +2195,15 @@ recv_apply_log_recs_for_backup(void) recv_addr->space, zip_size, recv_addr->page_no, 0, zip_size, - block->page.zip.data, NULL, 0, 0); + block->page.zip.data, NULL, 0, 0, false); } else { error = fil_io(OS_FILE_WRITE, true, recv_addr->space, 0, recv_addr->page_no, 0, UNIV_PAGE_SIZE, - block->frame, NULL, 0, 0); + block->frame, NULL, 0, + block->latest_modification, + block->encrypt_later); } skip_this_recv_addr: recv_addr = HASH_GET_NEXT(addr_hash, recv_addr); @@ -3175,7 +3177,7 @@ recv_recovery_from_checkpoint_start_func( fil_io(OS_FILE_READ | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, LOG_FILE_HDR_SIZE, - log_hdr_buf, max_cp_group, 0, 0); + log_hdr_buf, max_cp_group, 0, 0, false); if (0 == ut_memcmp(log_hdr_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, (byte*)"ibbackup", (sizeof "ibbackup") - 1)) { @@ -3206,7 +3208,7 @@ recv_recovery_from_checkpoint_start_func( fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, OS_FILE_LOG_BLOCK_SIZE, - log_hdr_buf, max_cp_group, 0, 0); + log_hdr_buf, max_cp_group, 0, 0, false); } log_hdr_log_block_size @@ -3869,7 +3871,7 @@ ask_again: /* Read the archive file header */ fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->archive_space_id, 0, 0, 0, - LOG_FILE_HDR_SIZE, buf, NULL, 0, 0); + LOG_FILE_HDR_SIZE, buf, NULL, 0, 0, false); /* Check if the archive file header is consistent */ @@ -3943,7 +3945,7 @@ ask_again: fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->archive_space_id, 0, read_offset / UNIV_PAGE_SIZE, - read_offset % UNIV_PAGE_SIZE, len, buf, NULL, 0, 0); + read_offset % UNIV_PAGE_SIZE, len, buf, NULL, 0, 0, false); ret = recv_scan_log_recs( (buf_pool_get_n_pages() diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 05ab1490b98..a737ebb1fa0 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -273,6 +273,7 @@ struct os_aio_slot_t{ lsn_t lsn; /* lsn of the newest modification */ ulint file_block_size;/*!< file block size */ + bool encrypt_later; /*!< should we encrypt the page */ #ifdef LINUX_NATIVE_AIO struct iocb control; /* Linux control block for aio */ @@ -1982,6 +1983,9 @@ os_file_create_func( #else if (purpose == OS_FILE_AIO) { + bool encrypt_later; /*!< should the page be encrypted + before write */ + #ifdef WIN_ASYNC_IO /* If specified, use asynchronous (overlapped) io and no buffering of writes in the OS */ @@ -4663,7 +4667,9 @@ os_aio_array_reserve_slot( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn) /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest + modification */ + bool encrypt_later) /*!< in: should we encrypt the page */ { os_aio_slot_t* slot = NULL; #ifdef WIN_ASYNC_IO @@ -4755,15 +4761,14 @@ found: slot->lsn = lsn; slot->io_already_done = FALSE; slot->space_id = space_id; - slot->page_compression_success = FALSE; slot->page_encryption_success = FALSE; - slot->write_size = write_size; slot->page_compression_level = page_compression_level; slot->page_compression = page_compression; slot->page_encryption_key = page_encryption_key; slot->page_encryption = page_encryption; + slot->encrypt_later = encrypt_later; if (message1) { slot->file_block_size = fil_node_get_block_size(message1); @@ -4819,7 +4824,7 @@ found: /* If the space is page encryption and this is write operation then we encrypt the page */ - if (message1 && type == OS_FILE_WRITE && page_encryption ) { + if (message1 && type == OS_FILE_WRITE && (page_encryption || encrypt_later)) { /* Release the array mutex while encrypting */ os_mutex_exit(array->mutex); @@ -4843,6 +4848,8 @@ found: os_mutex_enter(array->mutex); } + slot->buf = (byte *)buf; + #ifdef WIN_ASYNC_IO control = &slot->control; control->Offset = (DWORD) offset & 0xFFFFFFFF; @@ -5130,9 +5137,11 @@ os_aio_func( actual page size does not decrease. */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key, /*!< page encryption key + ulint page_encryption_key, /*!< in: page encryption key to be used */ - lsn_t lsn) /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later) /*!< in: should we encrypt before + writing the page */ { os_aio_array_t* array; os_aio_slot_t* slot; @@ -5239,7 +5248,8 @@ try_again: slot = os_aio_array_reserve_slot(type, array, message1, message2, file, name, buf, offset, n, space_id, page_compression, page_compression_level, - page_encryption, page_encryption_key, write_size, lsn); + page_encryption, page_encryption_key, + write_size, lsn, encrypt_later); if (type == OS_FILE_READ) { if (srv_use_native_aio) { |