diff options
author | Jan Lindström <jan.lindstrom@skysql.com> | 2015-01-09 12:30:59 +0200 |
---|---|---|
committer | Sergei Golubchik <serg@mariadb.org> | 2015-02-10 10:21:18 +0100 |
commit | e2e809860e8a1ad6a4d82859558cfd89409ac6d4 (patch) | |
tree | 2b300109a168ec882f89aee88aa09bd588c0176d /storage | |
parent | e109a662c77ec8f6db42c940d6c82c00d2f9f5d9 (diff) | |
download | mariadb-git-e2e809860e8a1ad6a4d82859558cfd89409ac6d4.tar.gz |
Pass down the information should we encrypt the page at os0file.cc
when page compression and google encryption is used.
Diffstat (limited to 'storage')
28 files changed, 425 insertions, 249 deletions
diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index c0859bf82b6..59cbe633dec 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1006,6 +1006,7 @@ buf_block_init( block->page.comp_buf = NULL; block->page.comp_buf_free = NULL; block->page.key_version = 0; + block->page.encrypt_later = false; block->modify_clock = 0; @@ -3498,6 +3499,7 @@ buf_page_init_low( bpage->comp_buf = NULL; bpage->comp_buf_free = NULL; bpage->key_version = 0; + bpage->encrypt_later = false; HASH_INVALIDATE(bpage, hash); #if defined UNIV_DEBUG_FILE_ACCESSES || defined UNIV_DEBUG @@ -5609,6 +5611,8 @@ buf_page_encrypt_before_write( buf_page_t* bpage, /*!< in/out: buffer page to be flushed */ const byte* src_frame) /*!< in: src frame */ { + bpage->encrypt_later = false; + if (srv_encrypt_tables == FALSE) { /* Encryption is disabled */ return const_cast<byte*>(src_frame); @@ -5669,7 +5673,8 @@ buf_page_encrypt_before_write( ut_ad(key_version == 0 || key_version >= bpage->key_version); bpage->key_version = key_version; } else { - // We do compression and encryption later on os0file.cc + /** Compression and encryption is done later at os0file.cc */ + bpage->encrypt_later = true; dst_frame = (byte *)src_frame; } diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 37b8152f1fa..477bcfe0ee3 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -521,10 +521,18 @@ buf_dblwr_process() ulint zip_size = fil_space_get_zip_size(space_id); /* Read in the actual page from the file */ - fil_io(OS_FILE_READ, true, space_id, zip_size, - page_no, 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - read_buf, NULL, 0, 0); + fil_io(OS_FILE_READ, + true, + space_id, + zip_size, + page_no, + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + read_buf, + NULL, + 0, + 0, + false); if (fil_space_verify_crypt_checksum(read_buf, zip_size)) { /* page is encrypted and checksum is OK */ @@ -576,10 +584,18 @@ buf_dblwr_process() doublewrite buffer to the intended position */ - fil_io(OS_FILE_WRITE, true, space_id, - zip_size, page_no, 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - page, NULL, 0, 0); + fil_io(OS_FILE_WRITE, + true, + space_id, + zip_size, + page_no, + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + page, + NULL, + 0, + 0, + false); ib_logf(IB_LOG_LEVEL_INFO, "Recovered the page from" @@ -595,11 +611,18 @@ buf_dblwr_process() zeroes, while a valid copy is available in dblwr buffer. */ - fil_io(OS_FILE_WRITE, true, space_id, - zip_size, page_no, 0, - zip_size ? zip_size - : UNIV_PAGE_SIZE, - page, NULL, 0, 0); + fil_io(OS_FILE_WRITE, + true, + space_id, + zip_size, + page_no, + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + page, + NULL, + 0, + 0, + false); } } } @@ -621,9 +644,9 @@ buf_dblwr_process() memset(buf, 0, bytes); fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block1, 0, bytes, buf, NULL, NULL, 0); + buf_dblwr->block1, 0, bytes, buf, NULL, NULL, 0, false); fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block2, 0, bytes, buf, NULL, NULL, 0); + buf_dblwr->block2, 0, bytes, buf, NULL, NULL, 0, false); ut_free(unaligned_buf); } @@ -828,12 +851,18 @@ buf_dblwr_write_block_to_datafile( void * frame = buf_page_get_frame(bpage); if (bpage->zip.data) { - fil_io(flags, sync, buf_page_get_space(bpage), - buf_page_get_zip_size(bpage), - buf_page_get_page_no(bpage), 0, - buf_page_get_zip_size(bpage), - frame, - (void*) bpage, 0, bpage->newest_modification); + fil_io(flags, + sync, + buf_page_get_space(bpage), + buf_page_get_zip_size(bpage), + buf_page_get_page_no(bpage), + 0, + buf_page_get_zip_size(bpage), + frame, + (void*) bpage, + 0, + bpage->newest_modification, + bpage->encrypt_later); return; } @@ -843,12 +872,18 @@ buf_dblwr_write_block_to_datafile( ut_a(buf_block_get_state(block) == BUF_BLOCK_FILE_PAGE); buf_dblwr_check_page_lsn(block->frame); - fprintf(stderr, "JAN: space %lu page_type %lu\n", buf_block_get_space(block), - mach_read_from_2((byte *)frame+FIL_PAGE_TYPE)); - - fil_io(flags, sync, buf_block_get_space(block), 0, - buf_block_get_page_no(block), 0, UNIV_PAGE_SIZE, - frame, (void*) block, (ulint *)&bpage->write_size, bpage->newest_modification ); + fil_io(flags, + sync, + buf_block_get_space(block), + 0, + buf_block_get_page_no(block), + 0, + UNIV_PAGE_SIZE, + frame, + (void*) block, + (ulint *)&bpage->write_size, + bpage->newest_modification, + bpage->encrypt_later); } /********************************************************************//** @@ -942,7 +977,7 @@ try_again: fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, buf_dblwr->block1, 0, len, - (void*) write_buf, NULL, 0, 0); + (void*) write_buf, NULL, 0, 0, false); if (buf_dblwr->first_free <= TRX_SYS_DOUBLEWRITE_BLOCK_SIZE) { /* No unwritten pages in the second block. */ @@ -958,7 +993,7 @@ try_again: fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, buf_dblwr->block2, 0, len, - (void*) write_buf, NULL, 0, 0); + (void*) write_buf, NULL, 0, 0, false); flush: /* increment the doublewrite flushed pages counter */ @@ -1187,17 +1222,31 @@ retry: memset(buf_dblwr->write_buf + UNIV_PAGE_SIZE * i + zip_size, 0, UNIV_PAGE_SIZE - zip_size); - fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - offset, 0, UNIV_PAGE_SIZE, - (void*) (buf_dblwr->write_buf - + UNIV_PAGE_SIZE * i), NULL, 0, bpage->newest_modification); + fil_io(OS_FILE_WRITE, + true, + TRX_SYS_SPACE, 0, + offset, + 0, + UNIV_PAGE_SIZE, + (void*) (buf_dblwr->write_buf + UNIV_PAGE_SIZE * i), + NULL, + 0, + bpage->newest_modification, + bpage->encrypt_later); } else { /* It is a regular page. Write it directly to the doublewrite buffer */ - fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - offset, 0, UNIV_PAGE_SIZE, - frame, - NULL, 0, bpage->newest_modification); + fil_io(OS_FILE_WRITE, + true, + TRX_SYS_SPACE, 0, + offset, + 0, + UNIV_PAGE_SIZE, + frame, + NULL, + 0, + bpage->newest_modification, + bpage->encrypt_later); } /* Now flush the doublewrite buffer data to disk */ diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 9c11ae2b43e..9bd7a7c007d 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -913,16 +913,17 @@ buf_flush_write_block_low( if (!srv_use_doublewrite_buf || !buf_dblwr) { fil_io(OS_FILE_WRITE | OS_AIO_SIMULATED_WAKE_LATER, - sync, - buf_page_get_space(bpage), - zip_size, - buf_page_get_page_no(bpage), - 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - frame, - bpage, - &bpage->write_size, - bpage->newest_modification); + sync, + buf_page_get_space(bpage), + zip_size, + buf_page_get_page_no(bpage), + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + frame, + bpage, + &bpage->write_size, + bpage->newest_modification, + bpage->encrypt_later); } else { /* InnoDB uses doublewrite buffer and doublewrite buffer @@ -943,7 +944,8 @@ buf_flush_write_block_low( frame, bpage, &bpage->write_size, - bpage->newest_modification); + bpage->newest_modification, + bpage->encrypt_later); } else if (flush_type == BUF_FLUSH_SINGLE_PAGE) { buf_dblwr_write_single_page(bpage, sync); } else { diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 19d18dcd870..e91a5da6621 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -184,17 +184,17 @@ buf_read_page_low( if (zip_size) { *err = fil_io(OS_FILE_READ | wake_later - | ignore_nonexistent_pages, - sync, space, zip_size, offset, 0, zip_size, - frame, bpage, &bpage->write_size, 0); + | ignore_nonexistent_pages, + sync, space, zip_size, offset, 0, zip_size, + frame, bpage, &bpage->write_size, 0, false); } else { ut_a(buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); *err = fil_io(OS_FILE_READ | wake_later - | ignore_nonexistent_pages, - sync, space, 0, offset, 0, UNIV_PAGE_SIZE, - frame, bpage, - &bpage->write_size, 0); + | ignore_nonexistent_pages, + sync, space, 0, offset, 0, UNIV_PAGE_SIZE, + frame, bpage, + &bpage->write_size, 0, false); } if (sync) { diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 9d4c66d4343..d7b26eed49f 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -621,13 +621,25 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, byte key[MY_AES_MAX_KEY_LENGTH]; uint key_length; + ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); + if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR + || orig_page_type==FIL_PAGE_TYPE_XDES + || orig_page_type== FIL_PAGE_PAGE_ENCRYPTED + || orig_page_type== FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + //TODO: is this really needed ? + memcpy(dst_frame, src_frame, page_size); + return; + } + if (srv_encrypt_tables) { crypt_data = fil_space_get_crypt_data(space); + if (crypt_data == NULL) { //TODO: Is this really needed ? memcpy(dst_frame, src_frame, page_size); return; } + fil_crypt_get_latest_key(key, &key_length, crypt_data, &key_version); } else { key_version = encryption_key; @@ -646,9 +658,7 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, src_frame + FIL_PAGE_OFFSET); mach_write_to_4(iv + 4, space_offset); mach_write_to_8(iv + 8, lsn); - } - else - { + } else { // take the iv from the key provider int load_iv_rc = get_encryption_iv(key_version, (uchar *) iv, sizeof(iv)); @@ -666,21 +676,10 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, } } - ibool page_compressed = (mach_read_from_2(src_frame+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED); ibool page_encrypted = fil_space_is_page_encrypted(space); - ulint compression_alg = mach_read_from_8(src_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); - if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR - || orig_page_type==FIL_PAGE_TYPE_XDES - || orig_page_type== FIL_PAGE_PAGE_ENCRYPTED - || orig_page_type== FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - memcpy(dst_frame, src_frame, page_size); - return; - } - // copy page header memcpy(dst_frame, src_frame, FIL_PAGE_DATA); @@ -711,7 +710,6 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, srclen = page_size - FIL_PAGE_DATA;; } - int rc = (* my_aes_encrypt_dynamic)(src, srclen, dst, &dstlen, (unsigned char*)key, key_length, @@ -790,6 +788,7 @@ fil_space_check_encryption_read( ulint space) /*!< in: tablespace id */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); + if (crypt_data == NULL) return false; @@ -816,6 +815,7 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, || page_type == FIL_PAGE_PAGE_COMPRESSED); ulint orig_page_type=0; + if (page_type == FIL_PAGE_PAGE_ENCRYPTED) { key_version = mach_read_from_2( src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); @@ -847,7 +847,6 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, mach_write_to_2(dst_frame+FIL_PAGE_TYPE, orig_page_type); } - // get key byte key[MY_AES_MAX_KEY_LENGTH]; uint key_length; @@ -863,9 +862,7 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, mach_write_to_4(iv + 0, space); mach_write_to_4(iv + 4, offset); mach_write_to_8(iv + 8, lsn); - } - else - { + } else { // take the iv from the key provider int load_iv_rc = get_encryption_iv(key_version, (uchar *) iv, sizeof(iv)); @@ -887,7 +884,6 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, byte* dst = dst_frame + FIL_PAGE_DATA; uint32 dstlen; ulint srclen = page_size - (FIL_PAGE_DATA + FIL_PAGE_DATA_END); - ulint compressed_len; ulint compression_method; diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 7eb585609d2..7c46bdbca6b 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -284,7 +284,7 @@ fil_read( actual page size does not decrease. */ { return(fil_io(OS_FILE_READ, sync, space_id, zip_size, block_offset, - byte_offset, len, buf, message, write_size, 0)); + byte_offset, len, buf, message, write_size, 0, false)); } /********************************************************************//** @@ -316,12 +316,13 @@ fil_write( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn) /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later) /*!< in: encrypt later ? */ { ut_ad(!srv_read_only_mode); return(fil_io(OS_FILE_WRITE, sync, space_id, zip_size, block_offset, - byte_offset, len, buf, message, write_size, lsn)); + byte_offset, len, buf, message, write_size, lsn, encrypt_later)); } /*******************************************************************//** @@ -614,6 +615,8 @@ fil_node_open_file( ut_a(size_bytes != (os_offset_t) -1); node->file_block_size = os_file_get_block_size(node->handle, node->name); + space->file_block_size = node->file_block_size; + #ifdef UNIV_HOTBACKUP if (space->id == 0) { node->size = (ulint) (size_bytes / UNIV_PAGE_SIZE); @@ -780,6 +783,7 @@ add_size: if (node->file_block_size == 0) { node->file_block_size = os_file_get_block_size(node->handle, node->name); + space->file_block_size = node->file_block_size; } ut_a(ret); @@ -1831,7 +1835,7 @@ fil_write_lsn_and_arch_no_to_file( lsn); err = fil_write(TRUE, space, 0, sum_of_sizes, 0, - UNIV_PAGE_SIZE, buf, NULL, 0, lsn); + UNIV_PAGE_SIZE, buf, NULL, 0, lsn, false); } mem_free(buf1); @@ -5151,6 +5155,7 @@ retry: /* Determine correct file block size */ if (node->file_block_size == 0) { node->file_block_size = os_file_get_block_size(node->handle, node->name); + space->file_block_size = node->file_block_size; } #ifdef HAVE_POSIX_FALLOCATE @@ -5211,7 +5216,7 @@ retry: success = os_aio(OS_FILE_WRITE, OS_AIO_SYNC, node->name, node->handle, buf, offset, page_size * n_pages, - node, NULL, 0, FALSE, 0, 0, 0, 0); + node, NULL, 0, FALSE, 0, 0, 0, 0, false); #endif /* UNIV_HOTBACKUP */ if (success) { os_has_said_disk_full = FALSE; @@ -5592,7 +5597,8 @@ fil_io( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn) /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later) /*!< in: encrypt later ? */ { ulint mode; fil_space_t* space; @@ -5820,7 +5826,8 @@ fil_io( page_compression_level, page_encrypted, page_encryption_key, - lsn); + lsn, + encrypt_later); #endif /* UNIV_HOTBACKUP */ diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index 53bf4ee42ca..670af29e7b5 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -281,11 +281,10 @@ fil_compress_page( /* read original page type */ orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); - ut_ad(orig_page_type != 0); - /* Let's not compress file space header or extent descriptor */ - if (orig_page_type == FIL_PAGE_TYPE_FSP_HDR || + if (orig_page_type == 0 || + orig_page_type == FIL_PAGE_TYPE_FSP_HDR || orig_page_type == FIL_PAGE_TYPE_XDES || orig_page_type == FIL_PAGE_PAGE_COMPRESSED || orig_page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 2863ab01ff9..0f2f9083675 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1566,6 +1566,8 @@ struct buf_page_t{ can be read while it's being flushed */ byte* comp_buf_free; /*!< for compression, allocated buffer that is then alligned */ + bool encrypt_later; /*!< should we encrypt the page + at os0file.cc ? */ #ifndef UNIV_HOTBACKUP buf_page_t* hash; /*!< node used in chaining to diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 1ed8cbf3293..bd7f39939a3 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2015, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -346,6 +346,7 @@ struct fil_space_t { UT_LIST_NODE_T(fil_space_t) space_list; /*!< list of all spaces */ fil_space_crypt_t* crypt_data; + ulint file_block_size;/*!< file system block size */ ulint magic_n;/*!< FIL_SPACE_MAGIC_N */ }; @@ -998,7 +999,8 @@ fil_io( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn) /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later) /*!< in: should we encrypt the page */ __attribute__((nonnull(8))); /**********************************************************************//** Waits for an aio operation to complete. This function is used to write the diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h index e2d0cf26682..73daea32d72 100644 --- a/storage/innobase/include/os0file.h +++ b/storage/innobase/include/os0file.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2015, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -314,12 +314,12 @@ The wrapper functions have the prefix of "innodb_". */ # define os_aio(type, mode, name, file, buf, offset, \ n, message1, message2, write_size, \ page_compression, page_compression_level, \ - page_encryption, page_encryption_key, lsn) \ + page_encryption, page_encryption_key, lsn, encrypt) \ pfs_os_aio_func(type, mode, name, file, buf, offset, \ n, message1, message2, write_size, \ page_compression, page_compression_level, \ page_encryption, page_encryption_key, \ - lsn, __FILE__, __LINE__) + lsn, encrypt, __FILE__, __LINE__) # define os_file_read(file, buf, offset, n, compressed) \ @@ -363,11 +363,11 @@ to original un-instrumented file I/O APIs */ # define os_aio(type, mode, name, file, buf, offset, n, message1, \ message2, write_size, page_compression, page_compression_level, \ - page_encryption, page_encryption_key, lsn) \ + page_encryption, page_encryption_key, lsn, encrypt) \ os_aio_func(type, mode, name, file, buf, offset, n, \ message1, message2, write_size, \ page_compression, page_compression_level, \ - page_encryption, page_encryption_key, lsn) + page_encryption, page_encryption_key, lsn, encrypt) # define os_file_read(file, buf, offset, n, compressed) \ os_file_read_func(file, buf, offset, n, compressed) @@ -787,9 +787,10 @@ pfs_os_aio_func( level to be used */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key, /*!< page encryption + ulint page_encryption_key, /*!< in: page encryption key to be used */ - lsn_t lsn, /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later, /*!< in: should we encrypt ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line);/*!< in: line where the func invoked */ /*******************************************************************//** @@ -1170,9 +1171,10 @@ os_aio_func( level to be used */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key, /*!< page encryption key + ulint page_encryption_key, /*!< in: page encryption key to be used */ - lsn_t lsn); /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later); /*!< in: should we encrypt ? */ /************************************************************************//** Wakes up all async i/o threads so that they know to exit themselves in shutdown. */ diff --git a/storage/innobase/include/os0file.ic b/storage/innobase/include/os0file.ic index bbd3826d50b..9839a841188 100644 --- a/storage/innobase/include/os0file.ic +++ b/storage/innobase/include/os0file.ic @@ -222,13 +222,15 @@ pfs_os_aio_func( actual page size does not decrease. */ ibool page_compression, /*!< in: is page compression used on this file space */ - ulint page_compression_level, /*!< page compression + ulint page_compression_level, /*!< in: page compression level to be used */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key, /*!< page encryption + ulint page_encryption_key, /*!< in: page encryption key to be used */ - lsn_t lsn, /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest + modification */ + bool encrypt_later, /*!< in: encrypt later ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line)/*!< in: line where the func invoked */ { @@ -246,7 +248,7 @@ pfs_os_aio_func( result = os_aio_func(type, mode, name, file, buf, offset, n, message1, message2, write_size, page_compression, page_compression_level, - page_encryption, page_encryption_key, lsn); + page_encryption, page_encryption_key, lsn, encrypt_later); register_pfs_file_io_end(locker, n); diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index ba05987dfbe..779ad112b05 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -1276,7 +1276,7 @@ log_group_file_header_flush( (ulint) (dest_offset / UNIV_PAGE_SIZE), (ulint) (dest_offset % UNIV_PAGE_SIZE), OS_FILE_LOG_BLOCK_SIZE, - buf, group, 0, 0); + buf, group, 0, 0, false); srv_stats.os_log_pending_writes.dec(); } @@ -1443,7 +1443,7 @@ loop: fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, group->space_id, 0, (ulint) (next_offset / UNIV_PAGE_SIZE), (ulint) (next_offset % UNIV_PAGE_SIZE), write_len, buf, - group, 0, 0); + group, 0, 0, false); srv_stats.os_log_pending_writes.dec(); @@ -2011,7 +2011,7 @@ log_group_checkpoint( write_offset / UNIV_PAGE_SIZE, write_offset % UNIV_PAGE_SIZE, OS_FILE_LOG_BLOCK_SIZE, - buf, ((byte*) group + 1), 0, 0); + buf, ((byte*) group + 1), 0, 0, false); ut_ad(((ulint) group & 0x1UL) == 0); } @@ -2093,7 +2093,7 @@ log_group_read_checkpoint_info( fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->space_id, 0, field / UNIV_PAGE_SIZE, field % UNIV_PAGE_SIZE, - OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0, 0); + OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0, 0, false); } /******************************************************//** @@ -2417,7 +2417,7 @@ loop: fil_io(OS_FILE_READ | OS_FILE_LOG, sync, group->space_id, 0, (ulint) (source_offset / UNIV_PAGE_SIZE), (ulint) (source_offset % UNIV_PAGE_SIZE), - len, buf, NULL, 0, 0); + len, buf, NULL, 0, 0, false); if (recv_sys->recv_log_crypt_ver != UNENCRYPTED_KEY_VER && !log_group_decrypt_after_read(group, buf, len)) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index fbed6137cd7..adea9a0ce27 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -3101,7 +3101,7 @@ recv_recovery_from_checkpoint_start_func( fil_io(OS_FILE_READ | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, LOG_FILE_HDR_SIZE, - log_hdr_buf, max_cp_group, 0, 0); + log_hdr_buf, max_cp_group, 0, 0, false); if (0 == ut_memcmp(log_hdr_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, (byte*)"ibbackup", (sizeof "ibbackup") - 1)) { @@ -3132,7 +3132,7 @@ recv_recovery_from_checkpoint_start_func( fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, OS_FILE_LOG_BLOCK_SIZE, - log_hdr_buf, max_cp_group, 0, 0); + log_hdr_buf, max_cp_group, 0, 0, false); } #ifdef UNIV_LOG_ARCHIVE diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 0b623448e03..7495e0d570c 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -260,6 +260,9 @@ struct os_aio_slot_t{ ulint file_block_size;/*!< file block size */ + bool encrypt_later; /*!< should the page be encrypted + before write */ + #ifdef WIN_ASYNC_IO HANDLE handle; /*!< handle object we need in the OVERLAPPED struct */ @@ -4546,9 +4549,11 @@ os_aio_array_reserve_slot( level to be used */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key, /*!< page encryption key + ulint page_encryption_key, /*!< in: page encryption key to be used */ - lsn_t lsn) /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later) /*!< in: should we encrypt before + writing the page */ { os_aio_slot_t* slot = NULL; #ifdef WIN_ASYNC_IO @@ -4635,7 +4640,6 @@ found: slot->name = name; slot->len = len; slot->type = type; - slot->buf = static_cast<byte*>(buf); slot->offset = offset; slot->lsn = lsn; slot->io_already_done = FALSE; @@ -4646,6 +4650,7 @@ found: slot->page_compression = page_compression; slot->page_encryption_key = page_encryption_key; slot->page_encryption = page_encryption; + slot->encrypt_later = encrypt_later; if (message1) { slot->file_block_size = fil_node_get_block_size(message1); @@ -4700,7 +4705,8 @@ found: /* If the space is page encryption and this is write operation then we encrypt the page */ - if (message1 && type == OS_FILE_WRITE && page_encryption ) { + if (message1 && type == OS_FILE_WRITE && (page_encryption == 1 || encrypt_later)) { + ut_a(page_encryption == 1 || srv_encrypt_tables == 1); /* Release the array mutex while encrypting */ os_mutex_exit(array->mutex); @@ -4724,6 +4730,8 @@ found: os_mutex_enter(array->mutex); } + slot->buf = static_cast<byte*>(buf); + #ifdef WIN_ASYNC_IO control = &slot->control; control->Offset = (DWORD) offset & 0xFFFFFFFF; @@ -5012,9 +5020,11 @@ os_aio_func( level to be used */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key, /*!< page encryption key + ulint page_encryption_key, /*!< in: page encryption key to be used */ - lsn_t lsn) /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later) /*!< in: should we encrypt page + before write */ { os_aio_array_t* array; os_aio_slot_t* slot; @@ -5131,7 +5141,7 @@ try_again: slot = os_aio_array_reserve_slot(type, array, message1, message2, file, name, buf, offset, n, write_size, page_compression, page_compression_level, - page_encryption, page_encryption_key, lsn); + page_encryption, page_encryption_key, lsn, encrypt_later); if (type == OS_FILE_READ) { if (srv_use_native_aio) { @@ -5169,6 +5179,7 @@ try_again: buffer = buf; } } + ret = WriteFile(file, buffer, (DWORD) n, &len, &(slot->control)); diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 37b1c65dfe1..71008455f9b 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -1084,6 +1084,7 @@ buf_block_init( block->page.comp_buf = NULL; block->page.comp_buf_free = NULL; block->page.key_version = 0; + block->page.encrypt_later = false; block->modify_clock = 0; @@ -3587,6 +3588,7 @@ buf_page_init_low( bpage->comp_buf = NULL; bpage->comp_buf_free = NULL; bpage->key_version = 0; + bpage->encrypt_later = false; HASH_INVALIDATE(bpage, hash); bpage->is_corrupt = FALSE; @@ -5793,6 +5795,8 @@ buf_page_encrypt_before_write( buf_page_t* bpage, /*!< in/out: buffer page to be flushed */ const byte* src_frame) /*!< in: src frame */ { + bpage->encrypt_later = false; + if (srv_encrypt_tables == FALSE) { /* Encryption is disabled */ return const_cast<byte*>(src_frame); @@ -5853,7 +5857,8 @@ buf_page_encrypt_before_write( ut_ad(key_version == 0 || key_version >= bpage->key_version); bpage->key_version = key_version; } else { - // We do compression and encryption later on os0file.cc + /** Compression and encryption is done later at os0file.cc */ + bpage->encrypt_later = true; dst_frame = (byte *)src_frame; } diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index 75e86ac4326..871f723549b 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -521,10 +521,18 @@ buf_dblwr_process() ulint zip_size = fil_space_get_zip_size(space_id); /* Read in the actual page from the file */ - fil_io(OS_FILE_READ, true, space_id, zip_size, - page_no, 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - read_buf, NULL, 0, 0); + fil_io(OS_FILE_READ, + true, + space_id, + zip_size, + page_no, + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + read_buf, + NULL, + 0, + 0, + false); if (fil_space_verify_crypt_checksum(read_buf, zip_size)) { /* page is encrypted and checksum is OK */ @@ -576,10 +584,18 @@ buf_dblwr_process() doublewrite buffer to the intended position */ - fil_io(OS_FILE_WRITE, true, space_id, - zip_size, page_no, 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - page, NULL, 0, 0); + fil_io(OS_FILE_WRITE, + true, + space_id, + zip_size, + page_no, + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + page, + NULL, + 0, + 0, + false); ib_logf(IB_LOG_LEVEL_INFO, "Recovered the page from" @@ -595,11 +611,17 @@ buf_dblwr_process() zeroes, while a valid copy is available in dblwr buffer. */ - fil_io(OS_FILE_WRITE, true, space_id, - zip_size, page_no, 0, - zip_size ? zip_size - : UNIV_PAGE_SIZE, - page, NULL, 0, 0); + fil_io(OS_FILE_WRITE, + true, + space_id, + zip_size, + page_no, 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + page, + NULL, + 0, + 0, + false); } } } @@ -621,9 +643,9 @@ buf_dblwr_process() memset(buf, 0, bytes); fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block1, 0, bytes, buf, NULL, NULL, 0); + buf_dblwr->block1, 0, bytes, buf, NULL, NULL, 0, false); fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block2, 0, bytes, buf, NULL, NULL, 0); + buf_dblwr->block2, 0, bytes, buf, NULL, NULL, 0, false); ut_free(unaligned_buf); } @@ -828,13 +850,18 @@ buf_dblwr_write_block_to_datafile( void * frame = buf_page_get_frame(bpage); if (bpage->zip.data) { - fil_io(flags, sync, buf_page_get_space(bpage), - buf_page_get_zip_size(bpage), - buf_page_get_page_no(bpage), 0, - buf_page_get_zip_size(bpage), - frame, - (void*) bpage, 0, - bpage->newest_modification); + fil_io(flags, + sync, + buf_page_get_space(bpage), + buf_page_get_zip_size(bpage), + buf_page_get_page_no(bpage), + 0, + buf_page_get_zip_size(bpage), + frame, + (void*) bpage, + 0, + bpage->newest_modification, + bpage->encrypt_later); return; } @@ -844,10 +871,18 @@ buf_dblwr_write_block_to_datafile( ut_a(buf_block_get_state(block) == BUF_BLOCK_FILE_PAGE); buf_dblwr_check_page_lsn(block->frame); - fil_io(flags, sync, buf_block_get_space(block), 0, - buf_block_get_page_no(block), 0, UNIV_PAGE_SIZE, - frame, (void*) block, - (ulint *)&bpage->write_size, bpage->newest_modification); + fil_io(flags, + sync, + buf_block_get_space(block), + 0, + buf_block_get_page_no(block), + 0, + UNIV_PAGE_SIZE, + frame, + (void*) block, + (ulint *)&bpage->write_size, + bpage->newest_modification, + bpage->encrypt_later); } /********************************************************************//** @@ -939,9 +974,19 @@ try_again: len = ut_min(TRX_SYS_DOUBLEWRITE_BLOCK_SIZE, buf_dblwr->first_free) * UNIV_PAGE_SIZE; - fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block1, 0, len, - (void*) write_buf, NULL, 0, 0); + fil_io(OS_FILE_WRITE, + true, + TRX_SYS_SPACE, + 0, + buf_dblwr->block1, + 0, + len, + (void*) + write_buf, + NULL, + 0, + 0, + false); if (buf_dblwr->first_free <= TRX_SYS_DOUBLEWRITE_BLOCK_SIZE) { /* No unwritten pages in the second block. */ @@ -955,9 +1000,18 @@ try_again: write_buf = buf_dblwr->write_buf + TRX_SYS_DOUBLEWRITE_BLOCK_SIZE * UNIV_PAGE_SIZE; - fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - buf_dblwr->block2, 0, len, - (void*) write_buf, NULL, 0, 0); + fil_io(OS_FILE_WRITE, + true, + TRX_SYS_SPACE, + 0, + buf_dblwr->block2, + 0, + len, + (void*) write_buf, + NULL, + 0, + 0, + false); flush: /* increment the doublewrite flushed pages counter */ @@ -1187,17 +1241,32 @@ retry: memset(buf_dblwr->write_buf + UNIV_PAGE_SIZE * i + zip_size, 0, UNIV_PAGE_SIZE - zip_size); - fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - offset, 0, UNIV_PAGE_SIZE, - (void*) (buf_dblwr->write_buf - + UNIV_PAGE_SIZE * i), NULL, 0,bpage->newest_modification); + fil_io(OS_FILE_WRITE, + true, + TRX_SYS_SPACE, + 0, + offset, + 0, + UNIV_PAGE_SIZE, + (void*) (buf_dblwr->write_buf + UNIV_PAGE_SIZE * i), + NULL, + 0, + bpage->newest_modification, + bpage->encrypt_later); } else { /* It is a regular page. Write it directly to the doublewrite buffer */ - fil_io(OS_FILE_WRITE, true, TRX_SYS_SPACE, 0, - offset, 0, UNIV_PAGE_SIZE, - frame, - NULL, 0, bpage->newest_modification); + fil_io(OS_FILE_WRITE, + true, + TRX_SYS_SPACE, + 0, + offset, + 0, UNIV_PAGE_SIZE, + frame, + NULL, + 0, + bpage->newest_modification, + bpage->encrypt_later); } /* Now flush the doublewrite buffer data to disk */ diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index c7356015788..3411dc27d2c 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -955,16 +955,17 @@ buf_flush_write_block_low( if (!srv_use_doublewrite_buf || !buf_dblwr) { fil_io(OS_FILE_WRITE | OS_AIO_SIMULATED_WAKE_LATER, - sync, - buf_page_get_space(bpage), - zip_size, - buf_page_get_page_no(bpage), - 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - frame, - bpage, - &bpage->write_size, - bpage->newest_modification); + sync, + buf_page_get_space(bpage), + zip_size, + buf_page_get_page_no(bpage), + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + frame, + bpage, + &bpage->write_size, + bpage->newest_modification, + bpage->encrypt_later); } else { /* InnoDB uses doublewrite buffer and doublewrite buffer is initialized. User can define do we use atomic writes @@ -975,16 +976,17 @@ buf_flush_write_block_low( if (awrites == ATOMIC_WRITES_ON) { fil_io(OS_FILE_WRITE | OS_AIO_SIMULATED_WAKE_LATER, - FALSE, - buf_page_get_space(bpage), - zip_size, - buf_page_get_page_no(bpage), - 0, - zip_size ? zip_size : UNIV_PAGE_SIZE, - frame, - bpage, - &bpage->write_size, - bpage->newest_modification); + FALSE, + buf_page_get_space(bpage), + zip_size, + buf_page_get_page_no(bpage), + 0, + zip_size ? zip_size : UNIV_PAGE_SIZE, + frame, + bpage, + &bpage->write_size, + bpage->newest_modification, + bpage->encrypt_later); } else if (flush_type == BUF_FLUSH_SINGLE_PAGE) { buf_dblwr_write_single_page(bpage, sync); } else { diff --git a/storage/xtradb/buf/buf0rea.cc b/storage/xtradb/buf/buf0rea.cc index 271fcdc12aa..afa14f0df04 100644 --- a/storage/xtradb/buf/buf0rea.cc +++ b/storage/xtradb/buf/buf0rea.cc @@ -232,14 +232,14 @@ not_to_recover: *err = _fil_io(OS_FILE_READ | wake_later | ignore_nonexistent_pages, sync, space, zip_size, offset, 0, zip_size, - frame, bpage, 0, trx, 0); + frame, bpage, 0, trx, 0, false); } else { ut_a(buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); *err = _fil_io(OS_FILE_READ | wake_later | ignore_nonexistent_pages, sync, space, 0, offset, 0, UNIV_PAGE_SIZE, - frame, bpage, &bpage->write_size, trx, 0); + frame, bpage, &bpage->write_size, trx, 0, false); } if (sync) { diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 4719a37fab2..d7b26eed49f 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -303,9 +303,8 @@ fil_crypt_get_latest_key(byte *dst, uint* key_length, } return fil_crypt_get_key(dst, key_length, crypt_data, *version, false); - } else { - return fil_crypt_get_key(dst, key_length, NULL, *version, true); } + return fil_crypt_get_key(dst, key_length, NULL, *version, true); } /****************************************************************** @@ -622,13 +621,25 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, byte key[MY_AES_MAX_KEY_LENGTH]; uint key_length; + ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); + if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR + || orig_page_type==FIL_PAGE_TYPE_XDES + || orig_page_type== FIL_PAGE_PAGE_ENCRYPTED + || orig_page_type== FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + //TODO: is this really needed ? + memcpy(dst_frame, src_frame, page_size); + return; + } + if (srv_encrypt_tables) { crypt_data = fil_space_get_crypt_data(space); + if (crypt_data == NULL) { //TODO: Is this really needed ? memcpy(dst_frame, src_frame, page_size); return; } + fil_crypt_get_latest_key(key, &key_length, crypt_data, &key_version); } else { key_version = encryption_key; @@ -647,9 +658,7 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, src_frame + FIL_PAGE_OFFSET); mach_write_to_4(iv + 4, space_offset); mach_write_to_8(iv + 8, lsn); - } - else - { + } else { // take the iv from the key provider int load_iv_rc = get_encryption_iv(key_version, (uchar *) iv, sizeof(iv)); @@ -667,19 +676,9 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, } } - ibool page_compressed = (mach_read_from_2(src_frame+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED); ibool page_encrypted = fil_space_is_page_encrypted(space); ulint compression_alg = mach_read_from_8(src_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); - - if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR - || orig_page_type==FIL_PAGE_TYPE_XDES - || orig_page_type== FIL_PAGE_PAGE_ENCRYPTED - || orig_page_type== FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - memcpy(dst_frame, src_frame, page_size); - return; - } // copy page header memcpy(dst_frame, src_frame, FIL_PAGE_DATA); @@ -708,7 +707,7 @@ fil_space_encrypt(ulint space, ulint offset, lsn_t lsn, uint32 dstlen; if (page_compressed) { - srclen = page_size - FIL_PAGE_DATA; + srclen = page_size - FIL_PAGE_DATA;; } int rc = (* my_aes_encrypt_dynamic)(src, srclen, @@ -789,6 +788,7 @@ fil_space_check_encryption_read( ulint space) /*!< in: tablespace id */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); + if (crypt_data == NULL) return false; @@ -815,6 +815,7 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, || page_type == FIL_PAGE_PAGE_COMPRESSED); ulint orig_page_type=0; + if (page_type == FIL_PAGE_PAGE_ENCRYPTED) { key_version = mach_read_from_2( src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); @@ -846,7 +847,6 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, mach_write_to_2(dst_frame+FIL_PAGE_TYPE, orig_page_type); } - // get key byte key[MY_AES_MAX_KEY_LENGTH]; uint key_length; @@ -862,9 +862,7 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, mach_write_to_4(iv + 0, space); mach_write_to_4(iv + 4, offset); mach_write_to_8(iv + 8, lsn); - } - else - { + } else { // take the iv from the key provider int load_iv_rc = get_encryption_iv(key_version, (uchar *) iv, sizeof(iv)); @@ -886,7 +884,6 @@ fil_space_decrypt(fil_space_crypt_t* crypt_data, byte* dst = dst_frame + FIL_PAGE_DATA; uint32 dstlen; ulint srclen = page_size - (FIL_PAGE_DATA + FIL_PAGE_DATA_END); - ulint compressed_len; ulint compression_method; diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 530326c0f66..8a01dc7d7e6 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -287,7 +287,7 @@ fil_read( actual page size does not decrease. */ { return(fil_io(OS_FILE_READ, sync, space_id, zip_size, block_offset, - byte_offset, len, buf, message, write_size, 0)); + byte_offset, len, buf, message, write_size, 0, false)); } /********************************************************************//** @@ -319,12 +319,13 @@ fil_write( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn) /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later) /*!< in: encrypt later ? */ { ut_ad(!srv_read_only_mode); return(fil_io(OS_FILE_WRITE, sync, space_id, zip_size, block_offset, - byte_offset, len, buf, message, write_size, lsn)); + byte_offset, len, buf, message, write_size, lsn, encrypt_later)); } /*******************************************************************//** @@ -617,6 +618,8 @@ fil_node_open_file( ut_a(size_bytes != (os_offset_t) -1); node->file_block_size = os_file_get_block_size(node->handle, node->name); + space->file_block_size = node->file_block_size; + #ifdef UNIV_HOTBACKUP if (space->id == 0) { node->size = (ulint) (size_bytes / UNIV_PAGE_SIZE); @@ -782,6 +785,7 @@ add_size: if (node->file_block_size == 0) { node->file_block_size = os_file_get_block_size(node->handle, node->name); + space->file_block_size = node->file_block_size; } ut_a(ret); @@ -1874,7 +1878,7 @@ fil_write_lsn_and_arch_no_to_file( lsn); err = fil_write(TRUE, space, 0, sum_of_sizes, 0, - UNIV_PAGE_SIZE, buf, NULL, 0, 0); + UNIV_PAGE_SIZE, buf, NULL, 0, 0, false); } mem_free(buf1); @@ -5189,6 +5193,7 @@ retry: /* Determine correct file block size */ if (node->file_block_size == 0) { node->file_block_size = os_file_get_block_size(node->handle, node->name); + space->file_block_size = node->file_block_size; } #ifdef HAVE_POSIX_FALLOCATE @@ -5248,7 +5253,7 @@ retry: success = os_aio(OS_FILE_WRITE, OS_AIO_SYNC, node->name, node->handle, buf, offset, page_size * n_pages, - node, NULL, space_id, NULL, 0, 0, 0, 0, 0, 0); + node, NULL, space_id, NULL, 0, 0, 0, 0, 0, 0, false); #endif /* UNIV_HOTBACKUP */ if (success) { os_has_said_disk_full = FALSE; @@ -5630,8 +5635,8 @@ _fil_io( initialized we do not trim again if actual page size does not decrease. */ trx_t* trx, - lsn_t lsn) /* lsn of the newest modification */ - + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later) /*!< in: encrypt later ? */ { ulint mode; fil_space_t* space; @@ -5880,7 +5885,8 @@ _fil_io( write_size, page_encrypted, page_encryption_key, - lsn); + lsn, + encrypt_later); #else /* In mysqlbackup do normal i/o, not aio */ diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc index 2223701f444..987493c8d26 100644 --- a/storage/xtradb/fil/fil0pagecompress.cc +++ b/storage/xtradb/fil/fil0pagecompress.cc @@ -283,7 +283,8 @@ fil_compress_page( /* Let's not compress file space header or extent descriptor */ - if (orig_page_type == FIL_PAGE_TYPE_FSP_HDR || + if (orig_page_type == 0 || + orig_page_type == FIL_PAGE_TYPE_FSP_HDR || orig_page_type == FIL_PAGE_TYPE_XDES || orig_page_type == FIL_PAGE_PAGE_COMPRESSED || orig_page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { diff --git a/storage/xtradb/include/buf0buf.h b/storage/xtradb/include/buf0buf.h index a2097b5c3b9..7eb58216698 100644 --- a/storage/xtradb/include/buf0buf.h +++ b/storage/xtradb/include/buf0buf.h @@ -1598,6 +1598,8 @@ struct buf_page_t{ can be read while it's being flushed */ byte* comp_buf_free; /*!< for compression, allocated buffer that is then alligned */ + bool encrypt_later; /*!< should we encrypt the page + at os0file.cc ? */ #ifndef UNIV_HOTBACKUP buf_page_t* hash; /*!< node used in chaining to buf_pool->page_hash or diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 360a26df776..1847a314d06 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2015, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -340,6 +340,7 @@ struct fil_space_t { UT_LIST_NODE_T(fil_space_t) space_list; /*!< list of all spaces */ fil_space_crypt_t* crypt_data; + ulint file_block_size;/*!< file system block size */ ulint magic_n;/*!< FIL_SPACE_MAGIC_N */ }; @@ -959,8 +960,8 @@ fil_space_get_n_reserved_extents( Reads or writes data. This operation is asynchronous (aio). @return DB_SUCCESS, or DB_TABLESPACE_DELETED if we are trying to do i/o on a tablespace which does not exist */ -#define fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size, lsn) \ - _fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size, NULL, lsn) +#define fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size, lsn, encrypt) \ + _fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size, NULL, lsn, encrypt) UNIV_INTERN dberr_t @@ -997,7 +998,8 @@ _fil_io( initialized we do not trim again if actual page size does not decrease. */ trx_t* trx, - lsn_t lsn) /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later) /*!< in: encrypt later ? */ __attribute__((nonnull(8))); /**********************************************************************//** diff --git a/storage/xtradb/include/os0file.h b/storage/xtradb/include/os0file.h index af3ad734f48..f93bf026903 100644 --- a/storage/xtradb/include/os0file.h +++ b/storage/xtradb/include/os0file.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2015, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -324,11 +324,11 @@ The wrapper functions have the prefix of "innodb_". */ # define os_aio(type, mode, name, file, buf, offset, \ n, message1, message2, space_id, \ trx, page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key, lsn) \ + page_encryption, page_encryption_key, lsn, encrypt) \ pfs_os_aio_func(type, mode, name, file, buf, offset, \ n, message1, message2, space_id, trx, \ page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key, lsn, \ + page_encryption, page_encryption_key, lsn, encrypt, \ __FILE__, __LINE__) # define os_file_read(file, buf, offset, n, compressed) \ @@ -378,11 +378,11 @@ to original un-instrumented file I/O APIs */ # define os_aio(type, mode, name, file, buf, offset, n, message1, \ message2, space_id, trx, \ page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key, lsn) \ + page_encryption, page_encryption_key, lsn, encrypt) \ os_aio_func(type, mode, name, file, buf, offset, n, \ message1, message2, space_id, trx, \ page_compressed, page_compression_level, write_size, \ - page_encryption, page_encryption_key, lsn) + page_encryption, page_encryption_key, lsn, encrypt) # define os_file_read(file, buf, offset, n, compressed) \ os_file_read_func(file, buf, offset, n, NULL, compressed) @@ -812,9 +812,10 @@ pfs_os_aio_func( actual page size does not decrease. */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key, /*!< page encryption + ulint page_encryption_key, /*!< in: page encryption key to be used */ - lsn_t lsn, /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later, /*!< in: should we encrypt ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line);/*!< in: line where the func invoked */ /*******************************************************************//** @@ -1204,10 +1205,10 @@ os_aio_func( actual page size does not decrease. */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key, /*!< page encryption key + ulint page_encryption_key, /*!< in: page encryption key to be used */ - lsn_t lsn); /* lsn of the newest modification */ - + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later); /*!< in: should we encrypt ? */ /************************************************************************//** Wakes up all async i/o threads so that they know to exit themselves in shutdown. */ diff --git a/storage/xtradb/include/os0file.ic b/storage/xtradb/include/os0file.ic index 9ab191bd460..39599ba2047 100644 --- a/storage/xtradb/include/os0file.ic +++ b/storage/xtradb/include/os0file.ic @@ -222,7 +222,7 @@ pfs_os_aio_func( trx_t* trx, ibool page_compression, /*!< in: is page compression used on this file space */ - ulint page_compression_level, /*!< page compression + ulint page_compression_level, /*!< in: page compression level to be used */ ulint* write_size,/*!< in/out: Actual write size initialized after fist successfull trim @@ -231,10 +231,12 @@ pfs_os_aio_func( actual page size does not decrease. */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key, /*!< page encryption + ulint page_encryption_key, /*!< in: page encryption key to be used */ - lsn_t lsn, /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest + modification */ + bool encrypt_later, /*!< in: encrypt later ? */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line)/*!< in: line where the func invoked */ { @@ -252,7 +254,7 @@ pfs_os_aio_func( result = os_aio_func(type, mode, name, file, buf, offset, n, message1, message2, space_id, trx, page_compression, page_compression_level, write_size , - page_encryption, page_encryption_key, lsn); + page_encryption, page_encryption_key, lsn, encrypt_later); register_pfs_file_io_end(locker, n); diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index e007bcc3507..37cf18c1a0d 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -1384,7 +1384,7 @@ log_group_file_header_flush( (ulint) (dest_offset / UNIV_PAGE_SIZE), (ulint) (dest_offset % UNIV_PAGE_SIZE), OS_FILE_LOG_BLOCK_SIZE, - buf, group, 0, 0); + buf, group, 0, 0, false); srv_stats.os_log_pending_writes.dec(); } @@ -1551,7 +1551,7 @@ loop: fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, group->space_id, 0, (ulint) (next_offset / UNIV_PAGE_SIZE), (ulint) (next_offset % UNIV_PAGE_SIZE), write_len, buf, - group, 0, 0); + group, 0, 0, false); srv_stats.os_log_pending_writes.dec(); @@ -2144,7 +2144,7 @@ log_group_checkpoint( write_offset / UNIV_PAGE_SIZE, write_offset % UNIV_PAGE_SIZE, OS_FILE_LOG_BLOCK_SIZE, - buf, ((byte*) group + 1), 0, 0); + buf, ((byte*) group + 1), 0, 0, false); ut_ad(((ulint) group & 0x1UL) == 0); } @@ -2226,7 +2226,7 @@ log_group_read_checkpoint_info( fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->space_id, 0, field / UNIV_PAGE_SIZE, field % UNIV_PAGE_SIZE, - OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0, 0); + OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, 0, 0, false); } /******************************************************//** @@ -2639,7 +2639,7 @@ loop: fil_io(OS_FILE_READ | OS_FILE_LOG, sync, group->space_id, 0, (ulint) (source_offset / UNIV_PAGE_SIZE), (ulint) (source_offset % UNIV_PAGE_SIZE), - len, buf, (type == LOG_ARCHIVE) ? &log_archive_io : NULL, 0, 0); + len, buf, (type == LOG_ARCHIVE) ? &log_archive_io : NULL, 0, 0, false); if (recv_sys->recv_log_crypt_ver != UNENCRYPTED_KEY_VER && !log_group_decrypt_after_read(group, buf, len)) @@ -2771,7 +2771,7 @@ log_group_archive_file_header_write( dest_offset / UNIV_PAGE_SIZE, dest_offset % UNIV_PAGE_SIZE, 2 * OS_FILE_LOG_BLOCK_SIZE, - buf, &log_archive_io, 0, 0); + buf, &log_archive_io, 0, 0, false); } /******************************************************//** @@ -2808,7 +2808,7 @@ log_group_archive_completed_header_write( dest_offset % UNIV_PAGE_SIZE, OS_FILE_LOG_BLOCK_SIZE, buf + LOG_FILE_ARCH_COMPLETED, - &log_archive_io, 0, 0); + &log_archive_io, 0, 0, false); } /******************************************************//** @@ -2953,7 +2953,7 @@ loop: (ulint) (next_offset / UNIV_PAGE_SIZE), (ulint) (next_offset % UNIV_PAGE_SIZE), ut_calc_align(len, OS_FILE_LOG_BLOCK_SIZE), buf, - &log_archive_io, 0, 0); + &log_archive_io, 0, 0, false); start_lsn += len; next_offset += len; diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index bb0888d3db7..c7fb3be9625 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -2156,7 +2156,7 @@ recv_apply_log_recs_for_backup(void) error = fil_io(OS_FILE_READ, true, recv_addr->space, zip_size, recv_addr->page_no, 0, zip_size, - block->page.zip.data, NULL, 0, 0); + block->page.zip.data, NULL, 0, 0, false); if (error == DB_SUCCESS && !buf_zip_decompress(block, TRUE)) { exit(1); @@ -2166,7 +2166,7 @@ recv_apply_log_recs_for_backup(void) recv_addr->space, 0, recv_addr->page_no, 0, UNIV_PAGE_SIZE, - block->frame, NULL, 0, 0); + block->frame, NULL, 0, 0, false); } if (error != DB_SUCCESS) { @@ -2195,13 +2195,15 @@ recv_apply_log_recs_for_backup(void) recv_addr->space, zip_size, recv_addr->page_no, 0, zip_size, - block->page.zip.data, NULL, 0, 0); + block->page.zip.data, NULL, 0, 0, false); } else { error = fil_io(OS_FILE_WRITE, true, recv_addr->space, 0, recv_addr->page_no, 0, UNIV_PAGE_SIZE, - block->frame, NULL, 0, 0); + block->frame, NULL, 0, + block->latest_modification, + block->encrypt_later); } skip_this_recv_addr: recv_addr = HASH_GET_NEXT(addr_hash, recv_addr); @@ -3175,7 +3177,7 @@ recv_recovery_from_checkpoint_start_func( fil_io(OS_FILE_READ | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, LOG_FILE_HDR_SIZE, - log_hdr_buf, max_cp_group, 0, 0); + log_hdr_buf, max_cp_group, 0, 0, false); if (0 == ut_memcmp(log_hdr_buf + LOG_FILE_WAS_CREATED_BY_HOT_BACKUP, (byte*)"ibbackup", (sizeof "ibbackup") - 1)) { @@ -3206,7 +3208,7 @@ recv_recovery_from_checkpoint_start_func( fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, max_cp_group->space_id, 0, 0, 0, OS_FILE_LOG_BLOCK_SIZE, - log_hdr_buf, max_cp_group, 0, 0); + log_hdr_buf, max_cp_group, 0, 0, false); } log_hdr_log_block_size @@ -3869,7 +3871,7 @@ ask_again: /* Read the archive file header */ fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->archive_space_id, 0, 0, 0, - LOG_FILE_HDR_SIZE, buf, NULL, 0, 0); + LOG_FILE_HDR_SIZE, buf, NULL, 0, 0, false); /* Check if the archive file header is consistent */ @@ -3943,7 +3945,7 @@ ask_again: fil_io(OS_FILE_READ | OS_FILE_LOG, true, group->archive_space_id, 0, read_offset / UNIV_PAGE_SIZE, - read_offset % UNIV_PAGE_SIZE, len, buf, NULL, 0, 0); + read_offset % UNIV_PAGE_SIZE, len, buf, NULL, 0, 0, false); ret = recv_scan_log_recs( (buf_pool_get_n_pages() diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 05ab1490b98..a737ebb1fa0 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -273,6 +273,7 @@ struct os_aio_slot_t{ lsn_t lsn; /* lsn of the newest modification */ ulint file_block_size;/*!< file block size */ + bool encrypt_later; /*!< should we encrypt the page */ #ifdef LINUX_NATIVE_AIO struct iocb control; /* Linux control block for aio */ @@ -1982,6 +1983,9 @@ os_file_create_func( #else if (purpose == OS_FILE_AIO) { + bool encrypt_later; /*!< should the page be encrypted + before write */ + #ifdef WIN_ASYNC_IO /* If specified, use asynchronous (overlapped) io and no buffering of writes in the OS */ @@ -4663,7 +4667,9 @@ os_aio_array_reserve_slot( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ - lsn_t lsn) /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest + modification */ + bool encrypt_later) /*!< in: should we encrypt the page */ { os_aio_slot_t* slot = NULL; #ifdef WIN_ASYNC_IO @@ -4755,15 +4761,14 @@ found: slot->lsn = lsn; slot->io_already_done = FALSE; slot->space_id = space_id; - slot->page_compression_success = FALSE; slot->page_encryption_success = FALSE; - slot->write_size = write_size; slot->page_compression_level = page_compression_level; slot->page_compression = page_compression; slot->page_encryption_key = page_encryption_key; slot->page_encryption = page_encryption; + slot->encrypt_later = encrypt_later; if (message1) { slot->file_block_size = fil_node_get_block_size(message1); @@ -4819,7 +4824,7 @@ found: /* If the space is page encryption and this is write operation then we encrypt the page */ - if (message1 && type == OS_FILE_WRITE && page_encryption ) { + if (message1 && type == OS_FILE_WRITE && (page_encryption || encrypt_later)) { /* Release the array mutex while encrypting */ os_mutex_exit(array->mutex); @@ -4843,6 +4848,8 @@ found: os_mutex_enter(array->mutex); } + slot->buf = (byte *)buf; + #ifdef WIN_ASYNC_IO control = &slot->control; control->Offset = (DWORD) offset & 0xFFFFFFFF; @@ -5130,9 +5137,11 @@ os_aio_func( actual page size does not decrease. */ ibool page_encryption, /*!< in: is page encryption used on this file space */ - ulint page_encryption_key, /*!< page encryption key + ulint page_encryption_key, /*!< in: page encryption key to be used */ - lsn_t lsn) /* lsn of the newest modification */ + lsn_t lsn, /*!< in: lsn of the newest modification */ + bool encrypt_later) /*!< in: should we encrypt before + writing the page */ { os_aio_array_t* array; os_aio_slot_t* slot; @@ -5239,7 +5248,8 @@ try_again: slot = os_aio_array_reserve_slot(type, array, message1, message2, file, name, buf, offset, n, space_id, page_compression, page_compression_level, - page_encryption, page_encryption_key, write_size, lsn); + page_encryption, page_encryption_key, + write_size, lsn, encrypt_later); if (type == OS_FILE_READ) { if (srv_use_native_aio) { |