diff options
author | Marko Mäkelä <marko.makela@mariadb.com> | 2018-12-13 17:57:10 +0200 |
---|---|---|
committer | Marko Mäkelä <marko.makela@mariadb.com> | 2018-12-13 17:57:18 +0200 |
commit | 1a780eefc9ff8050b44bca07c981bd6a42bdbaf6 (patch) | |
tree | 1fceeee919d3d71a244b60f9c1a644a91b84de38 | |
parent | 2e5aea4bab65544a44c7983759f80c5823780f76 (diff) | |
download | mariadb-git-1a780eefc9ff8050b44bca07c981bd6a42bdbaf6.tar.gz |
MDEV-17958 Make bug-endian innodb_checksum_algorithm=crc32 optional
In MySQL 5.7, it was noticed that files are not portable between
big-endian and little-endian processor architectures
(such as SPARC and x86), because the original implementation of
innodb_checksum_algorithm=crc32 was not byte order agnostic.
A byte order agnostic implementation of innodb_checksum_algorithm=crc32
was only added to MySQL 5.7, not backported to 5.6. Consequently,
MariaDB Server versions 10.0 and 10.1 only contain the CRC-32C
implementation that works incorrectly on big-endian architectures,
and MariaDB Server 10.2.2 got the byte-order agnostic CRC-32C
implementation from MySQL 5.7.
MySQL 5.7 introduced a "legacy crc32" variant that is functionally
equivalent to the big-endian version of the original crc32 implementation.
Thanks to this variant, old data files can be transferred from big-endian
systems to newer versions.
Introducing new variants of checksum algorithms (without introducing
new names for them, or something on the pages themselves to identify
the algorithm) generally is a bad idea, because each checksum algorithm
is like a lottery ticket. The more algorithms you try, the more likely
it will be for the checksum to match on a corrupted page.
So, essentially MySQL 5.7 weakened innodb_checksum_algorithm=crc32,
and MariaDB 10.2.2 inherited this weakening.
We introduce a build option that together with MDEV-17957
makes innodb_checksum_algorithm=strict_crc32 strict again
by only allowing one variant of the checksum to match.
WITH_INNODB_BUG_ENDIAN_CRC32: A new cmake option for enabling the
bug-compatible "legacy crc32" checksum. This is only enabled on
big-endian systems by default, to facilitate an upgrade from
MariaDB 10.0 or 10.1. Checked by #ifdef INNODB_BUG_ENDIAN_CRC32.
ut_crc32_byte_by_byte: Remove (unused function).
legacy_big_endian_checksum: Remove. This variable seems to have
unnecessarily complicated the logic. When the weakening is enabled,
we must always fall back to the buggy checksum.
buf_page_check_crc32(): A helper function to compute one or
two CRC-32C variants.
-rw-r--r-- | storage/innobase/buf/buf0buf.cc | 138 | ||||
-rw-r--r-- | storage/innobase/buf/buf0checksum.cc | 82 | ||||
-rw-r--r-- | storage/innobase/fil/fil0crypt.cc | 17 | ||||
-rw-r--r-- | storage/innobase/include/buf0buf.h | 4 | ||||
-rw-r--r-- | storage/innobase/include/buf0checksum.h | 32 | ||||
-rw-r--r-- | storage/innobase/include/page0zip.h | 11 | ||||
-rw-r--r-- | storage/innobase/include/ut0crc32.h | 8 | ||||
-rw-r--r-- | storage/innobase/innodb.cmake | 7 | ||||
-rw-r--r-- | storage/innobase/page/page0zip.cc | 101 | ||||
-rw-r--r-- | storage/innobase/ut/ut0crc32.cc | 64 |
10 files changed, 211 insertions, 253 deletions
diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index b89c0412a39..747899313c1 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -756,17 +756,14 @@ buf_page_is_zeroes( @param[in] read_buf database page @param[in] checksum_field1 new checksum field @param[in] checksum_field2 old checksum field -@param[in] use_legacy_big_endian use legacy big endian algorithm @return true if the page is in crc32 checksum format. */ bool buf_page_is_checksum_valid_crc32( const byte* read_buf, ulint checksum_field1, - ulint checksum_field2, - bool use_legacy_big_endian) + ulint checksum_field2) { - const uint32_t crc32 = buf_calc_page_crc32(read_buf, - use_legacy_big_endian); + const uint32_t crc32 = buf_calc_page_crc32(read_buf); #ifdef UNIV_INNOCHECKSUM if (log_file @@ -783,17 +780,11 @@ buf_page_is_checksum_valid_crc32( return false; } - if (checksum_field1 == crc32) { - return(true); - } else { - const uint32_t crc32_legacy = buf_calc_page_crc32(read_buf, true); - - if (checksum_field1 == crc32_legacy) { - return(true); - } - } - - return(false); + return checksum_field1 == crc32 +#ifdef INNODB_BUG_ENDIAN_CRC32 + || checksum_field1 == buf_calc_page_crc32(read_buf, true) +#endif + ; } /** Checks if the page is in innodb checksum format. @@ -922,6 +913,29 @@ buf_page_is_checksum_valid_none( && checksum_field1 == BUF_NO_CHECKSUM_MAGIC); } +#ifdef INNODB_BUG_ENDIAN_CRC32 +/** Validate the CRC-32C checksum of a page. +@param[in] page buffer page (srv_page_size bytes) +@param[in] checksum CRC-32C checksum stored on page +@return computed checksum */ +static uint32_t buf_page_check_crc32(const byte* page, uint32_t checksum) +{ + uint32_t crc32 = buf_calc_page_crc32(page); + + if (checksum != crc32) { + crc32 = buf_calc_page_crc32(page, true); + } + + return crc32; +} +#else /* INNODB_BUG_ENDIAN_CRC32 */ +/** Validate the CRC-32C checksum of a page. +@param[in] page buffer page (srv_page_size bytes) +@param[in] checksum CRC-32C checksum stored on page +@return computed checksum */ +# define buf_page_check_crc32(page, checksum) buf_calc_page_crc32(page) +#endif /* INNODB_BUG_ENDIAN_CRC32 */ + /** Check if a page is corrupt. @param[in] check_lsn whether the LSN should be checked @param[in] read_buf database page @@ -1037,26 +1051,20 @@ buf_page_is_corrupted( && *reinterpret_cast<const ib_uint64_t*>( read_buf + FIL_PAGE_LSN) == 0) { - ulint i; - /* make sure that the page is really empty */ - for (ulint i = 0; i < UNIV_PAGE_SIZE; i++) { + for (ulint i = 0; i < page_size.logical(); i++) { if (read_buf[i] != 0) { return(true); } } #ifdef UNIV_INNOCHECKSUM - if (i >= page_size.logical()) { - if (log_file) { - fprintf(log_file, "Page::%llu" - " is empty and uncorrupted\n", - cur_page_num); - } - return(false); + if (log_file) { + fprintf(log_file, "Page::%llu" + " is empty and uncorrupted\n", + cur_page_num); } -#else - return(i < page_size.logical()); #endif /* UNIV_INNOCHECKSUM */ + return(false); } const srv_checksum_algorithm_t curr_algo = @@ -1065,10 +1073,7 @@ buf_page_is_corrupted( switch (curr_algo) { case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: return !buf_page_is_checksum_valid_crc32( - read_buf, checksum_field1, checksum_field2, false) - && !buf_page_is_checksum_valid_crc32( - read_buf, checksum_field1, checksum_field2, - true); + read_buf, checksum_field1, checksum_field2); case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: return !buf_page_is_checksum_valid_innodb( read_buf, checksum_field1, checksum_field2); @@ -1111,19 +1116,10 @@ buf_page_is_corrupted( if (srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_CRC32) { - - crc32 = buf_calc_page_crc32( - read_buf, legacy_big_endian_checksum); + crc32 = buf_page_check_crc32(read_buf, + checksum_field2); crc32_inited = true; - if (!legacy_big_endian_checksum - && checksum_field2 != crc32) { - crc32 = buf_calc_page_crc32(read_buf, - true); - legacy_big_endian_checksum = - checksum_field2 == crc32; - } - if (checksum_field2 != crc32 && checksum_field2 != buf_calc_page_old_checksum(read_buf)) { @@ -1135,19 +1131,10 @@ buf_page_is_corrupted( if (checksum_field2 != buf_calc_page_old_checksum(read_buf)) { - crc32 = buf_calc_page_crc32( - read_buf, - legacy_big_endian_checksum); + crc32 = buf_page_check_crc32( + read_buf, checksum_field2); crc32_inited = true; - if (!legacy_big_endian_checksum - && checksum_field2 != crc32) { - crc32 = buf_calc_page_crc32( - read_buf, true); - legacy_big_endian_checksum = - checksum_field2 == crc32; - } - if (checksum_field2 != crc32) { return true; } @@ -1161,18 +1148,9 @@ buf_page_is_corrupted( == SRV_CHECKSUM_ALGORITHM_CRC32) { if (!crc32_inited) { - crc32 = buf_calc_page_crc32( - read_buf, - legacy_big_endian_checksum); + crc32 = buf_page_check_crc32( + read_buf, checksum_field2); crc32_inited = true; - - if (!legacy_big_endian_checksum - && checksum_field2 != crc32) { - crc32 = buf_calc_page_crc32( - read_buf, true); - legacy_big_endian_checksum = - checksum_field2 == crc32; - } } if (checksum_field1 != crc32 @@ -1188,18 +1166,9 @@ buf_page_is_corrupted( != buf_calc_page_new_checksum(read_buf)) { if (!crc32_inited) { - crc32 = buf_calc_page_crc32( - read_buf, - legacy_big_endian_checksum); + crc32 = buf_page_check_crc32( + read_buf, checksum_field2); crc32_inited = true; - - if (!legacy_big_endian_checksum - && checksum_field2 != crc32) { - crc32 = buf_calc_page_crc32( - read_buf, true); - legacy_big_endian_checksum = - checksum_field2 == crc32; - } } if (checksum_field1 != crc32) { @@ -1255,10 +1224,12 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size) << page_zip_calc_checksum( read_buf, page_size.physical(), SRV_CHECKSUM_ALGORITHM_CRC32) +#ifdef INNODB_BUG_ENDIAN_CRC32 << "/" << page_zip_calc_checksum( read_buf, page_size.physical(), SRV_CHECKSUM_ALGORITHM_CRC32, true) +#endif << ", " << buf_checksum_algorithm_name( SRV_CHECKSUM_ALGORITHM_INNODB) @@ -1284,9 +1255,10 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size) } else { const uint32_t crc32 = buf_calc_page_crc32(read_buf); - +#ifdef INNODB_BUG_ENDIAN_CRC32 const uint32_t crc32_legacy = buf_calc_page_crc32(read_buf, true); +#endif /* INNODB_BUG_ENDIAN_CRC32 */ ulint page_type = fil_page_get_type(read_buf); ib::info() << "Uncompressed page, stored checksum in field1 " @@ -1295,7 +1267,10 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size) << ", calculated checksums for field1: " << buf_checksum_algorithm_name( SRV_CHECKSUM_ALGORITHM_CRC32) << " " - << crc32 << "/" << crc32_legacy + << crc32 +#ifdef INNODB_BUG_ENDIAN_CRC32 + << "/" << crc32_legacy +#endif << ", " << buf_checksum_algorithm_name( SRV_CHECKSUM_ALGORITHM_INNODB) << " " @@ -1312,7 +1287,10 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size) << ", calculated checksums for field2: " << buf_checksum_algorithm_name( SRV_CHECKSUM_ALGORITHM_CRC32) << " " - << crc32 << "/" << crc32_legacy + << crc32 +#ifdef INNODB_BUG_ENDIAN_CRC32 + << "/" << crc32_legacy +#endif << ", " << buf_checksum_algorithm_name( SRV_CHECKSUM_ALGORITHM_INNODB) << " " @@ -3950,10 +3928,12 @@ buf_zip_decompress( << ", crc32: " << page_zip_calc_checksum( frame, size, SRV_CHECKSUM_ALGORITHM_CRC32) +#ifdef INNODB_BUG_ENDIAN_CRC32 << "/" << page_zip_calc_checksum( frame, size, SRV_CHECKSUM_ALGORITHM_CRC32, true) +#endif << " innodb: " << page_zip_calc_checksum( frame, size, SRV_CHECKSUM_ALGORITHM_INNODB) diff --git a/storage/innobase/buf/buf0checksum.cc b/storage/innobase/buf/buf0checksum.cc index 78b49e49690..543331de150 100644 --- a/storage/innobase/buf/buf0checksum.cc +++ b/storage/innobase/buf/buf0checksum.cc @@ -39,44 +39,56 @@ ha_innodb.cc:12251: error: cannot convert 'srv_checksum_algorithm_t*' to 'long unsigned int*' in initialization */ ulong srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB; -/** set if we have found pages matching legacy big endian checksum */ -bool legacy_big_endian_checksum = false; -/** Calculates the CRC32 checksum of a page. The value is stored to the page +#ifdef INNODB_BUG_ENDIAN_CRC32 +/** Calculate the CRC32 checksum of a page. The value is stored to the page when it is written to a file and also checked for a match when reading from -the file. When reading we allow both normal CRC32 and CRC-legacy-big-endian -variants. Note that we must be careful to calculate the same value on 32-bit -and 64-bit architectures. -@param[in] page buffer page (UNIV_PAGE_SIZE bytes) -@param[in] use_legacy_big_endian if true then use big endian -byteorder when converting byte strings to integers -@return checksum */ -uint32_t -buf_calc_page_crc32( - const byte* page, - bool use_legacy_big_endian /* = false */) +the file. Note that we must be careful to calculate the same value on all +architectures. +@param[in] page buffer page (srv_page_size bytes) +@param[in] bug_endian whether to use big endian byteorder +when converting byte strings to integers, for bug-compatibility with +big-endian architecture running MySQL 5.6, MariaDB 10.0 or MariaDB 10.1 +@return CRC-32C */ +uint32_t buf_calc_page_crc32(const byte* page, bool bug_endian) { - /* Since the field FIL_PAGE_FILE_FLUSH_LSN, and in versions <= 4.1.x - FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, are written outside the buffer pool - to the first pages of data files, we have to skip them in the page - checksum calculation. - We must also skip the field FIL_PAGE_SPACE_OR_CHKSUM where the - checksum is stored, and also the last 8 bytes of page because - there we store the old formula checksum. */ - - ut_crc32_func_t crc32_func = use_legacy_big_endian - ? ut_crc32_legacy_big_endian - : ut_crc32; - - const uint32_t c1 = crc32_func( - page + FIL_PAGE_OFFSET, - FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION - FIL_PAGE_OFFSET); - - const uint32_t c2 = crc32_func( - page + FIL_PAGE_DATA, - UNIV_PAGE_SIZE - FIL_PAGE_DATA - FIL_PAGE_END_LSN_OLD_CHKSUM); - - return(c1 ^ c2); + return bug_endian + ? ut_crc32_legacy_big_endian( + page + FIL_PAGE_OFFSET, + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + - FIL_PAGE_OFFSET) + ^ ut_crc32_legacy_big_endian(page + FIL_PAGE_DATA, + srv_page_size + - (FIL_PAGE_DATA + + FIL_PAGE_END_LSN_OLD_CHKSUM)) + : ut_crc32(page + FIL_PAGE_OFFSET, + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + - FIL_PAGE_OFFSET) + ^ ut_crc32(page + FIL_PAGE_DATA, + srv_page_size + - (FIL_PAGE_DATA + FIL_PAGE_END_LSN_OLD_CHKSUM)); +} +#else +/** Calculate the CRC32 checksum of a page. The value is stored to the page +when it is written to a file and also checked for a match when reading from +the file. Note that we must be careful to calculate the same value on all +architectures. +@param[in] page buffer page (srv_page_size bytes) +@return CRC-32C */ +uint32_t buf_calc_page_crc32(const byte* page) +{ + /* Note: innodb_checksum_algorithm=crc32 could and should have + included the entire page in the checksum, and CRC-32 values + should be combined with the CRC-32 function, not with + exclusive OR. We stick to the current algorithm in order to + remain compatible with old data files. */ + return ut_crc32(page + FIL_PAGE_OFFSET, + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + - FIL_PAGE_OFFSET) + ^ ut_crc32(page + FIL_PAGE_DATA, + srv_page_size + - (FIL_PAGE_DATA + FIL_PAGE_END_LSN_OLD_CHKSUM)); } +#endif /** Calculate a checksum which is stored to the page when it is written to a file. Note that we must be careful to calculate the same value on diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 6218870f704..854037f7df6 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -2636,10 +2636,8 @@ fil_space_verify_crypt_checksum( srv_checksum_algorithm); switch (algorithm) { case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: - /* We never supported upgrade from the "legacy crc32" - on big endian systems from MariaDB 10.1 to later. */ valid = buf_page_is_checksum_valid_crc32( - page, checksum1, checksum2, false); + page, checksum1, checksum2); break; case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: valid = buf_page_is_checksum_valid_innodb( @@ -2649,13 +2647,11 @@ fil_space_verify_crypt_checksum( case SRV_CHECKSUM_ALGORITHM_CRC32: case SRV_CHECKSUM_ALGORITHM_INNODB: case SRV_CHECKSUM_ALGORITHM_NONE: - /* We never supported upgrade from the "legacy crc32" - on big endian systems from MariaDB 10.1 to later. - We also never supported + /* never supported innodb_checksum_algorithm=none or strict_none for encrypted pages. */ valid = buf_page_is_checksum_valid_crc32( - page, checksum1, checksum2, false) + page, checksum1, checksum2) || buf_page_is_checksum_valid_innodb( page, checksum1, checksum2); break; @@ -2684,8 +2680,11 @@ fil_space_verify_crypt_checksum( ib::info() << "If unencrypted: stored checksum [" << checksum1 << ":" << checksum2 << "] calculated crc32 [" - << buf_calc_page_crc32(page, false) << ":" - << buf_calc_page_crc32(page, true) << "] innodb [" + << buf_calc_page_crc32(page) +# ifdef INNODB_BUG_ENDIAN_CRC32 + << ":" << buf_calc_page_crc32(page, true) +# endif /* INNODB_BUG_ENDIAN_CRC32 */ + << "] innodb [" << buf_calc_page_old_checksum(page) << ":" << buf_calc_page_new_checksum(page) << "] LSN " << mach_read_from_4(page + FIL_PAGE_LSN); diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 8e13b3876e4..da344a3216c 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -716,14 +716,12 @@ buf_block_unfix( @param[in] read_buf database page @param[in] checksum_field1 new checksum field @param[in] checksum_field2 old checksum field -@param[in] use_legacy_big_endian use legacy big endian algorithm @return true if the page is in crc32 checksum format. */ bool buf_page_is_checksum_valid_crc32( const byte* read_buf, ulint checksum_field1, - ulint checksum_field2, - bool use_legacy_big_endian) + ulint checksum_field2) MY_ATTRIBUTE((nonnull(1), warn_unused_result)); /** Checks if the page is in innodb checksum format. diff --git a/storage/innobase/include/buf0checksum.h b/storage/innobase/include/buf0checksum.h index 0bac2b911ee..06eb37906d2 100644 --- a/storage/innobase/include/buf0checksum.h +++ b/storage/innobase/include/buf0checksum.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -29,19 +29,26 @@ Created Aug 11, 2011 Vasil Dimov #include "buf0types.h" +#ifdef INNODB_BUG_ENDIAN_CRC32 /** Calculate the CRC32 checksum of a page. The value is stored to the page when it is written to a file and also checked for a match when reading from -the file. When reading we allow both normal CRC32 and CRC-legacy-big-endian -variants. Note that we must be careful to calculate the same value on 32-bit -and 64-bit architectures. -@param[in] page buffer page (UNIV_PAGE_SIZE bytes) -@param[in] use_legacy_big_endian if true then use big endian -byteorder when converting byte strings to integers -@return checksum */ -uint32_t -buf_calc_page_crc32( - const byte* page, - bool use_legacy_big_endian = false); +the file. Note that we must be careful to calculate the same value on all +architectures. +@param[in] page buffer page (srv_page_size bytes) +@param[in] bug_endian whether to use big endian byteorder +when converting byte strings to integers, for bug-compatibility with +big-endian architecture running MySQL 5.6, MariaDB 10.0 or MariaDB 10.1 +@return CRC-32C */ +uint32_t buf_calc_page_crc32(const byte* page, bool bug_endian = false); +#else +/** Calculate the CRC32 checksum of a page. The value is stored to the page +when it is written to a file and also checked for a match when reading from +the file. Note that we must be careful to calculate the same value on all +architectures. +@param[in] page buffer page (srv_page_size bytes) +@return CRC-32C */ +uint32_t buf_calc_page_crc32(const byte* page); +#endif /** Calculate a checksum which is stored to the page when it is written to a file. Note that we must be careful to calculate the same value on @@ -69,6 +76,5 @@ const char* buf_checksum_algorithm_name(srv_checksum_algorithm_t algo); extern ulong srv_checksum_algorithm; -extern bool legacy_big_endian_checksum; #endif /* buf0checksum_h */ diff --git a/storage/innobase/include/page0zip.h b/storage/innobase/include/page0zip.h index d72d5662f78..f458b29c4b4 100644 --- a/storage/innobase/include/page0zip.h +++ b/storage/innobase/include/page0zip.h @@ -492,16 +492,17 @@ page_zip_parse_compress( @param[in] data compressed page @param[in] size size of compressed page @param[in] algo algorithm to use -@param[in] use_legacy_big_endian only used if algo is -SRV_CHECKSUM_ALGORITHM_CRC32 or SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 - if true -then use big endian byteorder when converting byte strings to integers. @return page checksum */ uint32_t page_zip_calc_checksum( const void* data, ulint size, - srv_checksum_algorithm_t algo, - bool use_legacy_big_endian = false); + srv_checksum_algorithm_t algo +#ifdef INNODB_BUG_ENDIAN_CRC32 + /** for crc32, use the big-endian bug-compatible crc32 variant */ + , bool use_legacy_big_endian = false +#endif +); /**********************************************************************//** Verify a compressed page's checksum. diff --git a/storage/innobase/include/ut0crc32.h b/storage/innobase/include/ut0crc32.h index 36b389b5bd2..175a55e51c0 100644 --- a/storage/innobase/include/ut0crc32.h +++ b/storage/innobase/include/ut0crc32.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, MariaDB Corporation. +Copyright (c) 2016, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -47,13 +47,11 @@ typedef uint32_t (*ut_crc32_func_t)(const byte* ptr, ulint len); /** Pointer to CRC32 calculation function. */ extern ut_crc32_func_t ut_crc32; +#ifdef INNODB_BUG_ENDIAN_CRC32 /** Pointer to CRC32 calculation function, which uses big-endian byte order when converting byte strings to integers internally. */ extern ut_crc32_func_t ut_crc32_legacy_big_endian; - -/** Pointer to CRC32-byte-by-byte calculation function (byte order agnostic, -but very slow). */ -extern ut_crc32_func_t ut_crc32_byte_by_byte; +#endif /* INNODB_BUG_ENDIAN_CRC32 */ extern const char* ut_crc32_implementation; diff --git a/storage/innobase/innodb.cmake b/storage/innobase/innodb.cmake index 7272585dcce..b24343afacd 100644 --- a/storage/innobase/innodb.cmake +++ b/storage/innobase/innodb.cmake @@ -25,6 +25,7 @@ INCLUDE(lzma.cmake) INCLUDE(bzip2.cmake) INCLUDE(snappy.cmake) INCLUDE(numa) +INCLUDE(TestBigEndian) MYSQL_CHECK_LZ4() MYSQL_CHECK_LZO() @@ -32,6 +33,7 @@ MYSQL_CHECK_LZMA() MYSQL_CHECK_BZIP2() MYSQL_CHECK_SNAPPY() MYSQL_CHECK_NUMA() +TEST_BIG_ENDIAN(IS_BIG_ENDIAN) IF(CMAKE_CROSSCOMPILING) # Use CHECK_C_SOURCE_COMPILES instead of CHECK_C_SOURCE_RUNS when @@ -123,6 +125,11 @@ ELSEIF(WITH_INNODB_ROOT_GUESS) ADD_DEFINITIONS(-DBTR_CUR_ADAPT) ENDIF() +OPTION(WITH_INNODB_BUG_ENDIAN_CRC32 "Weaken innodb_checksum_algorithm=crc32 by supporting upgrade from big-endian systems running 5.6/10.0/10.1" ${IS_BIG_ENDIAN}) +IF(WITH_INNODB_BUG_ENDIAN_CRC32) + ADD_DEFINITIONS(-DINNODB_BUG_ENDIAN_CRC32) +ENDIF() + OPTION(WITH_INNODB_EXTRA_DEBUG "Enable extra InnoDB debug checks" OFF) IF(WITH_INNODB_EXTRA_DEBUG) IF(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index 82e95a79f4e..50e22059e7d 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -4909,18 +4909,17 @@ corrupt: @param[in] data compressed page @param[in] size size of compressed page @param[in] algo algorithm to use -@param[in] use_legacy_big_endian only used if algo is -SRV_CHECKSUM_ALGORITHM_CRC32 or SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 - if true -then use big endian byteorder when converting byte strings to integers. -SRV_CHECKSUM_ALGORITHM_CRC32 or SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 - if true -then use big endian byteorder when converting byte strings to integers. @return page checksum */ uint32_t page_zip_calc_checksum( const void* data, ulint size, - srv_checksum_algorithm_t algo, - bool use_legacy_big_endian /* = false */) + srv_checksum_algorithm_t algo +#ifdef INNODB_BUG_ENDIAN_CRC32 + /** for crc32, use the big-endian bug-compatible crc32 variant */ + , bool use_legacy_big_endian +#endif +) { uLong adler; const Bytef* s = static_cast<const byte*>(data); @@ -4931,25 +4930,25 @@ page_zip_calc_checksum( switch (algo) { case SRV_CHECKSUM_ALGORITHM_CRC32: case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: - { - ut_ad(size > FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - - ut_crc32_func_t crc32_func = use_legacy_big_endian - ? ut_crc32_legacy_big_endian - : ut_crc32; - - const uint32_t crc32 - = crc32_func( - s + FIL_PAGE_OFFSET, - FIL_PAGE_LSN - FIL_PAGE_OFFSET) - ^ crc32_func( + ut_ad(size > FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); +#ifdef INNODB_BUG_ENDIAN_CRC32 + if (use_legacy_big_endian) { + return ut_crc32_legacy_big_endian(s + FIL_PAGE_OFFSET, + FIL_PAGE_LSN + - FIL_PAGE_OFFSET) + ^ ut_crc32_legacy_big_endian( s + FIL_PAGE_TYPE, 2) - ^ crc32_func( + ^ ut_crc32_legacy_big_endian( s + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, - size - FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - - return(crc32); + size + - FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); } +#endif + return ut_crc32(s + FIL_PAGE_OFFSET, + FIL_PAGE_LSN - FIL_PAGE_OFFSET) + ^ ut_crc32(s + FIL_PAGE_TYPE, 2) + ^ ut_crc32(s + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, + size - FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); case SRV_CHECKSUM_ALGORITHM_INNODB: case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: ut_ad(size > FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); @@ -4984,13 +4983,8 @@ page_zip_verify_checksum( const void* data, /*!< in: compressed page */ ulint size) /*!< in: size of compressed page */ { - ib_uint32_t stored; - ib_uint32_t calc; - ib_uint32_t crc32 = 0; - ib_uint32_t innodb = 0; - - stored = static_cast<ib_uint32_t>(mach_read_from_4( - static_cast<const unsigned char*>(data) + FIL_PAGE_SPACE_OR_CHKSUM)); + const uint32_t stored = mach_read_from_4( + static_cast<const byte*>(data) + FIL_PAGE_SPACE_OR_CHKSUM); #if FIL_PAGE_LSN % 8 #error "FIL_PAGE_LSN must be 64 bit aligned" @@ -5034,8 +5028,7 @@ page_zip_verify_checksum( return(TRUE); } - calc = static_cast<ib_uint32_t>(page_zip_calc_checksum( - data, size, curr_algo)); + uint32_t calc = page_zip_calc_checksum(data, size, curr_algo); #ifdef UNIV_INNOCHECKSUM if (log_file) { @@ -5070,13 +5063,11 @@ page_zip_verify_checksum( switch (curr_algo) { case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: - calc = page_zip_calc_checksum(data, size, curr_algo, true); - if (calc == stored) { - legacy_big_endian_checksum = true; - return TRUE; - } - - return FALSE; +#ifdef INNODB_BUG_ENDIAN_CRC32 + return stored == page_zip_calc_checksum(data, size, curr_algo, + true); +#endif + /* fall through */ case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: return FALSE; @@ -5085,29 +5076,29 @@ page_zip_verify_checksum( return(TRUE); } - calc = page_zip_calc_checksum(data, size, curr_algo, true); - crc32 = calc; - - if (crc32 == stored) { - legacy_big_endian_checksum = true; - return TRUE; - } - - innodb = static_cast<ib_uint32_t>(page_zip_calc_checksum( - data, size, SRV_CHECKSUM_ALGORITHM_INNODB)); - break; + return +#ifdef INNODB_BUG_ENDIAN_CRC32 + stored == page_zip_calc_checksum(data, size, curr_algo, + true) || +#endif + stored == page_zip_calc_checksum( + data, size, SRV_CHECKSUM_ALGORITHM_INNODB); case SRV_CHECKSUM_ALGORITHM_INNODB: if (stored == BUF_NO_CHECKSUM_MAGIC) { return TRUE; } - crc32 = static_cast<ib_uint32_t>(page_zip_calc_checksum( - data, size, SRV_CHECKSUM_ALGORITHM_CRC32)); - innodb = calc; - break; + return stored == page_zip_calc_checksum( + data, size, SRV_CHECKSUM_ALGORITHM_CRC32) +#ifdef INNODB_BUG_ENDIAN_CRC32 + || stored == page_zip_calc_checksum( + data, size, + SRV_CHECKSUM_ALGORITHM_CRC32, true) +#endif + ; case SRV_CHECKSUM_ALGORITHM_NONE: return TRUE; } - return (stored == crc32 || stored == innodb); + return FALSE; } diff --git a/storage/innobase/ut/ut0crc32.cc b/storage/innobase/ut/ut0crc32.cc index cbb571e8f47..0c04cc2fc49 100644 --- a/storage/innobase/ut/ut0crc32.cc +++ b/storage/innobase/ut/ut0crc32.cc @@ -2,7 +2,7 @@ Copyright (c) 2009, 2010 Facebook, Inc. All Rights Reserved. Copyright (c) 2011, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, MariaDB Corporation. +Copyright (c) 2016, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -92,13 +92,11 @@ mysys/my_perf.c, contributed by Facebook under the following license. /** Pointer to CRC32 calculation function. */ ut_crc32_func_t ut_crc32; +#ifdef INNODB_BUG_ENDIAN_CRC32 /** Pointer to CRC32 calculation function, which uses big-endian byte order when converting byte strings to integers internally. */ ut_crc32_func_t ut_crc32_legacy_big_endian; - -/** Pointer to CRC32-byte-by-byte calculation function (byte order agnostic, -but very slow). */ -ut_crc32_func_t ut_crc32_byte_by_byte; +#endif /* INNODB_BUG_ENDIAN_CRC32 */ /** Text description of CRC32 implementation */ const char* ut_crc32_implementation; @@ -278,6 +276,7 @@ ut_crc32_64_hw( *len -= 8; } +#ifdef INNODB_BUG_ENDIAN_CRC32 /** Calculate CRC32 over 64-bit byte string using a hardware/CPU instruction. The byte string is converted to a 64-bit integer using big endian byte order. @param[in,out] crc crc32 checksum so far when this function is called, @@ -308,6 +307,7 @@ ut_crc32_64_legacy_big_endian_hw( *data += 8; *len -= 8; } +#endif /* INNODB_BUG_ENDIAN_CRC32 */ /** Calculates CRC32 using hardware/CPU instructions. @param[in] buf data over which to calculate CRC32 @@ -396,6 +396,7 @@ ut_crc32_hw( return(~crc); } +# ifdef INNODB_BUG_ENDIAN_CRC32 /** Calculates CRC32 using hardware/CPU instructions. This function uses big endian byte ordering when converting byte sequence to integers. @@ -445,26 +446,7 @@ ut_crc32_legacy_big_endian_hw( return(~crc); } - -/** Calculates CRC32 using hardware/CPU instructions. -This function processes one byte at a time (very slow) and thus it does -not depend on the byte order of the machine. -@param[in] buf data over which to calculate CRC32 -@param[in] len data length -@return CRC-32C (polynomial 0x11EDC6F41) */ -uint32_t -ut_crc32_byte_by_byte_hw( - const byte* buf, - ulint len) -{ - uint32_t crc = 0xFFFFFFFFU; - - while (len > 0) { - ut_crc32_8_hw(&crc, &buf, &len); - } - - return(~crc); -} +# endif /* INNODB_BUG_ENDIAN_CRC32 */ #endif /* defined(__GNUC__) && defined(__x86_64__) || (_WIN64) */ /* CRC32 software implementation. */ @@ -577,6 +559,7 @@ ut_crc32_64_sw( *len -= 8; } +#ifdef INNODB_BUG_ENDIAN_CRC32 /** Calculate CRC32 over 64-bit byte string using a software implementation. The byte string is converted to a 64-bit integer using big endian byte order. @param[in,out] crc crc32 checksum so far when this function is called, @@ -602,6 +585,7 @@ ut_crc32_64_legacy_big_endian_sw( *data += 8; *len -= 8; } +#endif /* INNODB_BUG_ENDIAN_CRC32 */ /** Calculates CRC32 in software, without using CPU instructions. @param[in] buf data over which to calculate CRC32 @@ -653,6 +637,7 @@ ut_crc32_sw( return(~crc); } +#ifdef INNODB_BUG_ENDIAN_CRC32 /** Calculates CRC32 in software, without using CPU instructions. This function uses big endian byte ordering when converting byte sequence to integers. @@ -704,28 +689,7 @@ ut_crc32_legacy_big_endian_sw( return(~crc); } - -/** Calculates CRC32 in software, without using CPU instructions. -This function processes one byte at a time (very slow) and thus it does -not depend on the byte order of the machine. -@param[in] buf data over which to calculate CRC32 -@param[in] len data length -@return CRC-32C (polynomial 0x11EDC6F41) */ -uint32_t -ut_crc32_byte_by_byte_sw( - const byte* buf, - ulint len) -{ - uint32_t crc = 0xFFFFFFFFU; - - ut_a(ut_crc32_slice8_table_initialized); - - while (len > 0) { - ut_crc32_8_sw(&crc, &buf, &len); - } - - return(~crc); -} +#endif /* INNODB_BUG_ENDIAN_CRC32 */ /********************************************************************//** Initializes the data structures used by ut_crc32*(). Does not do any @@ -736,8 +700,9 @@ ut_crc32_init() { ut_crc32_slice8_table_init(); ut_crc32 = ut_crc32_sw; +#ifdef INNODB_BUG_ENDIAN_CRC32 ut_crc32_legacy_big_endian = ut_crc32_legacy_big_endian_sw; - ut_crc32_byte_by_byte = ut_crc32_byte_by_byte_sw; +#endif /* INNODB_BUG_ENDIAN_CRC32 */ ut_crc32_implementation = "Using generic crc32 instructions"; #if (defined(__GNUC__) && defined(__x86_64__)) || defined(_MSC_VER) @@ -770,8 +735,9 @@ ut_crc32_init() if (features_ecx & 1 << 20) { ut_crc32 = ut_crc32_hw; +#ifdef INNODB_BUG_ENDIAN_CRC32 ut_crc32_legacy_big_endian = ut_crc32_legacy_big_endian_hw; - ut_crc32_byte_by_byte = ut_crc32_byte_by_byte_hw; +#endif /* INNODB_BUG_ENDIAN_CRC32 */ ut_crc32_implementation = "Using SSE2 crc32 instructions"; } |