diff options
author | Jan Lindström <jan.lindstrom@mariadb.com> | 2017-08-04 13:57:26 +0300 |
---|---|---|
committer | Jan Lindström <jan.lindstrom@mariadb.com> | 2017-08-08 09:41:09 +0300 |
commit | 34eef269eb93ee42c0e5e1894781c61f66cb6f14 (patch) | |
tree | 311a6e0201c46acb99456dfafbc6ffdfa1b3a656 | |
parent | 36e81a23c567f81d08b6434087cf284e1bee06f9 (diff) | |
download | mariadb-git-34eef269eb93ee42c0e5e1894781c61f66cb6f14.tar.gz |
MDEV-11939: innochecksum mistakes a file for an encrypted one (page 0 invalid)
Always read full page 0 to determine does tablespace contain
encryption metadata. Tablespaces that are page compressed or
page compressed and encrypted do not compare checksum as
it does not exists. For encrypted tables use checksum
verification written for encrypted tables and normal tables
use normal method.
buf_page_is_checksum_valid_crc32
buf_page_is_checksum_valid_innodb
buf_page_is_checksum_valid_none
Modify Innochecksum logging to file to avoid compilation
warnings.
fil0crypt.cc fil0crypt.h
Modify to be able to use in innochecksum compilation and
move fil_space_verify_crypt_checksum to end of the file.
Add innochecksum logging to file.
univ.i
Add innochecksum strict_verify, log_file and cur_page_num
variables as extern.
page_zip_verify_checksum
Add innochecksum logging to file and remove unnecessary code.
innochecksum.cc
Lot of changes most notable able to read encryption
metadata from page 0 of the tablespace.
Added test case where we corrupt intentionally
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION (encryption key version)
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4 (post encryption checksum)
FIL_DATA+10 (data)
21 files changed, 923 insertions, 656 deletions
diff --git a/extra/CMakeLists.txt b/extra/CMakeLists.txt index 9cc5e7faa86..8c73e23d72b 100644 --- a/extra/CMakeLists.txt +++ b/extra/CMakeLists.txt @@ -79,7 +79,6 @@ IF(WITH_INNOBASE_STORAGE_ENGINE OR WITH_XTRADB_STORAGE_ENGINE) ../storage/innobase/ut/ut0ut.cc ../storage/innobase/buf/buf0buf.cc ../storage/innobase/page/page0zip.cc - ../storage/innobase/os/os0file.cc ../storage/innobase/fil/fil0crypt.cc ) diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc index fe828026b4b..a0976970db7 100644 --- a/extra/innochecksum.cc +++ b/extra/innochecksum.cc @@ -26,40 +26,58 @@ Published with a permission. */ -#include <my_config.h> #include <my_global.h> #include <stdio.h> #include <stdlib.h> #include <time.h> #include <sys/types.h> #include <sys/stat.h> -#ifdef HAVE_UNISTD_H +#ifndef __WIN__ # include <unistd.h> #endif #include <my_getopt.h> #include <m_string.h> -#include <welcome_copyright_notice.h> /* ORACLE_WELCOME_COPYRIGHT_NOTICE */ +#include <welcome_copyright_notice.h> /* ORACLE_WELCOME_COPYRIGHT_NOTICE */ /* Only parts of these files are included from the InnoDB codebase. The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */ -#include "univ.i" /* include all of this */ -#include "page0size.h" /* page_size_t */ -#include "page0zip.h" /* page_zip_calc_checksum() */ -#include "page0page.h" /* PAGE_* */ -#include "trx0undo.h" /* TRX_UNDO_* */ -#include "fut0lst.h" /* FLST_NODE_SIZE */ -#include "buf0checksum.h" /* buf_calc_page_*() */ -#include "fil0fil.h" /* FIL_* */ -#include "fil0crypt.h" -#include "os0file.h" -#include "fsp0fsp.h" /* fsp_flags_get_page_size() & - fsp_flags_get_zip_size() */ -#include "mach0data.h" /* mach_read_from_4() */ -#include "ut0crc32.h" /* ut_crc32_init() */ -#include "fsp0pagecompress.h" /* fil_get_compression_alg_name */ +typedef void fil_space_t; + +#include "univ.i" /* include all of this */ +#include "page0size.h" + +#define FLST_BASE_NODE_SIZE (4 + 2 * FIL_ADDR_SIZE) +#define FLST_NODE_SIZE (2 * FIL_ADDR_SIZE) +#define FSEG_PAGE_DATA FIL_PAGE_DATA +#define FSEG_HEADER_SIZE 10 +#define UT_BITS_IN_BYTES(b) (((b) + 7) / 8) + +#include "ut0ut.h" #include "ut0byte.h" +#include "mtr0types.h" #include "mach0data.h" +#include "fsp0types.h" +#include "rem0rec.h" +#include "buf0checksum.h" /* buf_calc_page_*() */ +#include "buf0buf.h" /* buf_page_is_corrupted */ +#include "fil0fil.h" /* FIL_* */ +#include "page0page.h" /* PAGE_* */ +#include "page0zip.h" /* page_zip_*() */ +#include "trx0undo.h" /* TRX_* */ +#include "fsp0fsp.h" /* fsp_flags_get_page_size() & + fsp_flags_get_zip_size() */ +#include "ut0crc32.h" /* ut_crc32_init() */ +#include "fsp0pagecompress.h" /* fil_get_compression_alg_name */ +#include "fil0crypt.h" /* fil_space_verify_crypt_checksum */ + +#include <string.h> + +#ifdef UNIV_NONINL +# include "fsp0fsp.ic" +# include "mach0data.ic" +# include "ut0rnd.ic" +#endif #ifndef PRIuMAX #define PRIuMAX "llu" @@ -68,25 +86,29 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */ /* Global variables */ static bool verbose; static bool just_count; -static uintmax_t start_page; -static uintmax_t end_page; -static uintmax_t do_page; +static unsigned long long start_page; +static unsigned long long end_page; +static unsigned long long do_page; static bool use_end_page; static bool do_one_page; -/* replaces declaration in srv0srv.c */ -ulong srv_page_size; -page_size_t univ_page_size(0, 0, false); +static my_bool do_leaf; +static my_bool per_page_details; +static ulong n_merge; extern ulong srv_checksum_algorithm; +static ulong physical_page_size; /* Page size in bytes on disk. */ +static ulong logical_page_size; /* Page size when uncompressed. */ +ulong srv_page_size; +page_size_t univ_page_size(0, 0, false); /* Current page number (0 based). */ -uintmax_t cur_page_num; +unsigned long long cur_page_num; /* Skip the checksum verification. */ static bool no_check; /* Enabled for strict checksum verification. */ -bool strict_verify; +bool strict_verify = 0; /* Enabled for rewrite checksum. */ static bool do_write; /* Mismatches count allowed (0 by default). */ -static uintmax_t allow_mismatches; +static unsigned long long allow_mismatches=0; static bool page_type_summary; static bool page_type_dump; /* Store filename for page-type-dump option. */ @@ -99,8 +121,7 @@ char* log_filename = NULL; FILE* log_file = NULL; /* Enabled for log write option. */ static bool is_log_enabled = false; -static my_bool do_leaf; -static ulong n_merge; + #ifndef _WIN32 /* advisory lock for non-window system. */ struct flock lk; @@ -198,75 +219,6 @@ struct per_index_stats { std::map<unsigned long long, per_index_stats> index_ids; -bool encrypted = false; - -ulint -page_is_comp( -/*=========*/ - const page_t* page) /*!< in: index page */ -{ - return(page_header_get_field(page, PAGE_N_HEAP) & 0x8000); -} - -bool -page_is_leaf( -/*=========*/ - const page_t* page) /*!< in: page */ -{ - return(!*(const uint16*) (page + (PAGE_HEADER + PAGE_LEVEL))); -} - -ulint -page_get_page_no( -/*=============*/ - const page_t* page) /*!< in: page */ -{ - return(mach_read_from_4(page + FIL_PAGE_OFFSET)); -} -#define FSEG_HEADER_SIZE 10 /*!< Length of the file system - header, in bytes */ -#define REC_N_NEW_EXTRA_BYTES 5 -#define REC_N_OLD_EXTRA_BYTES 6 -#define PAGE_DATA (PAGE_HEADER + 36 + 2 * FSEG_HEADER_SIZE) - /* start of data on the page */ -#define PAGE_NEW_SUPREMUM (PAGE_DATA + 2 * REC_N_NEW_EXTRA_BYTES + 8) - /* offset of the page supremum record on a - new-style compact page */ -#define PAGE_NEW_SUPREMUM_END (PAGE_NEW_SUPREMUM + 8) -#define PAGE_OLD_SUPREMUM (PAGE_DATA + 2 + 2 * REC_N_OLD_EXTRA_BYTES + 8) - /* offset of the page supremum record on an - old-style page */ -#define PAGE_OLD_SUPREMUM_END (PAGE_OLD_SUPREMUM + 9) - /* offset of the page supremum record end on - an old-style page */ -#define FLST_BASE_NODE_SIZE (4 + 2 * 6) -#define XDES_ARR_OFFSET (FSP_HEADER_OFFSET + FSP_HEADER_SIZE) -#define XDES_FREE_BIT 0 -#define XDES_BITMAP (FLST_NODE_SIZE + 12) -#define XDES_BITS_PER_PAGE 2 -#define UT_BITS_IN_BYTES(b) (((b) + 7) / 8) -#define XDES_SIZE \ - (XDES_BITMAP \ - + UT_BITS_IN_BYTES(FSP_EXTENT_SIZE * XDES_BITS_PER_PAGE)) - -ulint -page_get_data_size( -/*===============*/ - const page_t* page) /*!< in: index page */ -{ - ulint ret; - - ret = (ulint)(page_header_get_field(page, PAGE_HEAP_TOP) - - (page_is_comp(page) - ? PAGE_NEW_SUPREMUM_END - : PAGE_OLD_SUPREMUM_END) - - page_header_get_field(page, PAGE_GARBAGE)); - - ut_ad(ret < UNIV_PAGE_SIZE); - - return(ret); -} - void print_index_leaf_stats( unsigned long long id, const per_index_stats& index, @@ -280,8 +232,7 @@ void print_index_leaf_stats( fprintf(fil_out, "page_no\tdata_size\tn_recs\n"); while (it_page != index.leaves.end()) { const per_page_stats& stat = it_page->second; - fprintf(fil_out, "%llu\t" ULINTPF "\t" ULINTPF "\n", - it_page->first, stat.data_size, stat.n_recs); + fprintf(fil_out, "%llu\t" ULINTPF "\t" ULINTPF "\n", it_page->first, stat.data_size, stat.n_recs); page_no = stat.right_page_no; it_page = index.leaves.find(page_no); } @@ -307,6 +258,7 @@ void defrag_analysis( break; } } + if (index.max_data_size) { n_pages += data_size_total / index.max_data_size; if (data_size_total % index.max_data_size != 0) { @@ -316,15 +268,16 @@ void defrag_analysis( } if (index.leaf_pages) { - fprintf(fil_out, "count = " ULINTPF " free = " ULINTPF "\n", - index.count, index.free_pages); + fprintf(fil_out, "count = " ULINTPF " free = " ULINTPF "\n", index.count, index.free_pages); + } + + if (!n_leaf_pages) { + n_leaf_pages = 1; } - fprintf(fil_out, "%llu\t\t%llu\t\t" - ULINTPF "\t\t%lu\t\t" ULINTPF "\t\t%.2f\t" ULINTPF "\n", + fprintf(fil_out, "%llu\t\t%llu\t\t" ULINTPF "\t\t" ULINTPF "\t\t" ULINTPF "\t\t%.2f\t" ULINTPF "\n", id, index.leaf_pages, n_leaf_pages, n_merge, n_pages, - 1.0 - (double)n_pages / (double)n_leaf_pages, - index.max_data_size); + 1.0 - (double)n_pages / (double)n_leaf_pages, index.max_data_size); } void print_leaf_stats( @@ -372,29 +325,6 @@ get_page_size( return(page_size_t(flags)); } -#ifdef MYSQL_COMPRESSION -/** Decompress a page -@param[in,out] buf Page read from disk, uncompressed data will - also be copied to this page -@param[in, out] scratch Page to use for temporary decompress -@param[in] page_size scratch physical size -@return true if decompress succeeded */ -static -bool page_decompress( - byte* buf, - byte* scratch, - page_size_t page_size) -{ - dberr_t err; - - /* Set the dblwr recover flag to false. */ - err = os_file_decompress_page( - false, buf, scratch, page_size.physical()); - - return(err == DB_SUCCESS); -} -#endif - #ifdef _WIN32 /***********************************************//* @param [in] error error no. from the getLastError(). @@ -439,6 +369,7 @@ open_file( access = GENERIC_READ; flags = _O_RDONLY | _O_BINARY; } + /* CreateFile() also provide advisory lock with the usage of access and share mode of the file.*/ hFile = CreateFile( @@ -462,8 +393,7 @@ open_file( if (do_write) { create_flag = O_RDWR; lk.l_type = F_WRLCK; - } - else { + } else { create_flag = O_RDONLY; lk.l_type = F_RDLCK; } @@ -525,12 +455,18 @@ ulong read_file( /** Check if page is corrupted or not. @param[in] buf page frame @param[in] page_size page size +@param[in] is_encrypted true if page0 contained cryp_data + with crypt_scheme encrypted +@param[in] is_compressed true if page0 fsp_flags contained + page compression flag @retval true if page is corrupted otherwise false. */ static bool is_page_corrupted( - const byte* buf, - const page_size_t& page_size) + byte* buf, + const page_size_t& page_size, + bool is_encrypted, + bool is_compressed) { /* enable if page is corrupted. */ @@ -538,8 +474,22 @@ is_page_corrupted( /* use to store LSN values. */ ulint logseq; ulint logseqfield; + ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE); + uint key_version = mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + ulint space_id = mach_read_from_4( + buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + + /* We can't trust only a page type, thus we take account + also fsp_flags or crypt_data on page 0 */ + if ((page_type == FIL_PAGE_PAGE_COMPRESSED && is_compressed) || + (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED && + is_compressed && is_encrypted)) { + /* Page compressed tables do not contain post compression + checksum. */ + return (false); + } - if (!page_size.is_compressed()) { + if (page_size.is_compressed()) { /* check the stored log sequence numbers for uncompressed tablespace. */ logseq = mach_read_from_4(buf + FIL_PAGE_LSN + 4); @@ -549,35 +499,32 @@ is_page_corrupted( if (is_log_enabled) { fprintf(log_file, - "page::%" PRIuMAX + "space::" ULINTPF " page::%llu" "; log sequence number:first = " ULINTPF "; second = " ULINTPF "\n", - cur_page_num, logseq, logseqfield); + space_id, cur_page_num, logseq, logseqfield); if (logseq != logseqfield) { fprintf(log_file, - "Fail; page %" PRIuMAX + "Fail; space::" ULINTPF " page::%llu" " invalid (fails log " "sequence number check)\n", - cur_page_num); + space_id, cur_page_num); } } } - /* FIXME: Read the page number from the tablespace header, - and check that every page carries the same page number. */ + /* Again we can't trust only FIL_PAGE_FILE_FLUSH_LSN field + now repurposed as FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, + we need to check also crypt_data contents. - /* If page is encrypted, use different checksum calculation + If page is encrypted, use different checksum calculation as innochecksum can't decrypt pages. Note that some old InnoDB versions did not initialize FIL_PAGE_FILE_FLUSH_LSN field so if crypt checksum does not match we verify checksum using - normal method. - */ - if (mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0) { - is_corrupted = fil_space_verify_crypt_checksum( - const_cast<byte*>(buf), page_size, - mach_read_from_4(buf - + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID), - cur_page_num); + normal method. */ + if (is_encrypted && key_version != 0) { + is_corrupted = !fil_space_verify_crypt_checksum(buf, + page_size, space_id, cur_page_num); } else { is_corrupted = true; } @@ -628,8 +575,9 @@ is_page_empty( if (*page++) { return (false); } - } - return (true); + } + + return (true); } /********************************************************************//** @@ -682,7 +630,7 @@ update_checksum( mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); if (is_log_enabled) { - fprintf(log_file, "page::%" PRIuMAX "; Updated checksum =" + fprintf(log_file, "page::%llu; Updated checksum =" " %u\n", cur_page_num, checksum); } @@ -713,7 +661,7 @@ update_checksum( mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); if (is_log_enabled) { - fprintf(log_file, "page::%" PRIuMAX "; Updated checksum field1" + fprintf(log_file, "page::%llu; Updated checksum field1" " = %u\n", cur_page_num, checksum); } @@ -727,7 +675,7 @@ update_checksum( FIL_PAGE_END_LSN_OLD_CHKSUM,checksum); if (is_log_enabled) { - fprintf(log_file, "page::%" PRIuMAX "; Updated checksum " + fprintf(log_file, "page::%llu; Updated checksum " "field2 = %u\n", cur_page_num, checksum); } @@ -802,7 +750,7 @@ write_file( if (page_size != fwrite(buf, 1, page_size, file == stdin ? stdout : file)) { - fprintf(stderr, "Failed to write page %" PRIuMAX " to %s: %s\n", + fprintf(stderr, "Failed to write page::%llu to %s: %s\n", cur_page_num, filename, strerror(errno)); return(false); @@ -819,93 +767,32 @@ write_file( return(true); } -/********************************************************//** -Gets the next index page number. -@return next page number */ -ulint -btr_page_get_next( -/*==============*/ - const page_t* page) /*!< in: index page */ -{ - return(mach_read_from_4(page + FIL_PAGE_NEXT)); -} - -/********************************************************//** -Gets the previous index page number. -@return prev page number */ -ulint -btr_page_get_prev( -/*==============*/ - const page_t* page) /*!< in: index page */ -{ - return(mach_read_from_4(page + FIL_PAGE_PREV)); -} - -ulint -mach_read_ulint( - const byte* ptr, - mlog_id_t type) -{ - switch (type) { - case MLOG_1BYTE: - return(mach_read_from_1(ptr)); - case MLOG_2BYTES: - return(mach_read_from_2(ptr)); - case MLOG_4BYTES: - return(mach_read_from_4(ptr)); - default: - break; - } - - ut_error; - return(0); -} -ibool -xdes_get_bit( -/*=========*/ - const xdes_t* descr, /*!< in: descriptor */ - ulint bit, /*!< in: XDES_FREE_BIT or XDES_CLEAN_BIT */ - ulint offset) /*!< in: page offset within extent: - 0 ... FSP_EXTENT_SIZE - 1 */ -{ - ulint index = bit + XDES_BITS_PER_PAGE * offset; - - ulint bit_index = index % 8; - ulint byte_index = index / 8; - - return(ut_bit_get_nth( - mach_read_ulint(descr + XDES_BITMAP + byte_index, - MLOG_1BYTE), - bit_index)); -} - /* Parse the page and collect/dump the information about page type @param [in] page buffer page -@param [in] xdes xdes page +@param [out] xdes extend descriptor page @param [in] file file for diagnosis. -@param [in] page_size page size +@param [in] page_size page_size +@param [in] is_encrypted tablespace is encrypted */ void parse_page( const byte* page, - const byte* xdes, + byte* xdes, FILE* file, - page_size_t page_size) + const page_size_t& page_size, + bool is_encrypted) { - unsigned long long id=0; - ulint undo_page_type=0; - ulint n_recs; - ulint page_no=0; - ulint right_page_no=0; - ulint left_page_no=0; - ulint data_bytes=0; - bool is_leaf=false; - ulint size_range_id=0; - ulint data_types=0; - ulint key_version = 0; - + unsigned long long id; + ulint undo_page_type; char str[20]={'\0'}; + ulint n_recs; + ulint page_no; + ulint left_page_no; + ulint right_page_no; + ulint data_bytes; + bool is_leaf; + int size_range_id; /* Check whether page is doublewrite buffer. */ if(skip_page) { @@ -916,102 +803,105 @@ parse_page( switch (mach_read_from_2(page + FIL_PAGE_TYPE)) { - case FIL_PAGE_INDEX: + case FIL_PAGE_INDEX: { + uint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); page_type.n_fil_page_index++; - id = mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID); - n_recs = page_header_get_field(page, PAGE_N_RECS); - page_no = page_get_page_no(page); - left_page_no = btr_page_get_prev(page); - right_page_no = btr_page_get_next(page); - key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); /* If page is encrypted we can't read index header */ - if (!key_version) { - data_bytes = page_get_data_size(page); - } else { - data_bytes = 0; - } + if (!is_encrypted) { + id = mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID); + n_recs = mach_read_from_2(page + PAGE_HEADER + PAGE_N_RECS); + page_no = mach_read_from_4(page + FIL_PAGE_OFFSET); + left_page_no = mach_read_from_4(page + FIL_PAGE_PREV); + right_page_no = mach_read_from_4(page + FIL_PAGE_NEXT); + ulint is_comp = mach_read_from_2(page + PAGE_HEADER + PAGE_N_HEAP) & 0x8000; + ulint level = mach_read_from_2(page + PAGE_HEADER + PAGE_LEVEL); + ulint garbage = mach_read_from_2(page + PAGE_HEADER + PAGE_GARBAGE); - is_leaf = page_is_leaf(page); - if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tIndex page\t\t\t|" - "\tindex id=%llu,", cur_page_num, id); - - fprintf(file, - " page level=" ULINTPF " leaf %u" - ", No. of records=" ULINTPF - ", garbage=" ULINTPF - ", n_recs=" ULINTPF - ", %s\n", - page_header_get_field(page, PAGE_LEVEL), - is_leaf, - n_recs, - page_header_get_field(page, PAGE_GARBAGE), - data_types, - str); - } + data_bytes = (ulint)(mach_read_from_2(page + PAGE_HEADER + PAGE_HEAP_TOP) + - (is_comp + ? PAGE_NEW_SUPREMUM_END + : PAGE_OLD_SUPREMUM_END) + - garbage); - size_range_id = (data_bytes * SIZE_RANGES_FOR_PAGE - + page_size.logical() - 1) / - page_size.logical(); - - if (size_range_id > SIZE_RANGES_FOR_PAGE + 1) { - /* data_bytes is bigger than logical_page_size */ - size_range_id = SIZE_RANGES_FOR_PAGE + 1; - } + is_leaf = (!*(const uint16*) (page + (PAGE_HEADER + PAGE_LEVEL))); - /* update per-index statistics */ - { - if (index_ids.count(id) == 0) { - index_ids[id] = per_index_stats(); - } - - std::map<unsigned long long, per_index_stats>::iterator it; - it = index_ids.find(id); - per_index_stats &index = (it->second); - uchar* des = (uchar *)(xdes + XDES_ARR_OFFSET - + XDES_SIZE * ((page_no & (page_size.physical() - 1)) - / FSP_EXTENT_SIZE)); - - if (xdes_get_bit(des, XDES_FREE_BIT, - page_no % FSP_EXTENT_SIZE)) { - index.free_pages++; - return; + if (page_type_dump) { + fprintf(file, "#::%llu\t\t|\t\tIndex page\t\t\t|" + "\tindex id=%llu,", cur_page_num, id); + + fprintf(file, + " page level=" ULINTPF + ", No. of records=" ULINTPF + ", garbage=" ULINTPF ", %s\n", + level, n_recs, garbage, str); } - index.pages++; + size_range_id = (data_bytes * SIZE_RANGES_FOR_PAGE + + page_size.logical() - 1) / + page_size.logical(); - if (is_leaf) { - index.leaf_pages++; - if (data_bytes > index.max_data_size) { - index.max_data_size = data_bytes; + if (size_range_id > SIZE_RANGES_FOR_PAGE + 1) { + /* data_bytes is bigger than logical_page_size */ + size_range_id = SIZE_RANGES_FOR_PAGE + 1; + } + if (per_page_details) { + printf("index id=%llu page " ULINTPF " leaf %d n_recs " ULINTPF " data_bytes " ULINTPF + "\n", id, page_no, is_leaf, n_recs, data_bytes); + } + /* update per-index statistics */ + { + if (index_ids.count(id) == 0) { + index_ids[id] = per_index_stats(); + } + std::map<unsigned long long, per_index_stats>::iterator it; + it = index_ids.find(id); + per_index_stats &index = (it->second); + const byte* des = xdes + XDES_ARR_OFFSET + + XDES_SIZE * ((page_no & (page_size.physical() - 1)) + / FSP_EXTENT_SIZE); + if (xdes_get_bit(des, XDES_FREE_BIT, + page_no % FSP_EXTENT_SIZE)) { + index.free_pages++; + return; } - struct per_page_stats pp(n_recs, data_bytes, - left_page_no, right_page_no); + index.pages++; - index.leaves[page_no] = pp; + if (is_leaf) { + index.leaf_pages++; + if (data_bytes > index.max_data_size) { + index.max_data_size = data_bytes; + } + struct per_page_stats pp(n_recs, data_bytes, + left_page_no, right_page_no); - if (left_page_no == ULINT32_UNDEFINED) { - index.first_leaf_page = page_no; - index.count++; + index.leaves[page_no] = pp; + + if (left_page_no == ULINT32_UNDEFINED) { + index.first_leaf_page = page_no; + index.count++; + } } - } - index.total_n_recs += n_recs; - index.total_data_bytes += data_bytes; - index.pages_in_size_range[size_range_id] ++; + index.total_n_recs += n_recs; + index.total_data_bytes += data_bytes; + index.pages_in_size_range[size_range_id] ++; + } + } else { + fprintf(file, "#::%llu\t\t|\t\tEncrypted Index page\t\t\t|" + "\tkey_version %u,%s\n", cur_page_num, key_version, str); } break; - + } case FIL_PAGE_UNDO_LOG: page_type.n_fil_page_undo_log++; undo_page_type = mach_read_from_2(page + TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_TYPE); if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tUndo log page\t\t\t|", + fprintf(file, "#::%llu\t\t|\t\tUndo log page\t\t\t|", cur_page_num); } if (undo_page_type == TRX_UNDO_INSERT) { @@ -1085,7 +975,7 @@ parse_page( case FIL_PAGE_INODE: page_type.n_fil_page_inode++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInode page\t\t\t|" + fprintf(file, "#::%llu\t\t|\t\tInode page\t\t\t|" "\t%s\n",cur_page_num, str); } break; @@ -1093,7 +983,7 @@ parse_page( case FIL_PAGE_IBUF_FREE_LIST: page_type.n_fil_page_ibuf_free_list++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInsert buffer free list" + fprintf(file, "#::%llu\t\t|\t\tInsert buffer free list" " page\t|\t%s\n", cur_page_num, str); } break; @@ -1101,7 +991,7 @@ parse_page( case FIL_PAGE_TYPE_ALLOCATED: page_type.n_fil_page_type_allocated++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tFreshly allocated " + fprintf(file, "#::%llu\t\t|\t\tFreshly allocated " "page\t\t|\t%s\n", cur_page_num, str); } break; @@ -1109,7 +999,7 @@ parse_page( case FIL_PAGE_IBUF_BITMAP: page_type.n_fil_page_ibuf_bitmap++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInsert Buffer " + fprintf(file, "#::%llu\t\t|\t\tInsert Buffer " "Bitmap\t\t|\t%s\n", cur_page_num, str); } break; @@ -1117,7 +1007,7 @@ parse_page( case FIL_PAGE_TYPE_SYS: page_type.n_fil_page_type_sys++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tSystem page\t\t\t|" + fprintf(file, "#::%llu\t\t|\t\tSystem page\t\t\t|" "\t%s\n",cur_page_num, str); } break; @@ -1125,25 +1015,25 @@ parse_page( case FIL_PAGE_TYPE_TRX_SYS: page_type.n_fil_page_type_trx_sys++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tTransaction system " + fprintf(file, "#::%llu\t\t|\t\tTransaction system " "page\t\t|\t%s\n", cur_page_num, str); } break; case FIL_PAGE_TYPE_FSP_HDR: page_type.n_fil_page_type_fsp_hdr++; - memcpy((void *)xdes, (void *)page, page_size.physical()); + memcpy(xdes, page, page_size.physical()); if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tFile Space " + fprintf(file, "#::%llu\t\t|\t\tFile Space " "Header\t\t|\t%s\n", cur_page_num, str); } break; case FIL_PAGE_TYPE_XDES: page_type.n_fil_page_type_xdes++; - memcpy((void *)xdes, (void *)page, page_size.physical()); + memcpy(xdes, page, page_size.physical()); if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tExtent descriptor " + fprintf(file, "#::%llu\t\t|\t\tExtent descriptor " "page\t\t|\t%s\n", cur_page_num, str); } break; @@ -1151,7 +1041,7 @@ parse_page( case FIL_PAGE_TYPE_BLOB: page_type.n_fil_page_type_blob++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tBLOB page\t\t\t|\t%s\n", + fprintf(file, "#::%llu\t\t|\t\tBLOB page\t\t\t|\t%s\n", cur_page_num, str); } break; @@ -1159,7 +1049,7 @@ parse_page( case FIL_PAGE_TYPE_ZBLOB: page_type.n_fil_page_type_zblob++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tCompressed BLOB " + fprintf(file, "#::%llu\t\t|\t\tCompressed BLOB " "page\t\t|\t%s\n", cur_page_num, str); } break; @@ -1167,25 +1057,26 @@ parse_page( case FIL_PAGE_TYPE_ZBLOB2: page_type.n_fil_page_type_zblob2++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tSubsequent Compressed " + fprintf(file, "#::%llu\t\t|\t\tSubsequent Compressed " "BLOB page\t|\t%s\n", cur_page_num, str); } break; - case FIL_PAGE_PAGE_COMPRESSED: - page_type.n_fil_page_type_page_compressed++; - if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tPage compressed " + case FIL_PAGE_PAGE_COMPRESSED: + page_type.n_fil_page_type_page_compressed++; + if (page_type_dump) { + fprintf(file, "#::%llu\t\t|\t\tPage compressed " "page\t|\t%s\n", cur_page_num, str); - } - break; - case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED: - page_type.n_fil_page_type_page_compressed_encrypted++; - if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tPage compressed encrypted " + } + break; + + case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED: + page_type.n_fil_page_type_page_compressed_encrypted++; + if (page_type_dump) { + fprintf(file, "#::%llu\t\t|\t\tPage compressed encrypted " "page\t|\t%s\n", cur_page_num, str); - } - break; + } + break; default: page_type.n_fil_page_type_other++; break; @@ -1273,7 +1164,7 @@ print_summary( page_type.n_fil_page_type_page_compressed); fprintf(fil_out, "%8d\tPage compressed encrypted page\n", page_type.n_fil_page_type_page_compressed_encrypted); - fprintf(fil_out, "%8d\tOther type of page", + fprintf(fil_out, "%8d\tOther type of page\n", page_type.n_fil_page_type_other); fprintf(fil_out, "\n===============================================\n"); @@ -1331,8 +1222,10 @@ static struct my_option innochecksum_options[] = { 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, {"verbose", 'v', "Verbose (prints progress every 5 seconds).", &verbose, &verbose, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, +#ifndef DBUG_OFF {"debug", '#', "Output debug log. See " REFMAN "dbug-package.html", &dbug_setting, &dbug_setting, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, +#endif /* !DBUG_OFF */ {"count", 'c', "Print the count of pages in the file and exits.", &just_count, &just_count, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"start_page", 's', "Start on this page number (0 based).", @@ -1361,10 +1254,12 @@ static struct my_option innochecksum_options[] = { {"page-type-dump", 'D', "Dump the page type info for each page in a " "tablespace.", &page_dump_filename, &page_dump_filename, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"per-page-details", 'i', "Print out per-page detail information.", + &per_page_details, &per_page_details, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"log", 'l', "log output.", &log_filename, &log_filename, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"leaf", 'e', "Examine leaf index pages", + {"leaf", 'f', "Examine leaf index pages", &do_leaf, &do_leaf, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"merge", 'm', "leaf page count if merge given number of consecutive pages", &n_merge, &n_merge, 0, GET_ULONG, REQUIRED_ARG, 0, 0, (longlong)10L, 0, 1, 0}, @@ -1392,9 +1287,9 @@ static void usage(void) puts(ORACLE_WELCOME_COPYRIGHT_NOTICE("2000")); printf("InnoDB offline file checksum utility.\n"); printf("Usage: %s [-c] [-s <start page>] [-e <end page>] " - "[-p <page>] [-v] [-a <allow mismatches>] [-n] " + "[-p <page>] [-i] [-v] [-a <allow mismatches>] [-n] " "[-C <strict-check>] [-w <write>] [-S] [-D <page type dump>] " - "[-l <log>] [-e] <filename or [-]>\n", my_progname); + "[-l <log>] [-l] [-m <merge pages>] <filename or [-]>\n", my_progname); printf("See " REFMAN "innochecksum.html for usage hints.\n"); my_print_help(innochecksum_options); my_print_variables(innochecksum_options); @@ -1487,7 +1382,7 @@ get_options( char ***argv) { if (handle_options(argc, argv, innochecksum_options, - innochecksum_get_one_option)) { + innochecksum_get_one_option)) { my_end(0); exit(true); } @@ -1502,6 +1397,133 @@ get_options( return (false); } +/** Check from page 0 if table is encrypted. +@param[in] filename Filename +@param[in] page_size page size +@param[in] page Page 0 +@retval true if tablespace is encrypted, false if not +*/ +static +bool check_encryption( + const char* filename, + const page_size_t& page_size, + byte * page) +{ + ulint offset = (FSP_HEADER_OFFSET + (XDES_ARR_OFFSET + XDES_SIZE * + (page_size.physical()) / FSP_EXTENT_SIZE)); + + if (memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) { + return false; + } + + ulint type = mach_read_from_1(page + offset + MAGIC_SZ + 0); + + if (! (type == CRYPT_SCHEME_UNENCRYPTED || + type == CRYPT_SCHEME_1)) { + return false; + } + + ulint iv_length = mach_read_from_1(page + offset + MAGIC_SZ + 1); + + if (iv_length != CRYPT_SCHEME_1_IV_LEN) { + return false; + } + + uint min_key_version = mach_read_from_4 + (page + offset + MAGIC_SZ + 2 + iv_length); + + uint key_id = mach_read_from_4 + (page + offset + MAGIC_SZ + 2 + iv_length + 4); + + if (type == CRYPT_SCHEME_1 && is_log_enabled) { + fprintf(log_file,"Tablespace %s encrypted key_version %u key_id %u\n", + filename, min_key_version, key_id); + } + + return (type == CRYPT_SCHEME_1); +} + +/** +Verify page checksum. +@param[in] buf page to verify +@param[in] page_size page size +@param[in] is_encrypted true if tablespace is encrypted +@param[in] is_compressed true if tablespace is page compressed +@param[in,out] mismatch_count Number of pages failed in checksum verify +@retval 0 if page checksum matches or 1 if it does not match +*/ +static +int verify_checksum( + byte* buf, + const page_size_t& page_size, + bool is_encrypted, + bool is_compressed, + unsigned long long* mismatch_count) +{ + int exit_status = 0; + bool is_corrupted = false; + + is_corrupted = is_page_corrupted( + buf, page_size, is_encrypted, is_compressed); + + if (is_corrupted) { + fprintf(stderr, "Fail: page::%llu invalid\n", + cur_page_num); + + (*mismatch_count)++; + + if (*mismatch_count > allow_mismatches) { + fprintf(stderr, + "Exceeded the " + "maximum allowed " + "checksum mismatch " + "count::%llu current::%llu\n", + *mismatch_count, + allow_mismatches); + + exit_status = 1; + } + } + + return (exit_status); +} + +/** Rewrite page checksum if needed. +@param[in] filename File name +@param[in] fil_in File pointer +@param[in] buf page +@param[in] page_size page size +@param[in] pos File position +@param[in] is_encrypted true if tablespace is encrypted +@param[in] is_compressed true if tablespace is page compressed +@retval 0 if checksum rewrite was successful, 1 if error was detected */ +static +int +rewrite_checksum( + const char* filename, + FILE* fil_in, + byte* buf, + const page_size_t& page_size, + fpos_t* pos, + bool is_encrypted, + bool is_compressed) +{ + int exit_status = 0; + /* Rewrite checksum. Note that for encrypted and + page compressed tables this is not currently supported. */ + if (do_write && + !is_encrypted && + !is_compressed + && !write_file(filename, fil_in, buf, + page_size.is_compressed(), pos, + static_cast<ulong>(page_size.physical()))) { + + exit_status = 1; + } + + return (exit_status); +} + int main( int argc, char **argv) @@ -1511,15 +1533,12 @@ int main( /* our input filename. */ char* filename; /* Buffer to store pages read. */ + byte* buf_ptr = NULL; + byte* xdes_ptr = NULL; byte* buf = NULL; - /* Buffer for xdes */ byte* xdes = NULL; /* bytes read count */ ulong bytes; - /* Buffer to decompress page.*/ -#ifdef MYSQL_COMPRESSION - byte* tbuf = NULL; -#endif /* current time */ time_t now; /* last time */ @@ -1531,6 +1550,8 @@ int main( struct stat st; #endif /* _WIN32 */ + int exit_status = 0; + /* size of file (has to be 64 bits) */ unsigned long long int size = 0; /* number of pages in file */ @@ -1538,9 +1559,7 @@ int main( off_t offset = 0; /* count the no. of page corrupted. */ - ulint mismatch_count = 0; - /* Variable to ack the page is corrupted or not. */ - bool is_corrupted = false; + unsigned long long mismatch_count = 0; bool partial_page_read = false; /* Enabled when read from stdin is done. */ @@ -1560,32 +1579,37 @@ int main( DBUG_PROCESS(argv[0]); if (get_options(&argc,&argv)) { - DBUG_RETURN(1); + exit_status = 1; + goto my_exit; } if (strict_verify && no_check) { fprintf(stderr, "Error: --strict-check option cannot be used " "together with --no-check option.\n"); - DBUG_RETURN(1); + exit_status = 1; + goto my_exit; } if (no_check && !do_write) { fprintf(stderr, "Error: --no-check must be associated with " "--write option.\n"); - DBUG_RETURN(1); + exit_status = 1; + goto my_exit; } if (page_type_dump) { fil_page_type = create_file(page_dump_filename); if (!fil_page_type) { - DBUG_RETURN(1); + exit_status = 1; + goto my_exit; } } if (is_log_enabled) { log_file = create_file(log_filename); if (!log_file) { - DBUG_RETURN(1); + exit_status = 1; + goto my_exit; } fprintf(log_file, "InnoDB File Checksum Utility.\n"); } @@ -1595,17 +1619,17 @@ int main( } - buf = (byte*) malloc(UNIV_PAGE_SIZE_MAX * 2); - xdes = (byte *) malloc(UNIV_PAGE_SIZE_MAX *2); -#ifdef MYSQL_COMPRESSION - tbuf = buf + UNIV_PAGE_SIZE_MAX; -#endif + buf_ptr = (byte*) malloc(UNIV_PAGE_SIZE_MAX * 2); + xdes_ptr = (byte*)malloc(UNIV_PAGE_SIZE_MAX * 2); + buf = (byte *) ut_align(buf_ptr, UNIV_PAGE_SIZE_MAX); + xdes = (byte *) ut_align(xdes_ptr, UNIV_PAGE_SIZE_MAX); + /* The file name is not optional. */ for (int i = 0; i < argc; ++i) { + /* Reset parameters for each file. */ filename = argv[i]; memset(&page_type, 0, sizeof(innodb_page_type)); - is_corrupted = false; partial_page_read = false; skip_page = false; @@ -1630,7 +1654,8 @@ int main( fprintf(stderr, "Error: %s cannot be found\n", filename); - goto err_exit; + exit_status = 1; + goto my_exit; } if (!read_from_stdin) { @@ -1638,35 +1663,17 @@ int main( fil_in = open_file(filename); /*If fil_in is NULL, terminate as some error encountered */ if(fil_in == NULL) { - goto err_exit; + exit_status = 1; + goto my_exit; } /* Save the current file pointer in pos variable.*/ if (0 != fgetpos(fil_in, &pos)) { perror("fgetpos"); - goto err_exit; + exit_status = 1; + goto my_exit; } } - /* Testing for lock mechanism. The innochecksum - acquire lock on given file. So other tools accessing the same - file for processsing must fail. */ -#ifdef _WIN32 - DBUG_EXECUTE_IF("innochecksum_cause_mysqld_crash", - ut_ad(page_dump_filename); - while((_access( page_dump_filename, 0)) == 0) { - sleep(1); - } - DBUG_RETURN(0); ); -#else - DBUG_EXECUTE_IF("innochecksum_cause_mysqld_crash", - ut_ad(page_dump_filename); - struct stat status_buf; - while(stat(page_dump_filename, &status_buf) == 0) { - sleep(1); - } - DBUG_RETURN(0); ); -#endif /* _WIN32 */ - /* Read the minimum page size. */ bytes = ulong(fread(buf, 1, UNIV_ZIP_SIZE_MIN, fil_in)); partial_page_read = true; @@ -1674,10 +1681,11 @@ int main( if (bytes != UNIV_ZIP_SIZE_MIN) { fprintf(stderr, "Error: Was not able to read the " "minimum page size "); - fprintf(stderr, "of %d bytes. Bytes read was %lu\n", + fprintf(stderr, "of %d bytes. Bytes read was " ULINTPF "\n", UNIV_ZIP_SIZE_MIN, bytes); - goto err_exit; + exit_status = 1; + goto my_exit; } /* enable variable is_system_tablespace when space_id of given @@ -1687,8 +1695,79 @@ int main( FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, 4)) ? true : false; + /* Determine page size, zip_size and page compression + from fsp_flags and encryption metadata from page 0 */ const page_size_t& page_size = get_page_size(buf); + ulint flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf); + ulint zip_size = page_size.is_compressed() ? page_size.logical() : 0; + logical_page_size = page_size.is_compressed() ? zip_size : 0; + physical_page_size = page_size.physical(); + srv_page_size = page_size.logical(); + bool is_compressed = FSP_FLAGS_HAS_PAGE_COMPRESSION(flags); + + if (page_size.physical() > UNIV_ZIP_SIZE_MIN) { + /* Read rest of the page 0 to determine crypt_data */ + bytes = ulong(read_file(buf, partial_page_read, page_size.physical(), fil_in)); + + if (bytes != page_size.physical()) { + fprintf(stderr, "Error: Was not able to read the " + "rest of the page "); + fprintf(stderr, "of " ULINTPF " bytes. Bytes read was " ULINTPF "\n", + page_size.physical() - UNIV_ZIP_SIZE_MIN, bytes); + + exit_status = 1; + goto my_exit; + } + partial_page_read = false; + } + + /* Now that we have full page 0 in buffer, check encryption */ + bool is_encrypted = check_encryption(filename, page_size, buf); + + /* Verify page 0 contents. Note that we can't allow + checksum mismatch on page 0, because that would mean we + could not trust it content. */ + if (!no_check) { + unsigned long long tmp_allow_mismatches = allow_mismatches; + allow_mismatches = 0; + + exit_status = verify_checksum(buf, page_size, is_encrypted, is_compressed, &mismatch_count); + + if (exit_status) { + fprintf(stderr, "Error: Page 0 checksum mismatch, can't continue. \n"); + goto my_exit; + } + allow_mismatches = tmp_allow_mismatches; + } + + if ((exit_status = rewrite_checksum(filename, fil_in, buf, + page_size, &pos, is_encrypted, is_compressed))) { + goto my_exit; + } + + if (page_type_dump) { + fprintf(fil_page_type, + "\n\nFilename::%s\n", filename); + fprintf(fil_page_type, + "========================================" + "======================================\n"); + fprintf(fil_page_type, + "\tPAGE_NO\t\t|\t\tPAGE_TYPE\t\t" + "\t|\tEXTRA INFO\n"); + fprintf(fil_page_type, + "========================================" + "======================================\n"); + } + + if (per_page_details) { + printf("page %llu ", cur_page_num); + } + + if (page_type_summary || page_type_dump) { + parse_page(buf, xdes, fil_page_type, page_size, is_encrypted); + } + pages = (ulint) (size / page_size.physical()); if (just_count) { @@ -1704,14 +1783,14 @@ int main( "(" ULINTPF " pages)\n", filename, size, pages); if (do_one_page) { fprintf(log_file, "Innochecksum: " - "checking page %" PRIuMAX "\n", + "checking page::%llu;\n", do_page); } } } else { if (is_log_enabled) { fprintf(log_file, "Innochecksum: checking " - "pages in range %" PRIuMAX " to %" PRIuMAX "\n", + "pages in range::%llu to %llu\n", start_page, use_end_page ? end_page : (pages - 1)); } @@ -1735,14 +1814,16 @@ int main( perror("Error: Unable to seek to " "necessary offset"); - goto err_exit; + exit_status = 1; + goto my_exit; } /* Save the current file pointer in pos variable. */ if (0 != fgetpos(fil_in, &pos)) { perror("fgetpos"); - goto err_exit; + exit_status = 1; + goto my_exit; } } else { @@ -1762,7 +1843,7 @@ int main( bytes = read_file(buf, partial_page_read, static_cast<ulong>( - page_size.physical()), + page_size.physical()), fil_in); partial_page_read = false; @@ -1773,34 +1854,22 @@ int main( "to seek to necessary " "offset"); - goto err_exit; + exit_status = 1; + goto my_exit; } } } } - if (page_type_dump) { - fprintf(fil_page_type, - "\n\nFilename::%s\n", filename); - fprintf(fil_page_type, - "========================================" - "======================================\n"); - fprintf(fil_page_type, - "\tPAGE_NO\t\t|\t\tPAGE_TYPE\t\t" - "\t|\tEXTRA INFO\n"); - fprintf(fil_page_type, - "========================================" - "======================================\n"); - } - /* main checksumming loop */ - cur_page_num = start_page; + cur_page_num = start_page ? start_page : cur_page_num + 1; + lastt = 0; while (!feof(fil_in)) { bytes = read_file(buf, partial_page_read, static_cast<ulong>( - page_size.physical()), fil_in); + page_size.physical()), fil_in); partial_page_read = false; if (!bytes && feof(fil_in)) { @@ -1812,14 +1881,16 @@ int main( page_size.physical()); perror(" "); - goto err_exit; + exit_status = 1; + goto my_exit; } if (bytes != page_size.physical()) { - fprintf(stderr, "Error: bytes read (%lu) " + fprintf(stderr, "Error: bytes read (" ULINTPF ") " "doesn't match page size (" ULINTPF ")\n", bytes, page_size.physical()); - goto err_exit; + exit_status = 1; + goto my_exit; } if (is_system_tablespace) { @@ -1827,60 +1898,29 @@ int main( skip_page = is_page_doublewritebuffer(buf); } else { skip_page = false; -#ifdef MYSQL_COMPRESSION - if (!page_decompress(buf, tbuf, page_size)) { - - fprintf(stderr, - "Page decompress failed"); - - goto err_exit; - } -#endif } - ulint page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); + ulint cur_page_type = mach_read_from_2(buf+FIL_PAGE_TYPE); - if (page_type == FIL_PAGE_PAGE_COMPRESSED || - page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + /* FIXME: Page compressed or Page compressed and encrypted + pages do not contain checksum. */ + if (cur_page_type == FIL_PAGE_PAGE_COMPRESSED || + cur_page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { skip_page = true; } /* If no-check is enabled, skip the checksum verification.*/ - if (!no_check) { - /* Checksum verification */ - if (!skip_page) { - is_corrupted = is_page_corrupted( - buf, page_size); - - if (is_corrupted) { - fprintf(stderr, "Fail: page " - "%" PRIuMAX " invalid\n", - cur_page_num); - - mismatch_count++; - - if(mismatch_count > allow_mismatches) { - fprintf(stderr, - "Exceeded the " - "maximum allowed " - "checksum mismatch " - "count::%" PRIuMAX "\n", - allow_mismatches); - - goto err_exit; - } - } - } + if (!no_check + && !skip_page + && (exit_status = verify_checksum(buf, page_size, + is_encrypted, is_compressed, &mismatch_count))) { + goto my_exit; } - /* Rewrite checksum */ - if (do_write - && !write_file(filename, fil_in, buf, - page_size.is_compressed(), &pos, - static_cast<ulong>(page_size.physical()))) { - - goto err_exit; + if ((exit_status = rewrite_checksum(filename, fil_in, buf, + page_size, &pos, is_encrypted, is_compressed))) { + goto my_exit; } /* end if this was the last page we were supposed to check */ @@ -1888,12 +1928,17 @@ int main( break; } + if (per_page_details) { + printf("page %llu ", cur_page_num); + } + if (page_type_summary || page_type_dump) { - parse_page(buf, xdes , fil_page_type, page_size); + parse_page(buf, xdes, fil_page_type, page_size, is_encrypted); } /* do counter increase and progress printing */ cur_page_num++; + if (verbose && !read_from_stdin) { if ((cur_page_num % 64) == 0) { now = time(0); @@ -1902,7 +1947,7 @@ int main( } if (now - lastt >= 1 && is_log_enabled) { - fprintf(log_file, "page %" PRIuMAX " " + fprintf(log_file, "page::%llu " "okay: %.3f%% done\n", (cur_page_num - 1), (float) cur_page_num / pages * 100); @@ -1933,19 +1978,19 @@ int main( fclose(log_file); } - free(buf); - free(xdes); - my_end(0); + free(buf_ptr); + free(xdes_ptr); - DBUG_RETURN(0); + my_end(exit_status); + DBUG_RETURN(exit_status); -err_exit: - if (buf) { - free(buf); +my_exit: + if (buf_ptr) { + free(buf_ptr); } - if (xdes) { - free(xdes); + if (xdes_ptr) { + free(xdes_ptr); } if (!read_from_stdin && fil_in) { @@ -1956,7 +2001,6 @@ err_exit: fclose(log_file); } - my_end(1); - - DBUG_RETURN(1); + my_end(exit_status); + DBUG_RETURN(exit_status); } diff --git a/mysql-test/suite/encryption/r/innochecksum.result b/mysql-test/suite/encryption/r/innochecksum.result index 7cd7af7b93b..6ea54f3d053 100644 --- a/mysql-test/suite/encryption/r/innochecksum.result +++ b/mysql-test/suite/encryption/r/innochecksum.result @@ -1,3 +1,5 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; # Create and populate a tables CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; @@ -5,6 +7,7 @@ CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FOR CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO; CREATE TABLE t4 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1; CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4; +CREATE TABLE t6 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB; # Write file to make mysql-test-run.pl expect the "crash", but don't # start it until it's told to # We give 30 seconds to do a clean shutdown because we do not want @@ -16,6 +19,27 @@ CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_CO # Run innochecksum on t3 # Run innochecksum on t4 # Run innochecksum on t4 +# Run innochecksum on t5 +# Run innochecksum on t6 +# Backup tables before corrupting +# Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION +# Run innochecksum on t2 +# Run innochecksum on t3 +# no encryption corrupting the field should not have effect +# Run innochecksum on t6 +# no encryption corrupting the field should not have effect +# Restore the original tables +# Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4 (post encryption checksum) +# Run innochecksum on t2 +# Run innochecksum on t3 +# Run innochecksum on t6 +# no encryption corrupting the field should not have effect +# Restore the original tables +# Corrupt FIL_DATA+10 (data) +# Run innochecksum on t2 +# Run innochecksum on t3 +# Run innochecksum on t6 +# Restore the original tables # Write file to make mysql-test-run.pl start up the server again # Cleanup -DROP TABLE t1, t2, t3, t4, t5; +DROP TABLE t1, t2, t3, t4, t5, t6; diff --git a/mysql-test/suite/encryption/t/innochecksum.test b/mysql-test/suite/encryption/t/innochecksum.test index cb1b97ebfb3..6e3962c462f 100644 --- a/mysql-test/suite/encryption/t/innochecksum.test +++ b/mysql-test/suite/encryption/t/innochecksum.test @@ -2,21 +2,25 @@ # MDEV-8773: InnoDB innochecksum does not work with encrypted or page compressed tables # -# Don't test under embedded +--source include/innodb_page_size_small.inc +# Don't test under embedded as we restart server -- source include/not_embedded.inc # Require InnoDB -- source include/have_innodb.inc -- source include/have_file_key_management_plugin.inc +-- source include/innodb_page_size_small.inc if (!$INNOCHECKSUM) { --echo Need innochecksum binary --die Need innochecksum binary } -let $innodb_compression_algorithm_orig=`SELECT @@innodb_compression_algorithm`; - +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; # zlib set global innodb_compression_algorithm = 1; +--enable_warnings --echo # Create and populate a tables CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; @@ -24,9 +28,11 @@ CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FOR CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO; CREATE TABLE t4 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1; CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4; +CREATE TABLE t6 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB; --disable_query_log --let $i = 1000 +begin; while ($i) { INSERT INTO t1 (b) VALUES (REPEAT('abcdefghijklmnopqrstuvwxyz', 100)); @@ -36,6 +42,8 @@ INSERT INTO t2 SELECT * FROM t1; INSERT INTO t3 SELECT * FROM t1; INSERT INTO t4 SELECT * FROM t1; INSERT INTO t5 SELECT * FROM t1; +INSERT INTO t6 SELECT * FROM t1; +commit; --enable_query_log let $MYSQLD_DATADIR=`select @@datadir`; @@ -44,6 +52,10 @@ let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd; let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd; let t4_IBD = $MYSQLD_DATADIR/test/t4.ibd; let t5_IBD = $MYSQLD_DATADIR/test/t5.ibd; +let t6_IBD = $MYSQLD_DATADIR/test/t6.ibd; + +let INNODB_PAGE_SIZE=`select @@innodb_page_size`; +let MYSQLD_DATADIR=`select @@datadir`; --echo # Write file to make mysql-test-run.pl expect the "crash", but don't --echo # start it until it's told to @@ -77,17 +89,198 @@ shutdown_server 30; --exec $INNOCHECKSUM $t4_IBD +--echo # Run innochecksum on t5 + +--exec $INNOCHECKSUM $t5_IBD + +--echo # Run innochecksum on t6 + +--exec $INNOCHECKSUM $t6_IBD + +--enable_result_log + +--echo # Backup tables before corrupting +--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_DATADIR/test/t1.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t2.ibd $MYSQLD_DATADIR/test/t2.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t3.ibd $MYSQLD_DATADIR/test/t3.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t4.ibd $MYSQLD_DATADIR/test/t4.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t5.ibd $MYSQLD_DATADIR/test/t5.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t6.ibd $MYSQLD_DATADIR/test/t6.ibd.backup + +# +# MDEV-11939: innochecksum mistakes a file for an encrypted one +# + +--echo # Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + +perl; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +EOF + +-- disable_result_log +--error 1 +--exec $INNOCHECKSUM $t1_IBD + +--echo # Run innochecksum on t2 + +--error 1 +--exec $INNOCHECKSUM $t2_IBD + +--echo # Run innochecksum on t3 +--echo # no encryption corrupting the field should not have effect +--exec $INNOCHECKSUM $t3_IBD + +--echo # Run innochecksum on t6 +--echo # no encryption corrupting the field should not have effect +--exec $INNOCHECKSUM $t6_IBD + --enable_result_log +--echo # Restore the original tables +--remove_file $MYSQLD_DATADIR/test/t1.ibd +--remove_file $MYSQLD_DATADIR/test/t2.ibd +--remove_file $MYSQLD_DATADIR/test/t3.ibd +--remove_file $MYSQLD_DATADIR/test/t4.ibd +--remove_file $MYSQLD_DATADIR/test/t5.ibd +--remove_file $MYSQLD_DATADIR/test/t6.ibd +--copy_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd +--copy_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd +--copy_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd +--copy_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd +--copy_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd +--copy_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd + +--echo # Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4 (post encryption checksum) + +perl; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +EOF + +-- disable_result_log +--error 1 +--exec $INNOCHECKSUM $t1_IBD + +--echo # Run innochecksum on t2 +--error 1 +--exec $INNOCHECKSUM $t2_IBD + +--echo # Run innochecksum on t3 +--error 1 +--exec $INNOCHECKSUM $t3_IBD + +--echo # Run innochecksum on t6 +--echo # no encryption corrupting the field should not have effect +--exec $INNOCHECKSUM $t6_IBD + +--enable_result_log + +--echo # Restore the original tables +--remove_file $MYSQLD_DATADIR/test/t1.ibd +--remove_file $MYSQLD_DATADIR/test/t2.ibd +--remove_file $MYSQLD_DATADIR/test/t3.ibd +--remove_file $MYSQLD_DATADIR/test/t4.ibd +--remove_file $MYSQLD_DATADIR/test/t5.ibd +--remove_file $MYSQLD_DATADIR/test/t6.ibd +--copy_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd +--copy_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd +--copy_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd +--copy_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd +--copy_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd +--copy_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd + +--echo # Corrupt FIL_DATA+10 (data) + +perl; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +EOF + +-- disable_result_log +--error 1 +--exec $INNOCHECKSUM $t1_IBD + +--echo # Run innochecksum on t2 +--error 1 +--exec $INNOCHECKSUM $t2_IBD + +--echo # Run innochecksum on t3 +--error 1 +--exec $INNOCHECKSUM $t3_IBD + +--echo # Run innochecksum on t6 +--error 1 +--exec $INNOCHECKSUM $t6_IBD + +--enable_result_log + +--echo # Restore the original tables +--move_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd +--move_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd +--move_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd +--move_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd +--move_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd +--move_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd + --echo # Write file to make mysql-test-run.pl start up the server again --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --enable_reconnect --source include/wait_until_connected_again.inc --echo # Cleanup -DROP TABLE t1, t2, t3, t4, t5; - -# reset system ---disable_query_log -EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig; ---enable_query_log +DROP TABLE t1, t2, t3, t4, t5, t6; diff --git a/mysql-test/suite/innodb_zip/r/innochecksum_2.result b/mysql-test/suite/innodb_zip/r/innochecksum_2.result index 78d8a1cbf90..582bb42f0cb 100644 --- a/mysql-test/suite/innodb_zip/r/innochecksum_2.result +++ b/mysql-test/suite/innodb_zip/r/innochecksum_2.result @@ -31,6 +31,7 @@ allow-mismatches 0 write crc32 page-type-summary FALSE page-type-dump MYSQLTEST_VARDIR/tmp/dump.txt +per-page-details FALSE log (No default value) leaf FALSE merge 0 @@ -41,7 +42,7 @@ innochecksum Ver #.#.# Copyright (c) YEAR, YEAR , Oracle, MariaDB Corporation Ab and others. InnoDB offline file checksum utility. -Usage: innochecksum [-c] [-s <start page>] [-e <end page>] [-p <page>] [-v] [-a <allow mismatches>] [-n] [-C <strict-check>] [-w <write>] [-S] [-D <page type dump>] [-l <log>] [-e] <filename or [-]> +Usage: innochecksum [-c] [-s <start page>] [-e <end page>] [-p <page>] [-i] [-v] [-a <allow mismatches>] [-n] [-C <strict-check>] [-w <write>] [-S] [-D <page type dump>] [-l <log>] [-l] [-m <merge pages>] <filename or [-]> -?, --help Displays this help and exits. -I, --info Synonym for --help. -V, --version Displays version information and exits. @@ -62,8 +63,10 @@ Usage: innochecksum [-c] [-s <start page>] [-e <end page>] [-p <page>] [-v] [-a Display a count of each page type in a tablespace. -D, --page-type-dump=name Dump the page type info for each page in a tablespace. + -i, --per-page-details + Print out per-page detail information. -l, --log=name log output. - -e, --leaf Examine leaf index pages + -f, --leaf Examine leaf index pages -m, --merge=# leaf page count if merge given number of consecutive pages @@ -81,6 +84,7 @@ allow-mismatches 0 write crc32 page-type-summary FALSE page-type-dump (No default value) +per-page-details FALSE log (No default value) leaf FALSE merge 0 diff --git a/mysql-test/suite/innodb_zip/r/innochecksum_3.result b/mysql-test/suite/innodb_zip/r/innochecksum_3.result index 800556c4ff3..aaab68b3df9 100644 --- a/mysql-test/suite/innodb_zip/r/innochecksum_3.result +++ b/mysql-test/suite/innodb_zip/r/innochecksum_3.result @@ -1,5 +1,6 @@ # Set the environmental variables call mtr.add_suppression("InnoDB: Unable to read tablespace .* page no .* into the buffer pool after 100 attempts"); +call mtr.add_suppression("InnoDB: innodb_checksum_algorithm is set to.*"); [1]: Further Test are for rewrite checksum (innodb|crc32|none) for all ibd file & start the server. CREATE TABLE tab1 (pk INTEGER NOT NULL PRIMARY KEY, linestring_key GEOMETRY NOT NULL, @@ -105,6 +106,7 @@ File::tab#.ibd # Page compressed page # Page compressed encrypted page # Other type of page + =============================================== Additional information: Undo page type: # insert, # update, # other @@ -139,6 +141,7 @@ File::tab#.ibd # Page compressed page # Page compressed encrypted page # Other type of page + =============================================== Additional information: Undo page type: # insert, # update, # other @@ -160,14 +163,14 @@ Filename::tab#.ibd ============================================================================== PAGE_NO | PAGE_TYPE | EXTRA INFO ============================================================================== -#:: # | File Space Header | - -#:: # | Insert Buffer Bitmap | - -#:: # | Inode page | - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Freshly allocated page | - +#::# | File Space Header | - +#::# | Insert Buffer Bitmap | - +#::# | Inode page | - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Freshly allocated page | - # Variables used by page type dump for ibdata1 Variables (--variable-name=value) @@ -184,6 +187,7 @@ allow-mismatches 0 write crc32 page-type-summary FALSE page-type-dump MYSQLTEST_VARDIR/tmp/dump.txt +per-page-details FALSE log (No default value) leaf FALSE merge 0 @@ -194,14 +198,14 @@ Filename::tab#.ibd ============================================================================== PAGE_NO | PAGE_TYPE | EXTRA INFO ============================================================================== -#:: # | File Space Header | - -#:: # | Insert Buffer Bitmap | - -#:: # | Inode page | - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Freshly allocated page | - +#::# | File Space Header | - +#::# | Insert Buffer Bitmap | - +#::# | Inode page | - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Freshly allocated page | - [6]: check the valid lower bound values for option # allow-mismatches,page,start-page,end-page [9]: check the both short and long options "page" and "start-page" when diff --git a/mysql-test/suite/innodb_zip/t/innochecksum_3.test b/mysql-test/suite/innodb_zip/t/innochecksum_3.test index 54c67ff1a9b..dab10dcc997 100644 --- a/mysql-test/suite/innodb_zip/t/innochecksum_3.test +++ b/mysql-test/suite/innodb_zip/t/innochecksum_3.test @@ -15,7 +15,9 @@ let MYSQLD_BASEDIR= `SELECT @@basedir`; let MYSQLD_DATADIR= `SELECT @@datadir`; let SEARCH_FILE= $MYSQLTEST_VARDIR/log/my_restart.err; + call mtr.add_suppression("InnoDB: Unable to read tablespace .* page no .* into the buffer pool after 100 attempts"); +call mtr.add_suppression("InnoDB: innodb_checksum_algorithm is set to.*"); --echo [1]: Further Test are for rewrite checksum (innodb|crc32|none) for all ibd file & start the server. diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 61006c8d89d..d1476cca67f 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -31,16 +31,17 @@ The database buffer buf_pool Created 11/5/1995 Heikki Tuuri *******************************************************/ -#include "ha_prototypes.h" - +#include "univ.i" +#include "mtr0types.h" +#include "mach0data.h" #include "page0size.h" #include "buf0buf.h" -#include "os0api.h" +#include <string.h> + +#ifdef UNIV_NONINL +#include "buf0buf.ic" +#endif -#ifdef UNIV_INNOCHECKSUM -#include "string.h" -#include "mach0data.h" -#endif /* UNIV_INNOCHECKSUM */ #ifndef UNIV_INNOCHECKSUM #include "mem0mem.h" #include "btr0btr.h" @@ -64,7 +65,6 @@ Created 11/5/1995 Heikki Tuuri #include "fsp0sysspace.h" #endif /* !UNIV_INNOCHECKSUM */ #include "page0zip.h" -#include "buf0checksum.h" #include "sync0sync.h" #include "buf0dump.h" #include "ut0new.h" @@ -645,10 +645,10 @@ buf_page_is_checksum_valid_crc32( #ifdef UNIV_INNOCHECKSUM if (log_file && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " crc32 calculated = %u;" - " recorded checksum field1 = %lu recorded" - " checksum field2 =%lu\n", cur_page_num, + " recorded checksum field1 = " ULINTPF " recorded" + " checksum field2 =" ULINTPF "\n", cur_page_num, crc32, checksum_field1, checksum_field2); } #endif /* UNIV_INNOCHECKSUM */ @@ -702,33 +702,34 @@ buf_page_is_checksum_valid_innodb( #ifdef UNIV_INNOCHECKSUM if (log_file && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB) { - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " old style: calculated =" - " %lu; recorded = %lu\n", + " " ULINTPF "; recorded = " ULINTPF "\n", cur_page_num, old_checksum, checksum_field2); - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " new style: calculated =" - " %lu; crc32 = %u; recorded = %lu\n", + " " ULINTPF "; crc32 = %u; recorded = " ULINTPF "\n", cur_page_num, new_checksum, buf_calc_page_crc32(read_buf), checksum_field1); } if (log_file && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " old style: calculated =" - " %lu; recorded checksum = %lu\n", + " " ULINTPF "; recorded checksum = " ULINTPF "\n", cur_page_num, old_checksum, checksum_field2); - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " new style: calculated =" - " %lu; recorded checksum = %lu\n", + " " ULINTPF "; recorded checksum = " ULINTPF "\n", cur_page_num, new_checksum, checksum_field1); } #endif /* UNIV_INNOCHECKSUM */ + if (checksum_field2 != mach_read_from_4(read_buf + FIL_PAGE_LSN) && checksum_field2 != old_checksum) { DBUG_LOG("checksum", @@ -788,9 +789,9 @@ buf_page_is_checksum_valid_none( if (log_file && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_NONE) { fprintf(log_file, - "page::%lu; none checksum: calculated" - " = %lu; recorded checksum_field1 = %lu" - " recorded checksum_field2 = %lu\n", + "page::%llu; none checksum: calculated" + " = " ULINTPF "; recorded checksum_field1 = " ULINTPF + " recorded checksum_field2 = " ULINTPF "\n", cur_page_num, BUF_NO_CHECKSUM_MAGIC, checksum_field1, checksum_field2); } @@ -811,16 +812,18 @@ buf_page_is_corrupted( bool check_lsn, const byte* read_buf, const page_size_t& page_size, +#ifndef UNIV_INNOCHECKSUM const fil_space_t* space) +#else + const void* space) +#endif { - ulint checksum_field1; - ulint checksum_field2; - + size_t checksum_field1 = 0; + size_t checksum_field2 = 0; #ifndef UNIV_INNOCHECKSUM DBUG_EXECUTE_IF("buf_page_import_corrupt_failure", return(true); ); - - ulint page_type = mach_read_from_2( - read_buf + FIL_PAGE_TYPE); +#endif + ulint page_type = mach_read_from_2(read_buf + FIL_PAGE_TYPE); /* We can trust page type if page compression is set on tablespace flags because page compression flag means file must have been @@ -833,15 +836,12 @@ buf_page_is_corrupted( decompressed at this stage). */ if ((page_type == FIL_PAGE_PAGE_COMPRESSED || page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) - && space && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags)) { - return(false); - } -#else - if (mach_read_from_4(read_buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0 - || mach_read_from_2(read_buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { +#ifndef UNIV_INNOCHECKSUM + && space && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags) +#endif + ) { return(false); } -#endif if (!page_size.is_compressed() && memcmp(read_buf + FIL_PAGE_LSN + 4, @@ -850,14 +850,13 @@ buf_page_is_corrupted( /* Stored log sequence numbers at the start and the end of page do not match */ - #ifndef UNIV_INNOCHECKSUM ib::info() << "Log sequence number at the start " << mach_read_from_4(read_buf + FIL_PAGE_LSN + 4) << " and the end " << mach_read_from_4(read_buf + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + 4) << " do not match"; -#endif +#endif /* UNIV_INNOCHECKSUM */ return(true); } @@ -920,10 +919,9 @@ buf_page_is_corrupted( && *reinterpret_cast<const ib_uint64_t*>( read_buf + FIL_PAGE_LSN) == 0) { - /* make sure that the page is really empty */ - - ulint i; + ulint i; + /* make sure that the page is really empty */ for (i = 0; i < page_size.logical(); ++i) { /* The FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID has been @@ -944,7 +942,7 @@ buf_page_is_corrupted( #ifdef UNIV_INNOCHECKSUM if (i >= page_size.logical()) { if (log_file) { - fprintf(log_file, "Page::%lu" + fprintf(log_file, "Page::%llu" " is empty and uncorrupted\n", cur_page_num); } @@ -978,31 +976,34 @@ buf_page_is_corrupted( if (buf_page_is_checksum_valid_none(read_buf, checksum_field1, checksum_field2)) { -#ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_NONE, page_id); +#endif /* !UNIV_INNOCHECKSUM */ } -#else /* !UNIV_INNOCHECKSUM */ + +#ifdef UNIV_INNOCHECKSUM if (log_file) { - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " old style: calculated = %u;" - " recorded = " ULINTPF "\n", + " recorded = " ULINTPF ";\n", cur_page_num, buf_calc_page_old_checksum(read_buf), checksum_field2); - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " new style: calculated = %u;" - " crc32 = %u; recorded = " ULINTPF "\n", + " crc32 = %u; recorded = " ULINTPF ";\n", cur_page_num, buf_calc_page_new_checksum(read_buf), buf_calc_page_crc32(read_buf), checksum_field1); } -#endif /* !UNIV_INNOCHECKSUM */ +#endif /* UNIV_INNOCHECKSUM */ + return(false); } @@ -1022,15 +1023,16 @@ buf_page_is_corrupted( if (buf_page_is_checksum_valid_innodb(read_buf, checksum_field1, checksum_field2)) { -#ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_INNODB, page_id); +#endif } -#endif /* !UNIV_INNOCHECKSUM */ + return(false); } @@ -1038,13 +1040,13 @@ buf_page_is_corrupted( if (!legacy_checksum_checked && buf_page_is_checksum_valid_crc32( read_buf, checksum_field1, checksum_field2, true)) { - legacy_big_endian_checksum = true; - return(false); + legacy_big_endian_checksum = true; + return(false); } #ifdef UNIV_INNOCHECKSUM if (log_file) { - fprintf(log_file, "Fail; page %lu" + fprintf(log_file, "Fail; page::%llu;" " invalid (fails crc32 checksum)\n", cur_page_num); } @@ -1061,30 +1063,32 @@ buf_page_is_corrupted( if (buf_page_is_checksum_valid_none(read_buf, checksum_field1, checksum_field2)) { -#ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_NONE, page_id); +#endif } -#else /* !UNIV_INNOCHECKSUM */ +#ifdef UNIV_INNOCHECKSUM if (log_file) { - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " old style: calculated = %u;" - " recorded = %lu\n", cur_page_num, + " recorded = %zu;\n", cur_page_num, buf_calc_page_old_checksum(read_buf), checksum_field2); - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " new style: calculated = %u;" - " crc32 = %u; recorded = %lu\n", + " crc32 = %u; recorded = %zu;\n", cur_page_num, buf_calc_page_new_checksum(read_buf), buf_calc_page_crc32(read_buf), checksum_field1); } -#endif /* !UNIV_INNOCHECKSUM */ +#endif /* UNIV_INNOCHECKSUM */ + return(false); } @@ -1092,26 +1096,28 @@ buf_page_is_corrupted( checksum_field1, checksum_field2, false) || buf_page_is_checksum_valid_crc32(read_buf, checksum_field1, checksum_field2, true)) { -#ifndef UNIV_INNOCHECKSUM + if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_CRC32, page_id); +#endif } -#endif /* !UNIV_INNOCHECKSUM */ return(false); } #ifdef UNIV_INNOCHECKSUM if (log_file) { - fprintf(log_file, "Fail; page %lu" + fprintf(log_file, "Fail; page::%llu;" " invalid (fails innodb checksum)\n", cur_page_num); } #endif /* UNIV_INNOCHECKSUM */ + return(true); case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: @@ -1147,11 +1153,12 @@ buf_page_is_corrupted( #ifdef UNIV_INNOCHECKSUM if (log_file) { - fprintf(log_file, "Fail; page %lu" + fprintf(log_file, "Fail; page::%llu;" " invalid (fails none checksum)\n", cur_page_num); } #endif /* UNIV_INNOCHECKSUM */ + return(true); case SRV_CHECKSUM_ALGORITHM_NONE: diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 7917cbb528b..6e431a6ee0f 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -24,6 +24,7 @@ Modified Jan Lindström jan.lindstrom@mariadb.com *******************************************************/ #include "fil0fil.h" +#include "mtr0types.h" #include "mach0data.h" #include "page0size.h" #include "page0zip.h" diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 9d5f373f5de..67de1174d25 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -211,6 +211,7 @@ struct buf_pools_list_size_t { ulint unzip_LRU_bytes; /*!< unzip_LRU size in bytes */ ulint flush_list_bytes; /*!< flush_list size in bytes */ }; +#endif /* !UNIV_INNOCHECKSUM */ /** Page identifier. */ class page_id_t { @@ -333,6 +334,7 @@ operator<<( std::ostream& out, const page_id_t& page_id); +#ifndef UNIV_INNOCHECKSUM /********************************************************************//** Acquire mutex on all buffer pool instances */ UNIV_INLINE @@ -823,9 +825,16 @@ buf_page_is_corrupted( bool check_lsn, const byte* read_buf, const page_size_t& page_size, +#ifndef UNIV_INNOCHECKSUM const fil_space_t* space = NULL) +#else + const void* space = NULL) +#endif MY_ATTRIBUTE((warn_unused_result)); + + #ifndef UNIV_INNOCHECKSUM + /**********************************************************************//** Gets the space id, page offset, and byte offset within page of a pointer pointing to a buffer frame containing a file page. */ diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index 06785430c25..13b3ec4e37e 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -30,6 +30,7 @@ Created 04/01/2015 Jan Lindström #include "os0event.h" #include "my_crypt.h" +#endif /*! UNIV_INNOCHECKSUM */ /** * Magic pattern in start of crypt data on page 0 @@ -101,6 +102,8 @@ struct fil_space_rotate_state_t } scrubbing; }; +#ifndef UNIV_INNOCHECKSUM + struct fil_space_crypt_t : st_encryption_scheme { public: diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index bf231565657..097be6a5f96 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -28,8 +28,6 @@ Created 10/25/1995 Heikki Tuuri #define fil0fil_h #include "univ.i" -struct fil_space_t; - #ifndef UNIV_INNOCHECKSUM #include "log0recv.h" @@ -281,22 +279,23 @@ but in the MySQL Embedded Server Library and mysqlbackup it is not the default directory, and we must set the base file path explicitly */ extern const char* fil_path_to_mysql_datadir; -/** Initial size of a single-table tablespace in pages */ -#define FIL_IBD_FILE_INITIAL_SIZE 4 - -/** 'null' (undefined) page offset in the context of file spaces */ -#define FIL_NULL ULINT32_UNDEFINED - /* Space address data type; this is intended to be used when addresses accurate to a byte are stored in file pages. If the page part of the address is FIL_NULL, the address is considered undefined. */ typedef byte fil_faddr_t; /*!< 'type' definition in C: an address stored in a file page is a string of bytes */ +#endif /* !UNIV_INNOCHECKSUM */ + +/** Initial size of a single-table tablespace in pages */ +#define FIL_IBD_FILE_INITIAL_SIZE 4 + +/** 'null' (undefined) page offset in the context of file spaces */ +#define FIL_NULL ULINT32_UNDEFINED + #define FIL_ADDR_PAGE 0 /* first in address is the page offset */ #define FIL_ADDR_BYTE 4 /* then comes 2-byte byte offset within page*/ -#endif /* !UNIV_INNOCHECKSUM */ #define FIL_ADDR_SIZE 6 /* address size is 6 bytes */ #ifndef UNIV_INNOCHECKSUM @@ -431,8 +430,6 @@ index */ #define fil_page_index_page_check(page) \ fil_page_type_is_index(fil_page_get_type(page)) -#ifndef UNIV_INNOCHECKSUM - /** Enum values for encryption table option */ enum fil_encryption_t { /** Encrypted if innodb_encrypt_tables=ON (srv_encrypt_tables) */ @@ -454,6 +451,8 @@ extern ulint fil_n_pending_tablespace_flushes; /** Number of files currently open */ extern ulint fil_n_file_opened; +#ifndef UNIV_INNOCHECKSUM + /** Look up a tablespace. The caller should hold an InnoDB table lock or a MDL that prevents the tablespace from being dropped during the operation, @@ -1315,6 +1314,7 @@ fil_page_reset_type( byte* page, ulint type, mtr_t* mtr); + /** Get the file page type. @param[in] page file page @return page type */ @@ -1325,6 +1325,7 @@ fil_page_get_type( { return(mach_read_from_2(page + FIL_PAGE_TYPE)); } + /** Check (and if needed, reset) the page type. Data files created before MySQL 5.1 may contain garbage in the FIL_PAGE_TYPE field. diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index f9167e0462e..e5057b30501 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -158,8 +158,6 @@ descriptor page, but used only in the first. */ FSP_FREE_LIMIT at a time */ /* @} */ -#ifndef UNIV_INNOCHECKSUM - /* @defgroup File Segment Inode Constants (moved from fsp0fsp.c) @{ */ /* FILE SEGMENT INODE @@ -293,6 +291,7 @@ the extent are free and which contain old tuple version to clean. */ /** Offset of the descriptor array on a descriptor page */ #define XDES_ARR_OFFSET (FSP_HEADER_OFFSET + FSP_HEADER_SIZE) +#ifndef UNIV_INNOCHECKSUM /* @} */ /**********************************************************************//** diff --git a/storage/innobase/include/fsp0fsp.ic b/storage/innobase/include/fsp0fsp.ic index 6317a67a089..2da3320eef7 100644 --- a/storage/innobase/include/fsp0fsp.ic +++ b/storage/innobase/include/fsp0fsp.ic @@ -53,6 +53,7 @@ xdes_calc_descriptor_index( return(ut_2pow_remainder(offset, page_size.physical()) / FSP_EXTENT_SIZE); } +#endif /*!UNIV_INNOCHECKSUM */ /**********************************************************************//** Gets a descriptor bit of a page. @@ -80,6 +81,7 @@ xdes_get_bit( bit_index)); } +#ifndef UNIV_INNOCHECKSUM /** Calculates the page where the descriptor of a page resides. @param[in] page_size page size @param[in] offset page offset diff --git a/storage/innobase/include/mach0data.h b/storage/innobase/include/mach0data.h index 5a5574b2537..684a3155a16 100644 --- a/storage/innobase/include/mach0data.h +++ b/storage/innobase/include/mach0data.h @@ -63,6 +63,7 @@ mach_write_to_2( /*============*/ byte* b, /*!< in: pointer to two bytes where to store */ ulint n); /*!< in: ulint integer to be stored, >= 0, < 64k */ +#endif /* !UNIV_INNOCHECKSUM */ /** The following function is used to fetch data from 2 consecutive bytes. The most significant byte is at the lowest address. @param[in] b pointer to 2 bytes where to store @@ -72,6 +73,8 @@ uint16_t mach_read_from_2( const byte* b) MY_ATTRIBUTE((warn_unused_result)); + +#ifndef UNIV_INNOCHECKSUM /********************************************************//** The following function is used to convert a 16-bit data item to the canonical format, for fast bytewise equality test @@ -362,6 +365,8 @@ mach_write_ulonglong( ulint len, /*!< in: length of dest */ bool usign); /*!< in: signed or unsigned flag */ +#endif /* !UNIV_INNOCHECKSUM */ + /** Read 1 to 4 bytes from a file page buffered in the buffer pool. @param[in] ptr pointer where to read @param[in] type MLOG_1BYTE, MLOG_2BYTES, or MLOG_4BYTES @@ -373,8 +378,6 @@ mach_read_ulint( mlog_id_t type) MY_ATTRIBUTE((warn_unused_result)); -#endif /* !UNIV_INNOCHECKSUM */ - #include "mach0data.ic" #endif diff --git a/storage/innobase/include/mach0data.ic b/storage/innobase/include/mach0data.ic index c89e4960480..9c4760b6d3a 100644 --- a/storage/innobase/include/mach0data.ic +++ b/storage/innobase/include/mach0data.ic @@ -865,6 +865,8 @@ mach_write_ulonglong( } } +#endif /* !UNIV_INNOCHECKSUM */ + /** Read 1 to 4 bytes from a file page buffered in the buffer pool. @param[in] ptr pointer where to read @param[in] type MLOG_1BYTE, MLOG_2BYTES, or MLOG_4BYTES @@ -889,5 +891,3 @@ mach_read_ulint( ut_error; return(0); } - -#endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/include/page0page.h b/storage/innobase/include/page0page.h index a8c5f11c17b..0aca13ac159 100644 --- a/storage/innobase/include/page0page.h +++ b/storage/innobase/include/page0page.h @@ -85,8 +85,6 @@ Otherwise written as 0. @see PAGE_ROOT_AUTO_INC */ This field should not be written to after page creation. */ -#ifndef UNIV_INNOCHECKSUM - #define PAGE_BTR_SEG_LEAF 36 /* file segment header for the leaf pages in a B-tree: defined only on the root page of a B-tree, but not in the root of an ibuf tree */ @@ -141,6 +139,8 @@ Otherwise written as 0. @see PAGE_ROOT_AUTO_INC */ #define PAGE_SAME_PAGE 4 #define PAGE_NO_DIRECTION 5 +#ifndef UNIV_INNOCHECKSUM + /* PAGE DIRECTORY ============== */ diff --git a/storage/innobase/include/page0page.ic b/storage/innobase/include/page0page.ic index 1c680ce7cc4..2ad5f26dcc1 100644 --- a/storage/innobase/include/page0page.ic +++ b/storage/innobase/include/page0page.ic @@ -27,8 +27,8 @@ Created 2/2/1994 Heikki Tuuri #ifndef page0page_ic #define page0page_ic -#include "mach0data.h" #ifndef UNIV_INNOCHECKSUM +#include "mach0data.h" #ifdef UNIV_DEBUG # include "log0recv.h" #endif /* !UNIV_DEBUG */ @@ -40,6 +40,7 @@ Created 2/2/1994 Heikki Tuuri #undef UNIV_INLINE #define UNIV_INLINE #endif +#endif /* !UNIV_INNOCHECKSUM */ /************************************************************//** Gets the start of a page. @@ -53,6 +54,7 @@ page_align( return((page_t*) ut_align_down(ptr, UNIV_PAGE_SIZE)); } +#ifndef UNIV_INNOCHECKSUM /************************************************************//** Gets the offset within a page. @return offset from the start of the page */ @@ -181,7 +183,6 @@ page_header_get_field( } #ifndef UNIV_INNOCHECKSUM - /*************************************************************//** Sets the given header field. */ UNIV_INLINE @@ -285,6 +286,8 @@ page_header_reset_last_insert( } } +#endif /* !UNIV_INNOCHECKSUM */ + /************************************************************//** Determine whether the page is in new-style compact format. @return nonzero if the page is in compact format, zero if it is in @@ -298,6 +301,7 @@ page_is_comp( return(page[PAGE_HEADER + PAGE_N_HEAP] & 0x80); } +#ifndef UNIV_INNOCHECKSUM /************************************************************//** TRUE if the record is on a page in compact format. @return nonzero if in compact format */ @@ -326,6 +330,8 @@ page_rec_get_heap_no( } } +#endif /* !UNIV_INNOCHECKSUM */ + /************************************************************//** Determine whether the page is a B-tree leaf. @return true if the page is a B-tree leaf (PAGE_LEVEL = 0) */ @@ -338,6 +344,7 @@ page_is_leaf( return(!*(const uint16*) (page + (PAGE_HEADER + PAGE_LEVEL))); } +#ifndef UNIV_INNOCHECKSUM /************************************************************//** Determine whether the page is empty. @return true if the page is empty (PAGE_N_RECS = 0) */ @@ -627,6 +634,8 @@ page_get_middle_rec( return(page_rec_get_nth(page, middle)); } +#endif /* !UNIV_INNOCHECKSUM */ + /*************************************************************//** Gets the page number. @return page number */ @@ -640,6 +649,7 @@ page_get_page_no( return(mach_read_from_4(page + FIL_PAGE_OFFSET)); } +#ifndef UNIV_INNOCHECKSUM /*************************************************************//** Gets the tablespace identifier. @return space id */ @@ -653,6 +663,8 @@ page_get_space_id( return(mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID)); } +#endif /* !UNIV_INNOCHECKSUM */ + /*************************************************************//** Gets the number of user records on page (infimum and supremum records are not user records). @@ -666,6 +678,7 @@ page_get_n_recs( return(page_header_get_field(page, PAGE_N_RECS)); } +#ifndef UNIV_INNOCHECKSUM /*************************************************************//** Gets the number of dir slots in directory. @return number of slots */ @@ -1054,6 +1067,8 @@ page_rec_get_base_extra_size( return(REC_N_NEW_EXTRA_BYTES + (ulint) !page_rec_is_comp(rec)); } +#endif /* UNIV_INNOCHECKSUM */ + /************************************************************//** Returns the sum of the sizes of the records in the record list, excluding the infimum and supremum records. @@ -1077,6 +1092,7 @@ page_get_data_size( return(ret); } +#ifndef UNIV_INNOCHECKSUM /************************************************************//** Allocates a block of memory from the free list of an index page. */ UNIV_INLINE diff --git a/storage/innobase/include/rem0rec.h b/storage/innobase/include/rem0rec.h index 96d3bdfab8f..8d3f87450f8 100644 --- a/storage/innobase/include/rem0rec.h +++ b/storage/innobase/include/rem0rec.h @@ -27,12 +27,14 @@ Created 5/30/1994 Heikki Tuuri #ifndef rem0rec_h #define rem0rec_h +#ifndef UNIV_INNOCHECKSUM #include "univ.i" #include "data0data.h" #include "rem0types.h" #include "mtr0types.h" #include "page0types.h" #include "trx0types.h" +#endif /*! UNIV_INNOCHECKSUM */ #include <ostream> #include <sstream> @@ -92,6 +94,7 @@ offsets[] array, first passed to rec_get_offsets() */ #define REC_OFFS_NORMAL_SIZE OFFS_IN_REC_NORMAL_SIZE #define REC_OFFS_SMALL_SIZE 10 +#ifndef UNIV_INNOCHECKSUM /******************************************************//** The following function is used to get the pointer of the next chained record on the same page. @@ -1112,4 +1115,5 @@ int wsrep_rec_get_foreign_key( #include "rem0rec.ic" +#endif /* !UNIV_INNOCHECKSUM */ #endif /* rem0rec_h */ diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index da4db5b9435..a5635fc4ecd 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -108,13 +108,8 @@ support cross-platform development and expose comonly used SQL names. */ #include <unistd.h> #endif -#ifdef UNIV_INNOCHECKSUM -extern bool strict_verify; -extern FILE* log_file; -extern uintmax_t cur_page_num; -#endif /* UNIV_INNOCHECKSUM */ - #include "my_pthread.h" + /* Following defines are to enable performance schema instrumentation in each of five InnoDB modules if HAVE_PSI_INTERFACE is defined. */ @@ -487,6 +482,12 @@ in both 32-bit and 64-bit environments. */ # define UINT64PFx "%016" PRIx64 #endif +#ifdef UNIV_INNOCHECKSUM +extern bool strict_verify; +extern FILE* log_file; +extern unsigned long long cur_page_num; +#endif /* UNIV_INNOCHECKSUM */ + typedef int64_t ib_int64_t; typedef uint64_t ib_uint64_t; typedef uint32_t ib_uint32_t; diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index ffb8808d6b4..b6e0409459b 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -4980,24 +4980,19 @@ page_zip_verify_checksum( const void* data, /*!< in: compressed page */ ulint size) /*!< in: size of compressed page */ { - const unsigned char* p = static_cast<const unsigned char*>(data) - + FIL_PAGE_SPACE_OR_CHKSUM; - - const uint32_t stored = static_cast<uint32_t>( - mach_read_from_4(p)); - -#ifdef UNIV_INNOCHECKSUM - p = static_cast<const unsigned char*>(data) + FIL_PAGE_TYPE; - bool no_checksum = (mach_read_from_2(p) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); - p = static_cast<const unsigned char*>(data) + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION; - bool encrypted = (mach_read_from_4(p) != 0); - p = static_cast<const unsigned char*>(data) + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4; - const uint32_t checksum = static_cast<uint32_t>(mach_read_from_4(p)); - - if (no_checksum) { - return (TRUE); - } -#endif + ib_uint32_t stored; + ib_uint32_t calc; + + stored = static_cast<ib_uint32_t>(mach_read_from_4( + static_cast<const unsigned char*>(data) + FIL_PAGE_SPACE_OR_CHKSUM)); + + ulint page_no MY_ATTRIBUTE((unused)) = + mach_read_from_4(static_cast<const unsigned char*> + (data) + FIL_PAGE_OFFSET); + ulint space_id MY_ATTRIBUTE((unused)) = + mach_read_from_4(static_cast<const unsigned char*> + (data) + FIL_PAGE_SPACE_ID); + const page_id_t page_id(space_id, page_no); #if FIL_PAGE_LSN % 8 #error "FIL_PAGE_LSN must be 64 bit aligned" @@ -5017,7 +5012,7 @@ page_zip_verify_checksum( } if (i >= size) { if (log_file) { - fprintf(log_file, "Page::%lu is empty and" + fprintf(log_file, "Page::%llu is empty and" " uncorrupted\n", cur_page_num); } @@ -5041,21 +5036,12 @@ page_zip_verify_checksum( return(TRUE); } -#ifndef UNIV_INNOCHECKSUM - ulint page_no = mach_read_from_4(static_cast< - const unsigned char*> - (data) + FIL_PAGE_OFFSET); - ulint space_id = mach_read_from_4(static_cast< - const unsigned char*> - (data) + FIL_PAGE_SPACE_ID); - const page_id_t page_id(space_id, page_no); -#endif /* UNIV_INNOCHECKSUM */ - - const uint32_t calc = page_zip_calc_checksum(data, size, curr_algo); + calc = static_cast<ib_uint32_t>(page_zip_calc_checksum( + data, size, curr_algo)); #ifdef UNIV_INNOCHECKSUM if (log_file) { - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " %s checksum: calculated = %u;" " recorded = %u\n", cur_page_num, buf_checksum_algorithm_name( @@ -5070,21 +5056,17 @@ page_zip_verify_checksum( data, size, SRV_CHECKSUM_ALGORITHM_CRC32); if (log_file) { - fprintf(log_file, "page::%lu: crc32 checksum:" + fprintf(log_file, "page::%llu: crc32 checksum:" " calculated = %u; recorded = %u\n", cur_page_num, crc32, stored); - fprintf(log_file, "page::%lu: none checksum:" + fprintf(log_file, "page::%llu: none checksum:" " calculated = %lu; recorded = %u\n", cur_page_num, BUF_NO_CHECKSUM_MAGIC, stored); } } #endif /* UNIV_INNOCHECKSUM */ - if (stored == calc -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && stored == checksum) -#endif - ) { + if (stored == calc) { return(TRUE); } @@ -5116,11 +5098,7 @@ page_zip_verify_checksum( if (legacy_big_endian_checksum) { const uint32_t calculated = page_zip_calc_checksum(data, size, curr_algo, true); - if (stored == calculated -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && calculated == checksum) -#endif - ) { + if (stored == calculated) { return(TRUE); } @@ -5130,11 +5108,7 @@ page_zip_verify_checksum( uint32_t calculated = page_zip_calc_checksum(data, size, SRV_CHECKSUM_ALGORITHM_INNODB); - if (stored == calculated -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && stored == checksum) -#endif - ) { + if (stored == calculated) { #ifndef UNIV_INNOCHECKSUM if (curr_algo @@ -5154,11 +5128,7 @@ page_zip_verify_checksum( /* If legacy checksum is not checked, do it now. */ if ((legacy_checksum_checked - && stored == calculated) -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && calculated == checksum) -#endif - ) { + && stored == calculated)) { legacy_big_endian_checksum = true; return(TRUE); } @@ -5168,11 +5138,7 @@ page_zip_verify_checksum( case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: case SRV_CHECKSUM_ALGORITHM_INNODB: { - if (stored == BUF_NO_CHECKSUM_MAGIC -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && checksum == BUF_NO_CHECKSUM_MAGIC) -#endif - ) { + if (stored == BUF_NO_CHECKSUM_MAGIC) { #ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { @@ -5191,14 +5157,8 @@ page_zip_verify_checksum( uint32_t calculated1; if (stored == calculated -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && checksum == calculated) -#endif - || stored == (calculated1 = + || stored == (calculated1 = page_zip_calc_checksum(data, size, SRV_CHECKSUM_ALGORITHM_CRC32, true)) -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && checksum == calculated1) -#endif ) { #ifndef UNIV_INNOCHECKSUM if (curr_algo @@ -5222,12 +5182,7 @@ page_zip_verify_checksum( data, size, SRV_CHECKSUM_ALGORITHM_CRC32, true); if (stored == calculated - || stored == calculated1 -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && checksum == calculated) - || ( encrypted == true && checksum == calculated1) -#endif - ) { + || stored == calculated1) { #ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, @@ -5240,11 +5195,7 @@ page_zip_verify_checksum( calculated = page_zip_calc_checksum( data, size, SRV_CHECKSUM_ALGORITHM_INNODB); - if (stored == calculated -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && checksum == calculated) -#endif - ) { + if (stored == calculated) { #ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( |