diff options
21 files changed, 923 insertions, 656 deletions
diff --git a/extra/CMakeLists.txt b/extra/CMakeLists.txt index 9cc5e7faa86..8c73e23d72b 100644 --- a/extra/CMakeLists.txt +++ b/extra/CMakeLists.txt @@ -79,7 +79,6 @@ IF(WITH_INNOBASE_STORAGE_ENGINE OR WITH_XTRADB_STORAGE_ENGINE) ../storage/innobase/ut/ut0ut.cc ../storage/innobase/buf/buf0buf.cc ../storage/innobase/page/page0zip.cc - ../storage/innobase/os/os0file.cc ../storage/innobase/fil/fil0crypt.cc ) diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc index fe828026b4b..a0976970db7 100644 --- a/extra/innochecksum.cc +++ b/extra/innochecksum.cc @@ -26,40 +26,58 @@ Published with a permission. */ -#include <my_config.h> #include <my_global.h> #include <stdio.h> #include <stdlib.h> #include <time.h> #include <sys/types.h> #include <sys/stat.h> -#ifdef HAVE_UNISTD_H +#ifndef __WIN__ # include <unistd.h> #endif #include <my_getopt.h> #include <m_string.h> -#include <welcome_copyright_notice.h> /* ORACLE_WELCOME_COPYRIGHT_NOTICE */ +#include <welcome_copyright_notice.h> /* ORACLE_WELCOME_COPYRIGHT_NOTICE */ /* Only parts of these files are included from the InnoDB codebase. The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */ -#include "univ.i" /* include all of this */ -#include "page0size.h" /* page_size_t */ -#include "page0zip.h" /* page_zip_calc_checksum() */ -#include "page0page.h" /* PAGE_* */ -#include "trx0undo.h" /* TRX_UNDO_* */ -#include "fut0lst.h" /* FLST_NODE_SIZE */ -#include "buf0checksum.h" /* buf_calc_page_*() */ -#include "fil0fil.h" /* FIL_* */ -#include "fil0crypt.h" -#include "os0file.h" -#include "fsp0fsp.h" /* fsp_flags_get_page_size() & - fsp_flags_get_zip_size() */ -#include "mach0data.h" /* mach_read_from_4() */ -#include "ut0crc32.h" /* ut_crc32_init() */ -#include "fsp0pagecompress.h" /* fil_get_compression_alg_name */ +typedef void fil_space_t; + +#include "univ.i" /* include all of this */ +#include "page0size.h" + +#define FLST_BASE_NODE_SIZE (4 + 2 * FIL_ADDR_SIZE) +#define FLST_NODE_SIZE (2 * FIL_ADDR_SIZE) +#define FSEG_PAGE_DATA FIL_PAGE_DATA +#define FSEG_HEADER_SIZE 10 +#define UT_BITS_IN_BYTES(b) (((b) + 7) / 8) + +#include "ut0ut.h" #include "ut0byte.h" +#include "mtr0types.h" #include "mach0data.h" +#include "fsp0types.h" +#include "rem0rec.h" +#include "buf0checksum.h" /* buf_calc_page_*() */ +#include "buf0buf.h" /* buf_page_is_corrupted */ +#include "fil0fil.h" /* FIL_* */ +#include "page0page.h" /* PAGE_* */ +#include "page0zip.h" /* page_zip_*() */ +#include "trx0undo.h" /* TRX_* */ +#include "fsp0fsp.h" /* fsp_flags_get_page_size() & + fsp_flags_get_zip_size() */ +#include "ut0crc32.h" /* ut_crc32_init() */ +#include "fsp0pagecompress.h" /* fil_get_compression_alg_name */ +#include "fil0crypt.h" /* fil_space_verify_crypt_checksum */ + +#include <string.h> + +#ifdef UNIV_NONINL +# include "fsp0fsp.ic" +# include "mach0data.ic" +# include "ut0rnd.ic" +#endif #ifndef PRIuMAX #define PRIuMAX "llu" @@ -68,25 +86,29 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */ /* Global variables */ static bool verbose; static bool just_count; -static uintmax_t start_page; -static uintmax_t end_page; -static uintmax_t do_page; +static unsigned long long start_page; +static unsigned long long end_page; +static unsigned long long do_page; static bool use_end_page; static bool do_one_page; -/* replaces declaration in srv0srv.c */ -ulong srv_page_size; -page_size_t univ_page_size(0, 0, false); +static my_bool do_leaf; +static my_bool per_page_details; +static ulong n_merge; extern ulong srv_checksum_algorithm; +static ulong physical_page_size; /* Page size in bytes on disk. */ +static ulong logical_page_size; /* Page size when uncompressed. */ +ulong srv_page_size; +page_size_t univ_page_size(0, 0, false); /* Current page number (0 based). */ -uintmax_t cur_page_num; +unsigned long long cur_page_num; /* Skip the checksum verification. */ static bool no_check; /* Enabled for strict checksum verification. */ -bool strict_verify; +bool strict_verify = 0; /* Enabled for rewrite checksum. */ static bool do_write; /* Mismatches count allowed (0 by default). */ -static uintmax_t allow_mismatches; +static unsigned long long allow_mismatches=0; static bool page_type_summary; static bool page_type_dump; /* Store filename for page-type-dump option. */ @@ -99,8 +121,7 @@ char* log_filename = NULL; FILE* log_file = NULL; /* Enabled for log write option. */ static bool is_log_enabled = false; -static my_bool do_leaf; -static ulong n_merge; + #ifndef _WIN32 /* advisory lock for non-window system. */ struct flock lk; @@ -198,75 +219,6 @@ struct per_index_stats { std::map<unsigned long long, per_index_stats> index_ids; -bool encrypted = false; - -ulint -page_is_comp( -/*=========*/ - const page_t* page) /*!< in: index page */ -{ - return(page_header_get_field(page, PAGE_N_HEAP) & 0x8000); -} - -bool -page_is_leaf( -/*=========*/ - const page_t* page) /*!< in: page */ -{ - return(!*(const uint16*) (page + (PAGE_HEADER + PAGE_LEVEL))); -} - -ulint -page_get_page_no( -/*=============*/ - const page_t* page) /*!< in: page */ -{ - return(mach_read_from_4(page + FIL_PAGE_OFFSET)); -} -#define FSEG_HEADER_SIZE 10 /*!< Length of the file system - header, in bytes */ -#define REC_N_NEW_EXTRA_BYTES 5 -#define REC_N_OLD_EXTRA_BYTES 6 -#define PAGE_DATA (PAGE_HEADER + 36 + 2 * FSEG_HEADER_SIZE) - /* start of data on the page */ -#define PAGE_NEW_SUPREMUM (PAGE_DATA + 2 * REC_N_NEW_EXTRA_BYTES + 8) - /* offset of the page supremum record on a - new-style compact page */ -#define PAGE_NEW_SUPREMUM_END (PAGE_NEW_SUPREMUM + 8) -#define PAGE_OLD_SUPREMUM (PAGE_DATA + 2 + 2 * REC_N_OLD_EXTRA_BYTES + 8) - /* offset of the page supremum record on an - old-style page */ -#define PAGE_OLD_SUPREMUM_END (PAGE_OLD_SUPREMUM + 9) - /* offset of the page supremum record end on - an old-style page */ -#define FLST_BASE_NODE_SIZE (4 + 2 * 6) -#define XDES_ARR_OFFSET (FSP_HEADER_OFFSET + FSP_HEADER_SIZE) -#define XDES_FREE_BIT 0 -#define XDES_BITMAP (FLST_NODE_SIZE + 12) -#define XDES_BITS_PER_PAGE 2 -#define UT_BITS_IN_BYTES(b) (((b) + 7) / 8) -#define XDES_SIZE \ - (XDES_BITMAP \ - + UT_BITS_IN_BYTES(FSP_EXTENT_SIZE * XDES_BITS_PER_PAGE)) - -ulint -page_get_data_size( -/*===============*/ - const page_t* page) /*!< in: index page */ -{ - ulint ret; - - ret = (ulint)(page_header_get_field(page, PAGE_HEAP_TOP) - - (page_is_comp(page) - ? PAGE_NEW_SUPREMUM_END - : PAGE_OLD_SUPREMUM_END) - - page_header_get_field(page, PAGE_GARBAGE)); - - ut_ad(ret < UNIV_PAGE_SIZE); - - return(ret); -} - void print_index_leaf_stats( unsigned long long id, const per_index_stats& index, @@ -280,8 +232,7 @@ void print_index_leaf_stats( fprintf(fil_out, "page_no\tdata_size\tn_recs\n"); while (it_page != index.leaves.end()) { const per_page_stats& stat = it_page->second; - fprintf(fil_out, "%llu\t" ULINTPF "\t" ULINTPF "\n", - it_page->first, stat.data_size, stat.n_recs); + fprintf(fil_out, "%llu\t" ULINTPF "\t" ULINTPF "\n", it_page->first, stat.data_size, stat.n_recs); page_no = stat.right_page_no; it_page = index.leaves.find(page_no); } @@ -307,6 +258,7 @@ void defrag_analysis( break; } } + if (index.max_data_size) { n_pages += data_size_total / index.max_data_size; if (data_size_total % index.max_data_size != 0) { @@ -316,15 +268,16 @@ void defrag_analysis( } if (index.leaf_pages) { - fprintf(fil_out, "count = " ULINTPF " free = " ULINTPF "\n", - index.count, index.free_pages); + fprintf(fil_out, "count = " ULINTPF " free = " ULINTPF "\n", index.count, index.free_pages); + } + + if (!n_leaf_pages) { + n_leaf_pages = 1; } - fprintf(fil_out, "%llu\t\t%llu\t\t" - ULINTPF "\t\t%lu\t\t" ULINTPF "\t\t%.2f\t" ULINTPF "\n", + fprintf(fil_out, "%llu\t\t%llu\t\t" ULINTPF "\t\t" ULINTPF "\t\t" ULINTPF "\t\t%.2f\t" ULINTPF "\n", id, index.leaf_pages, n_leaf_pages, n_merge, n_pages, - 1.0 - (double)n_pages / (double)n_leaf_pages, - index.max_data_size); + 1.0 - (double)n_pages / (double)n_leaf_pages, index.max_data_size); } void print_leaf_stats( @@ -372,29 +325,6 @@ get_page_size( return(page_size_t(flags)); } -#ifdef MYSQL_COMPRESSION -/** Decompress a page -@param[in,out] buf Page read from disk, uncompressed data will - also be copied to this page -@param[in, out] scratch Page to use for temporary decompress -@param[in] page_size scratch physical size -@return true if decompress succeeded */ -static -bool page_decompress( - byte* buf, - byte* scratch, - page_size_t page_size) -{ - dberr_t err; - - /* Set the dblwr recover flag to false. */ - err = os_file_decompress_page( - false, buf, scratch, page_size.physical()); - - return(err == DB_SUCCESS); -} -#endif - #ifdef _WIN32 /***********************************************//* @param [in] error error no. from the getLastError(). @@ -439,6 +369,7 @@ open_file( access = GENERIC_READ; flags = _O_RDONLY | _O_BINARY; } + /* CreateFile() also provide advisory lock with the usage of access and share mode of the file.*/ hFile = CreateFile( @@ -462,8 +393,7 @@ open_file( if (do_write) { create_flag = O_RDWR; lk.l_type = F_WRLCK; - } - else { + } else { create_flag = O_RDONLY; lk.l_type = F_RDLCK; } @@ -525,12 +455,18 @@ ulong read_file( /** Check if page is corrupted or not. @param[in] buf page frame @param[in] page_size page size +@param[in] is_encrypted true if page0 contained cryp_data + with crypt_scheme encrypted +@param[in] is_compressed true if page0 fsp_flags contained + page compression flag @retval true if page is corrupted otherwise false. */ static bool is_page_corrupted( - const byte* buf, - const page_size_t& page_size) + byte* buf, + const page_size_t& page_size, + bool is_encrypted, + bool is_compressed) { /* enable if page is corrupted. */ @@ -538,8 +474,22 @@ is_page_corrupted( /* use to store LSN values. */ ulint logseq; ulint logseqfield; + ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE); + uint key_version = mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + ulint space_id = mach_read_from_4( + buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + + /* We can't trust only a page type, thus we take account + also fsp_flags or crypt_data on page 0 */ + if ((page_type == FIL_PAGE_PAGE_COMPRESSED && is_compressed) || + (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED && + is_compressed && is_encrypted)) { + /* Page compressed tables do not contain post compression + checksum. */ + return (false); + } - if (!page_size.is_compressed()) { + if (page_size.is_compressed()) { /* check the stored log sequence numbers for uncompressed tablespace. */ logseq = mach_read_from_4(buf + FIL_PAGE_LSN + 4); @@ -549,35 +499,32 @@ is_page_corrupted( if (is_log_enabled) { fprintf(log_file, - "page::%" PRIuMAX + "space::" ULINTPF " page::%llu" "; log sequence number:first = " ULINTPF "; second = " ULINTPF "\n", - cur_page_num, logseq, logseqfield); + space_id, cur_page_num, logseq, logseqfield); if (logseq != logseqfield) { fprintf(log_file, - "Fail; page %" PRIuMAX + "Fail; space::" ULINTPF " page::%llu" " invalid (fails log " "sequence number check)\n", - cur_page_num); + space_id, cur_page_num); } } } - /* FIXME: Read the page number from the tablespace header, - and check that every page carries the same page number. */ + /* Again we can't trust only FIL_PAGE_FILE_FLUSH_LSN field + now repurposed as FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, + we need to check also crypt_data contents. - /* If page is encrypted, use different checksum calculation + If page is encrypted, use different checksum calculation as innochecksum can't decrypt pages. Note that some old InnoDB versions did not initialize FIL_PAGE_FILE_FLUSH_LSN field so if crypt checksum does not match we verify checksum using - normal method. - */ - if (mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0) { - is_corrupted = fil_space_verify_crypt_checksum( - const_cast<byte*>(buf), page_size, - mach_read_from_4(buf - + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID), - cur_page_num); + normal method. */ + if (is_encrypted && key_version != 0) { + is_corrupted = !fil_space_verify_crypt_checksum(buf, + page_size, space_id, cur_page_num); } else { is_corrupted = true; } @@ -628,8 +575,9 @@ is_page_empty( if (*page++) { return (false); } - } - return (true); + } + + return (true); } /********************************************************************//** @@ -682,7 +630,7 @@ update_checksum( mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); if (is_log_enabled) { - fprintf(log_file, "page::%" PRIuMAX "; Updated checksum =" + fprintf(log_file, "page::%llu; Updated checksum =" " %u\n", cur_page_num, checksum); } @@ -713,7 +661,7 @@ update_checksum( mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); if (is_log_enabled) { - fprintf(log_file, "page::%" PRIuMAX "; Updated checksum field1" + fprintf(log_file, "page::%llu; Updated checksum field1" " = %u\n", cur_page_num, checksum); } @@ -727,7 +675,7 @@ update_checksum( FIL_PAGE_END_LSN_OLD_CHKSUM,checksum); if (is_log_enabled) { - fprintf(log_file, "page::%" PRIuMAX "; Updated checksum " + fprintf(log_file, "page::%llu; Updated checksum " "field2 = %u\n", cur_page_num, checksum); } @@ -802,7 +750,7 @@ write_file( if (page_size != fwrite(buf, 1, page_size, file == stdin ? stdout : file)) { - fprintf(stderr, "Failed to write page %" PRIuMAX " to %s: %s\n", + fprintf(stderr, "Failed to write page::%llu to %s: %s\n", cur_page_num, filename, strerror(errno)); return(false); @@ -819,93 +767,32 @@ write_file( return(true); } -/********************************************************//** -Gets the next index page number. -@return next page number */ -ulint -btr_page_get_next( -/*==============*/ - const page_t* page) /*!< in: index page */ -{ - return(mach_read_from_4(page + FIL_PAGE_NEXT)); -} - -/********************************************************//** -Gets the previous index page number. -@return prev page number */ -ulint -btr_page_get_prev( -/*==============*/ - const page_t* page) /*!< in: index page */ -{ - return(mach_read_from_4(page + FIL_PAGE_PREV)); -} - -ulint -mach_read_ulint( - const byte* ptr, - mlog_id_t type) -{ - switch (type) { - case MLOG_1BYTE: - return(mach_read_from_1(ptr)); - case MLOG_2BYTES: - return(mach_read_from_2(ptr)); - case MLOG_4BYTES: - return(mach_read_from_4(ptr)); - default: - break; - } - - ut_error; - return(0); -} -ibool -xdes_get_bit( -/*=========*/ - const xdes_t* descr, /*!< in: descriptor */ - ulint bit, /*!< in: XDES_FREE_BIT or XDES_CLEAN_BIT */ - ulint offset) /*!< in: page offset within extent: - 0 ... FSP_EXTENT_SIZE - 1 */ -{ - ulint index = bit + XDES_BITS_PER_PAGE * offset; - - ulint bit_index = index % 8; - ulint byte_index = index / 8; - - return(ut_bit_get_nth( - mach_read_ulint(descr + XDES_BITMAP + byte_index, - MLOG_1BYTE), - bit_index)); -} - /* Parse the page and collect/dump the information about page type @param [in] page buffer page -@param [in] xdes xdes page +@param [out] xdes extend descriptor page @param [in] file file for diagnosis. -@param [in] page_size page size +@param [in] page_size page_size +@param [in] is_encrypted tablespace is encrypted */ void parse_page( const byte* page, - const byte* xdes, + byte* xdes, FILE* file, - page_size_t page_size) + const page_size_t& page_size, + bool is_encrypted) { - unsigned long long id=0; - ulint undo_page_type=0; - ulint n_recs; - ulint page_no=0; - ulint right_page_no=0; - ulint left_page_no=0; - ulint data_bytes=0; - bool is_leaf=false; - ulint size_range_id=0; - ulint data_types=0; - ulint key_version = 0; - + unsigned long long id; + ulint undo_page_type; char str[20]={'\0'}; + ulint n_recs; + ulint page_no; + ulint left_page_no; + ulint right_page_no; + ulint data_bytes; + bool is_leaf; + int size_range_id; /* Check whether page is doublewrite buffer. */ if(skip_page) { @@ -916,102 +803,105 @@ parse_page( switch (mach_read_from_2(page + FIL_PAGE_TYPE)) { - case FIL_PAGE_INDEX: + case FIL_PAGE_INDEX: { + uint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); page_type.n_fil_page_index++; - id = mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID); - n_recs = page_header_get_field(page, PAGE_N_RECS); - page_no = page_get_page_no(page); - left_page_no = btr_page_get_prev(page); - right_page_no = btr_page_get_next(page); - key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); /* If page is encrypted we can't read index header */ - if (!key_version) { - data_bytes = page_get_data_size(page); - } else { - data_bytes = 0; - } + if (!is_encrypted) { + id = mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID); + n_recs = mach_read_from_2(page + PAGE_HEADER + PAGE_N_RECS); + page_no = mach_read_from_4(page + FIL_PAGE_OFFSET); + left_page_no = mach_read_from_4(page + FIL_PAGE_PREV); + right_page_no = mach_read_from_4(page + FIL_PAGE_NEXT); + ulint is_comp = mach_read_from_2(page + PAGE_HEADER + PAGE_N_HEAP) & 0x8000; + ulint level = mach_read_from_2(page + PAGE_HEADER + PAGE_LEVEL); + ulint garbage = mach_read_from_2(page + PAGE_HEADER + PAGE_GARBAGE); - is_leaf = page_is_leaf(page); - if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tIndex page\t\t\t|" - "\tindex id=%llu,", cur_page_num, id); - - fprintf(file, - " page level=" ULINTPF " leaf %u" - ", No. of records=" ULINTPF - ", garbage=" ULINTPF - ", n_recs=" ULINTPF - ", %s\n", - page_header_get_field(page, PAGE_LEVEL), - is_leaf, - n_recs, - page_header_get_field(page, PAGE_GARBAGE), - data_types, - str); - } + data_bytes = (ulint)(mach_read_from_2(page + PAGE_HEADER + PAGE_HEAP_TOP) + - (is_comp + ? PAGE_NEW_SUPREMUM_END + : PAGE_OLD_SUPREMUM_END) + - garbage); - size_range_id = (data_bytes * SIZE_RANGES_FOR_PAGE - + page_size.logical() - 1) / - page_size.logical(); - - if (size_range_id > SIZE_RANGES_FOR_PAGE + 1) { - /* data_bytes is bigger than logical_page_size */ - size_range_id = SIZE_RANGES_FOR_PAGE + 1; - } + is_leaf = (!*(const uint16*) (page + (PAGE_HEADER + PAGE_LEVEL))); - /* update per-index statistics */ - { - if (index_ids.count(id) == 0) { - index_ids[id] = per_index_stats(); - } - - std::map<unsigned long long, per_index_stats>::iterator it; - it = index_ids.find(id); - per_index_stats &index = (it->second); - uchar* des = (uchar *)(xdes + XDES_ARR_OFFSET - + XDES_SIZE * ((page_no & (page_size.physical() - 1)) - / FSP_EXTENT_SIZE)); - - if (xdes_get_bit(des, XDES_FREE_BIT, - page_no % FSP_EXTENT_SIZE)) { - index.free_pages++; - return; + if (page_type_dump) { + fprintf(file, "#::%llu\t\t|\t\tIndex page\t\t\t|" + "\tindex id=%llu,", cur_page_num, id); + + fprintf(file, + " page level=" ULINTPF + ", No. of records=" ULINTPF + ", garbage=" ULINTPF ", %s\n", + level, n_recs, garbage, str); } - index.pages++; + size_range_id = (data_bytes * SIZE_RANGES_FOR_PAGE + + page_size.logical() - 1) / + page_size.logical(); - if (is_leaf) { - index.leaf_pages++; - if (data_bytes > index.max_data_size) { - index.max_data_size = data_bytes; + if (size_range_id > SIZE_RANGES_FOR_PAGE + 1) { + /* data_bytes is bigger than logical_page_size */ + size_range_id = SIZE_RANGES_FOR_PAGE + 1; + } + if (per_page_details) { + printf("index id=%llu page " ULINTPF " leaf %d n_recs " ULINTPF " data_bytes " ULINTPF + "\n", id, page_no, is_leaf, n_recs, data_bytes); + } + /* update per-index statistics */ + { + if (index_ids.count(id) == 0) { + index_ids[id] = per_index_stats(); + } + std::map<unsigned long long, per_index_stats>::iterator it; + it = index_ids.find(id); + per_index_stats &index = (it->second); + const byte* des = xdes + XDES_ARR_OFFSET + + XDES_SIZE * ((page_no & (page_size.physical() - 1)) + / FSP_EXTENT_SIZE); + if (xdes_get_bit(des, XDES_FREE_BIT, + page_no % FSP_EXTENT_SIZE)) { + index.free_pages++; + return; } - struct per_page_stats pp(n_recs, data_bytes, - left_page_no, right_page_no); + index.pages++; - index.leaves[page_no] = pp; + if (is_leaf) { + index.leaf_pages++; + if (data_bytes > index.max_data_size) { + index.max_data_size = data_bytes; + } + struct per_page_stats pp(n_recs, data_bytes, + left_page_no, right_page_no); - if (left_page_no == ULINT32_UNDEFINED) { - index.first_leaf_page = page_no; - index.count++; + index.leaves[page_no] = pp; + + if (left_page_no == ULINT32_UNDEFINED) { + index.first_leaf_page = page_no; + index.count++; + } } - } - index.total_n_recs += n_recs; - index.total_data_bytes += data_bytes; - index.pages_in_size_range[size_range_id] ++; + index.total_n_recs += n_recs; + index.total_data_bytes += data_bytes; + index.pages_in_size_range[size_range_id] ++; + } + } else { + fprintf(file, "#::%llu\t\t|\t\tEncrypted Index page\t\t\t|" + "\tkey_version %u,%s\n", cur_page_num, key_version, str); } break; - + } case FIL_PAGE_UNDO_LOG: page_type.n_fil_page_undo_log++; undo_page_type = mach_read_from_2(page + TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_TYPE); if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tUndo log page\t\t\t|", + fprintf(file, "#::%llu\t\t|\t\tUndo log page\t\t\t|", cur_page_num); } if (undo_page_type == TRX_UNDO_INSERT) { @@ -1085,7 +975,7 @@ parse_page( case FIL_PAGE_INODE: page_type.n_fil_page_inode++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInode page\t\t\t|" + fprintf(file, "#::%llu\t\t|\t\tInode page\t\t\t|" "\t%s\n",cur_page_num, str); } break; @@ -1093,7 +983,7 @@ parse_page( case FIL_PAGE_IBUF_FREE_LIST: page_type.n_fil_page_ibuf_free_list++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInsert buffer free list" + fprintf(file, "#::%llu\t\t|\t\tInsert buffer free list" " page\t|\t%s\n", cur_page_num, str); } break; @@ -1101,7 +991,7 @@ parse_page( case FIL_PAGE_TYPE_ALLOCATED: page_type.n_fil_page_type_allocated++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tFreshly allocated " + fprintf(file, "#::%llu\t\t|\t\tFreshly allocated " "page\t\t|\t%s\n", cur_page_num, str); } break; @@ -1109,7 +999,7 @@ parse_page( case FIL_PAGE_IBUF_BITMAP: page_type.n_fil_page_ibuf_bitmap++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInsert Buffer " + fprintf(file, "#::%llu\t\t|\t\tInsert Buffer " "Bitmap\t\t|\t%s\n", cur_page_num, str); } break; @@ -1117,7 +1007,7 @@ parse_page( case FIL_PAGE_TYPE_SYS: page_type.n_fil_page_type_sys++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tSystem page\t\t\t|" + fprintf(file, "#::%llu\t\t|\t\tSystem page\t\t\t|" "\t%s\n",cur_page_num, str); } break; @@ -1125,25 +1015,25 @@ parse_page( case FIL_PAGE_TYPE_TRX_SYS: page_type.n_fil_page_type_trx_sys++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tTransaction system " + fprintf(file, "#::%llu\t\t|\t\tTransaction system " "page\t\t|\t%s\n", cur_page_num, str); } break; case FIL_PAGE_TYPE_FSP_HDR: page_type.n_fil_page_type_fsp_hdr++; - memcpy((void *)xdes, (void *)page, page_size.physical()); + memcpy(xdes, page, page_size.physical()); if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tFile Space " + fprintf(file, "#::%llu\t\t|\t\tFile Space " "Header\t\t|\t%s\n", cur_page_num, str); } break; case FIL_PAGE_TYPE_XDES: page_type.n_fil_page_type_xdes++; - memcpy((void *)xdes, (void *)page, page_size.physical()); + memcpy(xdes, page, page_size.physical()); if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tExtent descriptor " + fprintf(file, "#::%llu\t\t|\t\tExtent descriptor " "page\t\t|\t%s\n", cur_page_num, str); } break; @@ -1151,7 +1041,7 @@ parse_page( case FIL_PAGE_TYPE_BLOB: page_type.n_fil_page_type_blob++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tBLOB page\t\t\t|\t%s\n", + fprintf(file, "#::%llu\t\t|\t\tBLOB page\t\t\t|\t%s\n", cur_page_num, str); } break; @@ -1159,7 +1049,7 @@ parse_page( case FIL_PAGE_TYPE_ZBLOB: page_type.n_fil_page_type_zblob++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tCompressed BLOB " + fprintf(file, "#::%llu\t\t|\t\tCompressed BLOB " "page\t\t|\t%s\n", cur_page_num, str); } break; @@ -1167,25 +1057,26 @@ parse_page( case FIL_PAGE_TYPE_ZBLOB2: page_type.n_fil_page_type_zblob2++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tSubsequent Compressed " + fprintf(file, "#::%llu\t\t|\t\tSubsequent Compressed " "BLOB page\t|\t%s\n", cur_page_num, str); } break; - case FIL_PAGE_PAGE_COMPRESSED: - page_type.n_fil_page_type_page_compressed++; - if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tPage compressed " + case FIL_PAGE_PAGE_COMPRESSED: + page_type.n_fil_page_type_page_compressed++; + if (page_type_dump) { + fprintf(file, "#::%llu\t\t|\t\tPage compressed " "page\t|\t%s\n", cur_page_num, str); - } - break; - case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED: - page_type.n_fil_page_type_page_compressed_encrypted++; - if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tPage compressed encrypted " + } + break; + + case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED: + page_type.n_fil_page_type_page_compressed_encrypted++; + if (page_type_dump) { + fprintf(file, "#::%llu\t\t|\t\tPage compressed encrypted " "page\t|\t%s\n", cur_page_num, str); - } - break; + } + break; default: page_type.n_fil_page_type_other++; break; @@ -1273,7 +1164,7 @@ print_summary( page_type.n_fil_page_type_page_compressed); fprintf(fil_out, "%8d\tPage compressed encrypted page\n", page_type.n_fil_page_type_page_compressed_encrypted); - fprintf(fil_out, "%8d\tOther type of page", + fprintf(fil_out, "%8d\tOther type of page\n", page_type.n_fil_page_type_other); fprintf(fil_out, "\n===============================================\n"); @@ -1331,8 +1222,10 @@ static struct my_option innochecksum_options[] = { 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, {"verbose", 'v', "Verbose (prints progress every 5 seconds).", &verbose, &verbose, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, +#ifndef DBUG_OFF {"debug", '#', "Output debug log. See " REFMAN "dbug-package.html", &dbug_setting, &dbug_setting, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, +#endif /* !DBUG_OFF */ {"count", 'c', "Print the count of pages in the file and exits.", &just_count, &just_count, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"start_page", 's', "Start on this page number (0 based).", @@ -1361,10 +1254,12 @@ static struct my_option innochecksum_options[] = { {"page-type-dump", 'D', "Dump the page type info for each page in a " "tablespace.", &page_dump_filename, &page_dump_filename, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"per-page-details", 'i', "Print out per-page detail information.", + &per_page_details, &per_page_details, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"log", 'l', "log output.", &log_filename, &log_filename, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"leaf", 'e', "Examine leaf index pages", + {"leaf", 'f', "Examine leaf index pages", &do_leaf, &do_leaf, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"merge", 'm', "leaf page count if merge given number of consecutive pages", &n_merge, &n_merge, 0, GET_ULONG, REQUIRED_ARG, 0, 0, (longlong)10L, 0, 1, 0}, @@ -1392,9 +1287,9 @@ static void usage(void) puts(ORACLE_WELCOME_COPYRIGHT_NOTICE("2000")); printf("InnoDB offline file checksum utility.\n"); printf("Usage: %s [-c] [-s <start page>] [-e <end page>] " - "[-p <page>] [-v] [-a <allow mismatches>] [-n] " + "[-p <page>] [-i] [-v] [-a <allow mismatches>] [-n] " "[-C <strict-check>] [-w <write>] [-S] [-D <page type dump>] " - "[-l <log>] [-e] <filename or [-]>\n", my_progname); + "[-l <log>] [-l] [-m <merge pages>] <filename or [-]>\n", my_progname); printf("See " REFMAN "innochecksum.html for usage hints.\n"); my_print_help(innochecksum_options); my_print_variables(innochecksum_options); @@ -1487,7 +1382,7 @@ get_options( char ***argv) { if (handle_options(argc, argv, innochecksum_options, - innochecksum_get_one_option)) { + innochecksum_get_one_option)) { my_end(0); exit(true); } @@ -1502,6 +1397,133 @@ get_options( return (false); } +/** Check from page 0 if table is encrypted. +@param[in] filename Filename +@param[in] page_size page size +@param[in] page Page 0 +@retval true if tablespace is encrypted, false if not +*/ +static +bool check_encryption( + const char* filename, + const page_size_t& page_size, + byte * page) +{ + ulint offset = (FSP_HEADER_OFFSET + (XDES_ARR_OFFSET + XDES_SIZE * + (page_size.physical()) / FSP_EXTENT_SIZE)); + + if (memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) { + return false; + } + + ulint type = mach_read_from_1(page + offset + MAGIC_SZ + 0); + + if (! (type == CRYPT_SCHEME_UNENCRYPTED || + type == CRYPT_SCHEME_1)) { + return false; + } + + ulint iv_length = mach_read_from_1(page + offset + MAGIC_SZ + 1); + + if (iv_length != CRYPT_SCHEME_1_IV_LEN) { + return false; + } + + uint min_key_version = mach_read_from_4 + (page + offset + MAGIC_SZ + 2 + iv_length); + + uint key_id = mach_read_from_4 + (page + offset + MAGIC_SZ + 2 + iv_length + 4); + + if (type == CRYPT_SCHEME_1 && is_log_enabled) { + fprintf(log_file,"Tablespace %s encrypted key_version %u key_id %u\n", + filename, min_key_version, key_id); + } + + return (type == CRYPT_SCHEME_1); +} + +/** +Verify page checksum. +@param[in] buf page to verify +@param[in] page_size page size +@param[in] is_encrypted true if tablespace is encrypted +@param[in] is_compressed true if tablespace is page compressed +@param[in,out] mismatch_count Number of pages failed in checksum verify +@retval 0 if page checksum matches or 1 if it does not match +*/ +static +int verify_checksum( + byte* buf, + const page_size_t& page_size, + bool is_encrypted, + bool is_compressed, + unsigned long long* mismatch_count) +{ + int exit_status = 0; + bool is_corrupted = false; + + is_corrupted = is_page_corrupted( + buf, page_size, is_encrypted, is_compressed); + + if (is_corrupted) { + fprintf(stderr, "Fail: page::%llu invalid\n", + cur_page_num); + + (*mismatch_count)++; + + if (*mismatch_count > allow_mismatches) { + fprintf(stderr, + "Exceeded the " + "maximum allowed " + "checksum mismatch " + "count::%llu current::%llu\n", + *mismatch_count, + allow_mismatches); + + exit_status = 1; + } + } + + return (exit_status); +} + +/** Rewrite page checksum if needed. +@param[in] filename File name +@param[in] fil_in File pointer +@param[in] buf page +@param[in] page_size page size +@param[in] pos File position +@param[in] is_encrypted true if tablespace is encrypted +@param[in] is_compressed true if tablespace is page compressed +@retval 0 if checksum rewrite was successful, 1 if error was detected */ +static +int +rewrite_checksum( + const char* filename, + FILE* fil_in, + byte* buf, + const page_size_t& page_size, + fpos_t* pos, + bool is_encrypted, + bool is_compressed) +{ + int exit_status = 0; + /* Rewrite checksum. Note that for encrypted and + page compressed tables this is not currently supported. */ + if (do_write && + !is_encrypted && + !is_compressed + && !write_file(filename, fil_in, buf, + page_size.is_compressed(), pos, + static_cast<ulong>(page_size.physical()))) { + + exit_status = 1; + } + + return (exit_status); +} + int main( int argc, char **argv) @@ -1511,15 +1533,12 @@ int main( /* our input filename. */ char* filename; /* Buffer to store pages read. */ + byte* buf_ptr = NULL; + byte* xdes_ptr = NULL; byte* buf = NULL; - /* Buffer for xdes */ byte* xdes = NULL; /* bytes read count */ ulong bytes; - /* Buffer to decompress page.*/ -#ifdef MYSQL_COMPRESSION - byte* tbuf = NULL; -#endif /* current time */ time_t now; /* last time */ @@ -1531,6 +1550,8 @@ int main( struct stat st; #endif /* _WIN32 */ + int exit_status = 0; + /* size of file (has to be 64 bits) */ unsigned long long int size = 0; /* number of pages in file */ @@ -1538,9 +1559,7 @@ int main( off_t offset = 0; /* count the no. of page corrupted. */ - ulint mismatch_count = 0; - /* Variable to ack the page is corrupted or not. */ - bool is_corrupted = false; + unsigned long long mismatch_count = 0; bool partial_page_read = false; /* Enabled when read from stdin is done. */ @@ -1560,32 +1579,37 @@ int main( DBUG_PROCESS(argv[0]); if (get_options(&argc,&argv)) { - DBUG_RETURN(1); + exit_status = 1; + goto my_exit; } if (strict_verify && no_check) { fprintf(stderr, "Error: --strict-check option cannot be used " "together with --no-check option.\n"); - DBUG_RETURN(1); + exit_status = 1; + goto my_exit; } if (no_check && !do_write) { fprintf(stderr, "Error: --no-check must be associated with " "--write option.\n"); - DBUG_RETURN(1); + exit_status = 1; + goto my_exit; } if (page_type_dump) { fil_page_type = create_file(page_dump_filename); if (!fil_page_type) { - DBUG_RETURN(1); + exit_status = 1; + goto my_exit; } } if (is_log_enabled) { log_file = create_file(log_filename); if (!log_file) { - DBUG_RETURN(1); + exit_status = 1; + goto my_exit; } fprintf(log_file, "InnoDB File Checksum Utility.\n"); } @@ -1595,17 +1619,17 @@ int main( } - buf = (byte*) malloc(UNIV_PAGE_SIZE_MAX * 2); - xdes = (byte *) malloc(UNIV_PAGE_SIZE_MAX *2); -#ifdef MYSQL_COMPRESSION - tbuf = buf + UNIV_PAGE_SIZE_MAX; -#endif + buf_ptr = (byte*) malloc(UNIV_PAGE_SIZE_MAX * 2); + xdes_ptr = (byte*)malloc(UNIV_PAGE_SIZE_MAX * 2); + buf = (byte *) ut_align(buf_ptr, UNIV_PAGE_SIZE_MAX); + xdes = (byte *) ut_align(xdes_ptr, UNIV_PAGE_SIZE_MAX); + /* The file name is not optional. */ for (int i = 0; i < argc; ++i) { + /* Reset parameters for each file. */ filename = argv[i]; memset(&page_type, 0, sizeof(innodb_page_type)); - is_corrupted = false; partial_page_read = false; skip_page = false; @@ -1630,7 +1654,8 @@ int main( fprintf(stderr, "Error: %s cannot be found\n", filename); - goto err_exit; + exit_status = 1; + goto my_exit; } if (!read_from_stdin) { @@ -1638,35 +1663,17 @@ int main( fil_in = open_file(filename); /*If fil_in is NULL, terminate as some error encountered */ if(fil_in == NULL) { - goto err_exit; + exit_status = 1; + goto my_exit; } /* Save the current file pointer in pos variable.*/ if (0 != fgetpos(fil_in, &pos)) { perror("fgetpos"); - goto err_exit; + exit_status = 1; + goto my_exit; } } - /* Testing for lock mechanism. The innochecksum - acquire lock on given file. So other tools accessing the same - file for processsing must fail. */ -#ifdef _WIN32 - DBUG_EXECUTE_IF("innochecksum_cause_mysqld_crash", - ut_ad(page_dump_filename); - while((_access( page_dump_filename, 0)) == 0) { - sleep(1); - } - DBUG_RETURN(0); ); -#else - DBUG_EXECUTE_IF("innochecksum_cause_mysqld_crash", - ut_ad(page_dump_filename); - struct stat status_buf; - while(stat(page_dump_filename, &status_buf) == 0) { - sleep(1); - } - DBUG_RETURN(0); ); -#endif /* _WIN32 */ - /* Read the minimum page size. */ bytes = ulong(fread(buf, 1, UNIV_ZIP_SIZE_MIN, fil_in)); partial_page_read = true; @@ -1674,10 +1681,11 @@ int main( if (bytes != UNIV_ZIP_SIZE_MIN) { fprintf(stderr, "Error: Was not able to read the " "minimum page size "); - fprintf(stderr, "of %d bytes. Bytes read was %lu\n", + fprintf(stderr, "of %d bytes. Bytes read was " ULINTPF "\n", UNIV_ZIP_SIZE_MIN, bytes); - goto err_exit; + exit_status = 1; + goto my_exit; } /* enable variable is_system_tablespace when space_id of given @@ -1687,8 +1695,79 @@ int main( FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, 4)) ? true : false; + /* Determine page size, zip_size and page compression + from fsp_flags and encryption metadata from page 0 */ const page_size_t& page_size = get_page_size(buf); + ulint flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf); + ulint zip_size = page_size.is_compressed() ? page_size.logical() : 0; + logical_page_size = page_size.is_compressed() ? zip_size : 0; + physical_page_size = page_size.physical(); + srv_page_size = page_size.logical(); + bool is_compressed = FSP_FLAGS_HAS_PAGE_COMPRESSION(flags); + + if (page_size.physical() > UNIV_ZIP_SIZE_MIN) { + /* Read rest of the page 0 to determine crypt_data */ + bytes = ulong(read_file(buf, partial_page_read, page_size.physical(), fil_in)); + + if (bytes != page_size.physical()) { + fprintf(stderr, "Error: Was not able to read the " + "rest of the page "); + fprintf(stderr, "of " ULINTPF " bytes. Bytes read was " ULINTPF "\n", + page_size.physical() - UNIV_ZIP_SIZE_MIN, bytes); + + exit_status = 1; + goto my_exit; + } + partial_page_read = false; + } + + /* Now that we have full page 0 in buffer, check encryption */ + bool is_encrypted = check_encryption(filename, page_size, buf); + + /* Verify page 0 contents. Note that we can't allow + checksum mismatch on page 0, because that would mean we + could not trust it content. */ + if (!no_check) { + unsigned long long tmp_allow_mismatches = allow_mismatches; + allow_mismatches = 0; + + exit_status = verify_checksum(buf, page_size, is_encrypted, is_compressed, &mismatch_count); + + if (exit_status) { + fprintf(stderr, "Error: Page 0 checksum mismatch, can't continue. \n"); + goto my_exit; + } + allow_mismatches = tmp_allow_mismatches; + } + + if ((exit_status = rewrite_checksum(filename, fil_in, buf, + page_size, &pos, is_encrypted, is_compressed))) { + goto my_exit; + } + + if (page_type_dump) { + fprintf(fil_page_type, + "\n\nFilename::%s\n", filename); + fprintf(fil_page_type, + "========================================" + "======================================\n"); + fprintf(fil_page_type, + "\tPAGE_NO\t\t|\t\tPAGE_TYPE\t\t" + "\t|\tEXTRA INFO\n"); + fprintf(fil_page_type, + "========================================" + "======================================\n"); + } + + if (per_page_details) { + printf("page %llu ", cur_page_num); + } + + if (page_type_summary || page_type_dump) { + parse_page(buf, xdes, fil_page_type, page_size, is_encrypted); + } + pages = (ulint) (size / page_size.physical()); if (just_count) { @@ -1704,14 +1783,14 @@ int main( "(" ULINTPF " pages)\n", filename, size, pages); if (do_one_page) { fprintf(log_file, "Innochecksum: " - "checking page %" PRIuMAX "\n", + "checking page::%llu;\n", do_page); } } } else { if (is_log_enabled) { fprintf(log_file, "Innochecksum: checking " - "pages in range %" PRIuMAX " to %" PRIuMAX "\n", + "pages in range::%llu to %llu\n", start_page, use_end_page ? end_page : (pages - 1)); } @@ -1735,14 +1814,16 @@ int main( perror("Error: Unable to seek to " "necessary offset"); - goto err_exit; + exit_status = 1; + goto my_exit; } /* Save the current file pointer in pos variable. */ if (0 != fgetpos(fil_in, &pos)) { perror("fgetpos"); - goto err_exit; + exit_status = 1; + goto my_exit; } } else { @@ -1762,7 +1843,7 @@ int main( bytes = read_file(buf, partial_page_read, static_cast<ulong>( - page_size.physical()), + page_size.physical()), fil_in); partial_page_read = false; @@ -1773,34 +1854,22 @@ int main( "to seek to necessary " "offset"); - goto err_exit; + exit_status = 1; + goto my_exit; } } } } - if (page_type_dump) { - fprintf(fil_page_type, - "\n\nFilename::%s\n", filename); - fprintf(fil_page_type, - "========================================" - "======================================\n"); - fprintf(fil_page_type, - "\tPAGE_NO\t\t|\t\tPAGE_TYPE\t\t" - "\t|\tEXTRA INFO\n"); - fprintf(fil_page_type, - "========================================" - "======================================\n"); - } - /* main checksumming loop */ - cur_page_num = start_page; + cur_page_num = start_page ? start_page : cur_page_num + 1; + lastt = 0; while (!feof(fil_in)) { bytes = read_file(buf, partial_page_read, static_cast<ulong>( - page_size.physical()), fil_in); + page_size.physical()), fil_in); partial_page_read = false; if (!bytes && feof(fil_in)) { @@ -1812,14 +1881,16 @@ int main( page_size.physical()); perror(" "); - goto err_exit; + exit_status = 1; + goto my_exit; } if (bytes != page_size.physical()) { - fprintf(stderr, "Error: bytes read (%lu) " + fprintf(stderr, "Error: bytes read (" ULINTPF ") " "doesn't match page size (" ULINTPF ")\n", bytes, page_size.physical()); - goto err_exit; + exit_status = 1; + goto my_exit; } if (is_system_tablespace) { @@ -1827,60 +1898,29 @@ int main( skip_page = is_page_doublewritebuffer(buf); } else { skip_page = false; -#ifdef MYSQL_COMPRESSION - if (!page_decompress(buf, tbuf, page_size)) { - - fprintf(stderr, - "Page decompress failed"); - - goto err_exit; - } -#endif } - ulint page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); + ulint cur_page_type = mach_read_from_2(buf+FIL_PAGE_TYPE); - if (page_type == FIL_PAGE_PAGE_COMPRESSED || - page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + /* FIXME: Page compressed or Page compressed and encrypted + pages do not contain checksum. */ + if (cur_page_type == FIL_PAGE_PAGE_COMPRESSED || + cur_page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { skip_page = true; } /* If no-check is enabled, skip the checksum verification.*/ - if (!no_check) { - /* Checksum verification */ - if (!skip_page) { - is_corrupted = is_page_corrupted( - buf, page_size); - - if (is_corrupted) { - fprintf(stderr, "Fail: page " - "%" PRIuMAX " invalid\n", - cur_page_num); - - mismatch_count++; - - if(mismatch_count > allow_mismatches) { - fprintf(stderr, - "Exceeded the " - "maximum allowed " - "checksum mismatch " - "count::%" PRIuMAX "\n", - allow_mismatches); - - goto err_exit; - } - } - } + if (!no_check + && !skip_page + && (exit_status = verify_checksum(buf, page_size, + is_encrypted, is_compressed, &mismatch_count))) { + goto my_exit; } - /* Rewrite checksum */ - if (do_write - && !write_file(filename, fil_in, buf, - page_size.is_compressed(), &pos, - static_cast<ulong>(page_size.physical()))) { - - goto err_exit; + if ((exit_status = rewrite_checksum(filename, fil_in, buf, + page_size, &pos, is_encrypted, is_compressed))) { + goto my_exit; } /* end if this was the last page we were supposed to check */ @@ -1888,12 +1928,17 @@ int main( break; } + if (per_page_details) { + printf("page %llu ", cur_page_num); + } + if (page_type_summary || page_type_dump) { - parse_page(buf, xdes , fil_page_type, page_size); + parse_page(buf, xdes, fil_page_type, page_size, is_encrypted); } /* do counter increase and progress printing */ cur_page_num++; + if (verbose && !read_from_stdin) { if ((cur_page_num % 64) == 0) { now = time(0); @@ -1902,7 +1947,7 @@ int main( } if (now - lastt >= 1 && is_log_enabled) { - fprintf(log_file, "page %" PRIuMAX " " + fprintf(log_file, "page::%llu " "okay: %.3f%% done\n", (cur_page_num - 1), (float) cur_page_num / pages * 100); @@ -1933,19 +1978,19 @@ int main( fclose(log_file); } - free(buf); - free(xdes); - my_end(0); + free(buf_ptr); + free(xdes_ptr); - DBUG_RETURN(0); + my_end(exit_status); + DBUG_RETURN(exit_status); -err_exit: - if (buf) { - free(buf); +my_exit: + if (buf_ptr) { + free(buf_ptr); } - if (xdes) { - free(xdes); + if (xdes_ptr) { + free(xdes_ptr); } if (!read_from_stdin && fil_in) { @@ -1956,7 +2001,6 @@ err_exit: fclose(log_file); } - my_end(1); - - DBUG_RETURN(1); + my_end(exit_status); + DBUG_RETURN(exit_status); } diff --git a/mysql-test/suite/encryption/r/innochecksum.result b/mysql-test/suite/encryption/r/innochecksum.result index 7cd7af7b93b..6ea54f3d053 100644 --- a/mysql-test/suite/encryption/r/innochecksum.result +++ b/mysql-test/suite/encryption/r/innochecksum.result @@ -1,3 +1,5 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; # Create and populate a tables CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; @@ -5,6 +7,7 @@ CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FOR CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO; CREATE TABLE t4 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1; CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4; +CREATE TABLE t6 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB; # Write file to make mysql-test-run.pl expect the "crash", but don't # start it until it's told to # We give 30 seconds to do a clean shutdown because we do not want @@ -16,6 +19,27 @@ CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_CO # Run innochecksum on t3 # Run innochecksum on t4 # Run innochecksum on t4 +# Run innochecksum on t5 +# Run innochecksum on t6 +# Backup tables before corrupting +# Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION +# Run innochecksum on t2 +# Run innochecksum on t3 +# no encryption corrupting the field should not have effect +# Run innochecksum on t6 +# no encryption corrupting the field should not have effect +# Restore the original tables +# Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4 (post encryption checksum) +# Run innochecksum on t2 +# Run innochecksum on t3 +# Run innochecksum on t6 +# no encryption corrupting the field should not have effect +# Restore the original tables +# Corrupt FIL_DATA+10 (data) +# Run innochecksum on t2 +# Run innochecksum on t3 +# Run innochecksum on t6 +# Restore the original tables # Write file to make mysql-test-run.pl start up the server again # Cleanup -DROP TABLE t1, t2, t3, t4, t5; +DROP TABLE t1, t2, t3, t4, t5, t6; diff --git a/mysql-test/suite/encryption/t/innochecksum.test b/mysql-test/suite/encryption/t/innochecksum.test index cb1b97ebfb3..6e3962c462f 100644 --- a/mysql-test/suite/encryption/t/innochecksum.test +++ b/mysql-test/suite/encryption/t/innochecksum.test @@ -2,21 +2,25 @@ # MDEV-8773: InnoDB innochecksum does not work with encrypted or page compressed tables # -# Don't test under embedded +--source include/innodb_page_size_small.inc +# Don't test under embedded as we restart server -- source include/not_embedded.inc # Require InnoDB -- source include/have_innodb.inc -- source include/have_file_key_management_plugin.inc +-- source include/innodb_page_size_small.inc if (!$INNOCHECKSUM) { --echo Need innochecksum binary --die Need innochecksum binary } -let $innodb_compression_algorithm_orig=`SELECT @@innodb_compression_algorithm`; - +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; # zlib set global innodb_compression_algorithm = 1; +--enable_warnings --echo # Create and populate a tables CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; @@ -24,9 +28,11 @@ CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FOR CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO; CREATE TABLE t4 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1; CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4; +CREATE TABLE t6 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB; --disable_query_log --let $i = 1000 +begin; while ($i) { INSERT INTO t1 (b) VALUES (REPEAT('abcdefghijklmnopqrstuvwxyz', 100)); @@ -36,6 +42,8 @@ INSERT INTO t2 SELECT * FROM t1; INSERT INTO t3 SELECT * FROM t1; INSERT INTO t4 SELECT * FROM t1; INSERT INTO t5 SELECT * FROM t1; +INSERT INTO t6 SELECT * FROM t1; +commit; --enable_query_log let $MYSQLD_DATADIR=`select @@datadir`; @@ -44,6 +52,10 @@ let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd; let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd; let t4_IBD = $MYSQLD_DATADIR/test/t4.ibd; let t5_IBD = $MYSQLD_DATADIR/test/t5.ibd; +let t6_IBD = $MYSQLD_DATADIR/test/t6.ibd; + +let INNODB_PAGE_SIZE=`select @@innodb_page_size`; +let MYSQLD_DATADIR=`select @@datadir`; --echo # Write file to make mysql-test-run.pl expect the "crash", but don't --echo # start it until it's told to @@ -77,17 +89,198 @@ shutdown_server 30; --exec $INNOCHECKSUM $t4_IBD +--echo # Run innochecksum on t5 + +--exec $INNOCHECKSUM $t5_IBD + +--echo # Run innochecksum on t6 + +--exec $INNOCHECKSUM $t6_IBD + +--enable_result_log + +--echo # Backup tables before corrupting +--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_DATADIR/test/t1.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t2.ibd $MYSQLD_DATADIR/test/t2.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t3.ibd $MYSQLD_DATADIR/test/t3.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t4.ibd $MYSQLD_DATADIR/test/t4.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t5.ibd $MYSQLD_DATADIR/test/t5.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t6.ibd $MYSQLD_DATADIR/test/t6.ibd.backup + +# +# MDEV-11939: innochecksum mistakes a file for an encrypted one +# + +--echo # Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + +perl; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +EOF + +-- disable_result_log +--error 1 +--exec $INNOCHECKSUM $t1_IBD + +--echo # Run innochecksum on t2 + +--error 1 +--exec $INNOCHECKSUM $t2_IBD + +--echo # Run innochecksum on t3 +--echo # no encryption corrupting the field should not have effect +--exec $INNOCHECKSUM $t3_IBD + +--echo # Run innochecksum on t6 +--echo # no encryption corrupting the field should not have effect +--exec $INNOCHECKSUM $t6_IBD + --enable_result_log +--echo # Restore the original tables +--remove_file $MYSQLD_DATADIR/test/t1.ibd +--remove_file $MYSQLD_DATADIR/test/t2.ibd +--remove_file $MYSQLD_DATADIR/test/t3.ibd +--remove_file $MYSQLD_DATADIR/test/t4.ibd +--remove_file $MYSQLD_DATADIR/test/t5.ibd +--remove_file $MYSQLD_DATADIR/test/t6.ibd +--copy_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd +--copy_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd +--copy_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd +--copy_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd +--copy_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd +--copy_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd + +--echo # Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4 (post encryption checksum) + +perl; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +EOF + +-- disable_result_log +--error 1 +--exec $INNOCHECKSUM $t1_IBD + +--echo # Run innochecksum on t2 +--error 1 +--exec $INNOCHECKSUM $t2_IBD + +--echo # Run innochecksum on t3 +--error 1 +--exec $INNOCHECKSUM $t3_IBD + +--echo # Run innochecksum on t6 +--echo # no encryption corrupting the field should not have effect +--exec $INNOCHECKSUM $t6_IBD + +--enable_result_log + +--echo # Restore the original tables +--remove_file $MYSQLD_DATADIR/test/t1.ibd +--remove_file $MYSQLD_DATADIR/test/t2.ibd +--remove_file $MYSQLD_DATADIR/test/t3.ibd +--remove_file $MYSQLD_DATADIR/test/t4.ibd +--remove_file $MYSQLD_DATADIR/test/t5.ibd +--remove_file $MYSQLD_DATADIR/test/t6.ibd +--copy_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd +--copy_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd +--copy_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd +--copy_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd +--copy_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd +--copy_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd + +--echo # Corrupt FIL_DATA+10 (data) + +perl; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +EOF + +-- disable_result_log +--error 1 +--exec $INNOCHECKSUM $t1_IBD + +--echo # Run innochecksum on t2 +--error 1 +--exec $INNOCHECKSUM $t2_IBD + +--echo # Run innochecksum on t3 +--error 1 +--exec $INNOCHECKSUM $t3_IBD + +--echo # Run innochecksum on t6 +--error 1 +--exec $INNOCHECKSUM $t6_IBD + +--enable_result_log + +--echo # Restore the original tables +--move_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd +--move_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd +--move_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd +--move_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd +--move_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd +--move_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd + --echo # Write file to make mysql-test-run.pl start up the server again --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --enable_reconnect --source include/wait_until_connected_again.inc --echo # Cleanup -DROP TABLE t1, t2, t3, t4, t5; - -# reset system ---disable_query_log -EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig; ---enable_query_log +DROP TABLE t1, t2, t3, t4, t5, t6; diff --git a/mysql-test/suite/innodb_zip/r/innochecksum_2.result b/mysql-test/suite/innodb_zip/r/innochecksum_2.result index 78d8a1cbf90..582bb42f0cb 100644 --- a/mysql-test/suite/innodb_zip/r/innochecksum_2.result +++ b/mysql-test/suite/innodb_zip/r/innochecksum_2.result @@ -31,6 +31,7 @@ allow-mismatches 0 write crc32 page-type-summary FALSE page-type-dump MYSQLTEST_VARDIR/tmp/dump.txt +per-page-details FALSE log (No default value) leaf FALSE merge 0 @@ -41,7 +42,7 @@ innochecksum Ver #.#.# Copyright (c) YEAR, YEAR , Oracle, MariaDB Corporation Ab and others. InnoDB offline file checksum utility. -Usage: innochecksum [-c] [-s <start page>] [-e <end page>] [-p <page>] [-v] [-a <allow mismatches>] [-n] [-C <strict-check>] [-w <write>] [-S] [-D <page type dump>] [-l <log>] [-e] <filename or [-]> +Usage: innochecksum [-c] [-s <start page>] [-e <end page>] [-p <page>] [-i] [-v] [-a <allow mismatches>] [-n] [-C <strict-check>] [-w <write>] [-S] [-D <page type dump>] [-l <log>] [-l] [-m <merge pages>] <filename or [-]> -?, --help Displays this help and exits. -I, --info Synonym for --help. -V, --version Displays version information and exits. @@ -62,8 +63,10 @@ Usage: innochecksum [-c] [-s <start page>] [-e <end page>] [-p <page>] [-v] [-a Display a count of each page type in a tablespace. -D, --page-type-dump=name Dump the page type info for each page in a tablespace. + -i, --per-page-details + Print out per-page detail information. -l, --log=name log output. - -e, --leaf Examine leaf index pages + -f, --leaf Examine leaf index pages -m, --merge=# leaf page count if merge given number of consecutive pages @@ -81,6 +84,7 @@ allow-mismatches 0 write crc32 page-type-summary FALSE page-type-dump (No default value) +per-page-details FALSE log (No default value) leaf FALSE merge 0 diff --git a/mysql-test/suite/innodb_zip/r/innochecksum_3.result b/mysql-test/suite/innodb_zip/r/innochecksum_3.result index 800556c4ff3..aaab68b3df9 100644 --- a/mysql-test/suite/innodb_zip/r/innochecksum_3.result +++ b/mysql-test/suite/innodb_zip/r/innochecksum_3.result @@ -1,5 +1,6 @@ # Set the environmental variables call mtr.add_suppression("InnoDB: Unable to read tablespace .* page no .* into the buffer pool after 100 attempts"); +call mtr.add_suppression("InnoDB: innodb_checksum_algorithm is set to.*"); [1]: Further Test are for rewrite checksum (innodb|crc32|none) for all ibd file & start the server. CREATE TABLE tab1 (pk INTEGER NOT NULL PRIMARY KEY, linestring_key GEOMETRY NOT NULL, @@ -105,6 +106,7 @@ File::tab#.ibd # Page compressed page # Page compressed encrypted page # Other type of page + =============================================== Additional information: Undo page type: # insert, # update, # other @@ -139,6 +141,7 @@ File::tab#.ibd # Page compressed page # Page compressed encrypted page # Other type of page + =============================================== Additional information: Undo page type: # insert, # update, # other @@ -160,14 +163,14 @@ Filename::tab#.ibd ============================================================================== PAGE_NO | PAGE_TYPE | EXTRA INFO ============================================================================== -#:: # | File Space Header | - -#:: # | Insert Buffer Bitmap | - -#:: # | Inode page | - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Freshly allocated page | - +#::# | File Space Header | - +#::# | Insert Buffer Bitmap | - +#::# | Inode page | - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Freshly allocated page | - # Variables used by page type dump for ibdata1 Variables (--variable-name=value) @@ -184,6 +187,7 @@ allow-mismatches 0 write crc32 page-type-summary FALSE page-type-dump MYSQLTEST_VARDIR/tmp/dump.txt +per-page-details FALSE log (No default value) leaf FALSE merge 0 @@ -194,14 +198,14 @@ Filename::tab#.ibd ============================================================================== PAGE_NO | PAGE_TYPE | EXTRA INFO ============================================================================== -#:: # | File Space Header | - -#:: # | Insert Buffer Bitmap | - -#:: # | Inode page | - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Freshly allocated page | - +#::# | File Space Header | - +#::# | Insert Buffer Bitmap | - +#::# | Inode page | - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Freshly allocated page | - [6]: check the valid lower bound values for option # allow-mismatches,page,start-page,end-page [9]: check the both short and long options "page" and "start-page" when diff --git a/mysql-test/suite/innodb_zip/t/innochecksum_3.test b/mysql-test/suite/innodb_zip/t/innochecksum_3.test index 54c67ff1a9b..dab10dcc997 100644 --- a/mysql-test/suite/innodb_zip/t/innochecksum_3.test +++ b/mysql-test/suite/innodb_zip/t/innochecksum_3.test @@ -15,7 +15,9 @@ let MYSQLD_BASEDIR= `SELECT @@basedir`; let MYSQLD_DATADIR= `SELECT @@datadir`; let SEARCH_FILE= $MYSQLTEST_VARDIR/log/my_restart.err; + call mtr.add_suppression("InnoDB: Unable to read tablespace .* page no .* into the buffer pool after 100 attempts"); +call mtr.add_suppression("InnoDB: innodb_checksum_algorithm is set to.*"); --echo [1]: Further Test are for rewrite checksum (innodb|crc32|none) for all ibd file & start the server. diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 61006c8d89d..d1476cca67f 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -31,16 +31,17 @@ The database buffer buf_pool Created 11/5/1995 Heikki Tuuri *******************************************************/ -#include "ha_prototypes.h" - +#include "univ.i" +#include "mtr0types.h" +#include "mach0data.h" #include "page0size.h" #include "buf0buf.h" -#include "os0api.h" +#include <string.h> + +#ifdef UNIV_NONINL +#include "buf0buf.ic" +#endif -#ifdef UNIV_INNOCHECKSUM -#include "string.h" -#include "mach0data.h" -#endif /* UNIV_INNOCHECKSUM */ #ifndef UNIV_INNOCHECKSUM #include "mem0mem.h" #include "btr0btr.h" @@ -64,7 +65,6 @@ Created 11/5/1995 Heikki Tuuri #include "fsp0sysspace.h" #endif /* !UNIV_INNOCHECKSUM */ #include "page0zip.h" -#include "buf0checksum.h" #include "sync0sync.h" #include "buf0dump.h" #include "ut0new.h" @@ -645,10 +645,10 @@ buf_page_is_checksum_valid_crc32( #ifdef UNIV_INNOCHECKSUM if (log_file && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " crc32 calculated = %u;" - " recorded checksum field1 = %lu recorded" - " checksum field2 =%lu\n", cur_page_num, + " recorded checksum field1 = " ULINTPF " recorded" + " checksum field2 =" ULINTPF "\n", cur_page_num, crc32, checksum_field1, checksum_field2); } #endif /* UNIV_INNOCHECKSUM */ @@ -702,33 +702,34 @@ buf_page_is_checksum_valid_innodb( #ifdef UNIV_INNOCHECKSUM if (log_file && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB) { - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " old style: calculated =" - " %lu; recorded = %lu\n", + " " ULINTPF "; recorded = " ULINTPF "\n", cur_page_num, old_checksum, checksum_field2); - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " new style: calculated =" - " %lu; crc32 = %u; recorded = %lu\n", + " " ULINTPF "; crc32 = %u; recorded = " ULINTPF "\n", cur_page_num, new_checksum, buf_calc_page_crc32(read_buf), checksum_field1); } if (log_file && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " old style: calculated =" - " %lu; recorded checksum = %lu\n", + " " ULINTPF "; recorded checksum = " ULINTPF "\n", cur_page_num, old_checksum, checksum_field2); - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " new style: calculated =" - " %lu; recorded checksum = %lu\n", + " " ULINTPF "; recorded checksum = " ULINTPF "\n", cur_page_num, new_checksum, checksum_field1); } #endif /* UNIV_INNOCHECKSUM */ + if (checksum_field2 != mach_read_from_4(read_buf + FIL_PAGE_LSN) && checksum_field2 != old_checksum) { DBUG_LOG("checksum", @@ -788,9 +789,9 @@ buf_page_is_checksum_valid_none( if (log_file && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_NONE) { fprintf(log_file, - "page::%lu; none checksum: calculated" - " = %lu; recorded checksum_field1 = %lu" - " recorded checksum_field2 = %lu\n", + "page::%llu; none checksum: calculated" + " = " ULINTPF "; recorded checksum_field1 = " ULINTPF + " recorded checksum_field2 = " ULINTPF "\n", cur_page_num, BUF_NO_CHECKSUM_MAGIC, checksum_field1, checksum_field2); } @@ -811,16 +812,18 @@ buf_page_is_corrupted( bool check_lsn, const byte* read_buf, const page_size_t& page_size, +#ifndef UNIV_INNOCHECKSUM const fil_space_t* space) +#else + const void* space) +#endif { - ulint checksum_field1; - ulint checksum_field2; - + size_t checksum_field1 = 0; + size_t checksum_field2 = 0; #ifndef UNIV_INNOCHECKSUM DBUG_EXECUTE_IF("buf_page_import_corrupt_failure", return(true); ); - - ulint page_type = mach_read_from_2( - read_buf + FIL_PAGE_TYPE); +#endif + ulint page_type = mach_read_from_2(read_buf + FIL_PAGE_TYPE); /* We can trust page type if page compression is set on tablespace flags because page compression flag means file must have been @@ -833,15 +836,12 @@ buf_page_is_corrupted( decompressed at this stage). */ if ((page_type == FIL_PAGE_PAGE_COMPRESSED || page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) - && space && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags)) { - return(false); - } -#else - if (mach_read_from_4(read_buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0 - || mach_read_from_2(read_buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { +#ifndef UNIV_INNOCHECKSUM + && space && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags) +#endif + ) { return(false); } -#endif if (!page_size.is_compressed() && memcmp(read_buf + FIL_PAGE_LSN + 4, @@ -850,14 +850,13 @@ buf_page_is_corrupted( /* Stored log sequence numbers at the start and the end of page do not match */ - #ifndef UNIV_INNOCHECKSUM ib::info() << "Log sequence number at the start " << mach_read_from_4(read_buf + FIL_PAGE_LSN + 4) << " and the end " << mach_read_from_4(read_buf + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + 4) << " do not match"; -#endif +#endif /* UNIV_INNOCHECKSUM */ return(true); } @@ -920,10 +919,9 @@ buf_page_is_corrupted( && *reinterpret_cast<const ib_uint64_t*>( read_buf + FIL_PAGE_LSN) == 0) { - /* make sure that the page is really empty */ - - ulint i; + ulint i; + /* make sure that the page is really empty */ for (i = 0; i < page_size.logical(); ++i) { /* The FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID has been @@ -944,7 +942,7 @@ buf_page_is_corrupted( #ifdef UNIV_INNOCHECKSUM if (i >= page_size.logical()) { if (log_file) { - fprintf(log_file, "Page::%lu" + fprintf(log_file, "Page::%llu" " is empty and uncorrupted\n", cur_page_num); } @@ -978,31 +976,34 @@ buf_page_is_corrupted( if (buf_page_is_checksum_valid_none(read_buf, checksum_field1, checksum_field2)) { -#ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_NONE, page_id); +#endif /* !UNIV_INNOCHECKSUM */ } -#else /* !UNIV_INNOCHECKSUM */ + +#ifdef UNIV_INNOCHECKSUM if (log_file) { - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " old style: calculated = %u;" - " recorded = " ULINTPF "\n", + " recorded = " ULINTPF ";\n", cur_page_num, buf_calc_page_old_checksum(read_buf), checksum_field2); - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " new style: calculated = %u;" - " crc32 = %u; recorded = " ULINTPF "\n", + " crc32 = %u; recorded = " ULINTPF ";\n", cur_page_num, buf_calc_page_new_checksum(read_buf), buf_calc_page_crc32(read_buf), checksum_field1); } -#endif /* !UNIV_INNOCHECKSUM */ +#endif /* UNIV_INNOCHECKSUM */ + return(false); } @@ -1022,15 +1023,16 @@ buf_page_is_corrupted( if (buf_page_is_checksum_valid_innodb(read_buf, checksum_field1, checksum_field2)) { -#ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_INNODB, page_id); +#endif } -#endif /* !UNIV_INNOCHECKSUM */ + return(false); } @@ -1038,13 +1040,13 @@ buf_page_is_corrupted( if (!legacy_checksum_checked && buf_page_is_checksum_valid_crc32( read_buf, checksum_field1, checksum_field2, true)) { - legacy_big_endian_checksum = true; - return(false); + legacy_big_endian_checksum = true; + return(false); } #ifdef UNIV_INNOCHECKSUM if (log_file) { - fprintf(log_file, "Fail; page %lu" + fprintf(log_file, "Fail; page::%llu;" " invalid (fails crc32 checksum)\n", cur_page_num); } @@ -1061,30 +1063,32 @@ buf_page_is_corrupted( if (buf_page_is_checksum_valid_none(read_buf, checksum_field1, checksum_field2)) { -#ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_NONE, page_id); +#endif } -#else /* !UNIV_INNOCHECKSUM */ +#ifdef UNIV_INNOCHECKSUM if (log_file) { - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " old style: calculated = %u;" - " recorded = %lu\n", cur_page_num, + " recorded = %zu;\n", cur_page_num, buf_calc_page_old_checksum(read_buf), checksum_field2); - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " new style: calculated = %u;" - " crc32 = %u; recorded = %lu\n", + " crc32 = %u; recorded = %zu;\n", cur_page_num, buf_calc_page_new_checksum(read_buf), buf_calc_page_crc32(read_buf), checksum_field1); } -#endif /* !UNIV_INNOCHECKSUM */ +#endif /* UNIV_INNOCHECKSUM */ + return(false); } @@ -1092,26 +1096,28 @@ buf_page_is_corrupted( checksum_field1, checksum_field2, false) || buf_page_is_checksum_valid_crc32(read_buf, checksum_field1, checksum_field2, true)) { -#ifndef UNIV_INNOCHECKSUM + if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_CRC32, page_id); +#endif } -#endif /* !UNIV_INNOCHECKSUM */ return(false); } #ifdef UNIV_INNOCHECKSUM if (log_file) { - fprintf(log_file, "Fail; page %lu" + fprintf(log_file, "Fail; page::%llu;" " invalid (fails innodb checksum)\n", cur_page_num); } #endif /* UNIV_INNOCHECKSUM */ + return(true); case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: @@ -1147,11 +1153,12 @@ buf_page_is_corrupted( #ifdef UNIV_INNOCHECKSUM if (log_file) { - fprintf(log_file, "Fail; page %lu" + fprintf(log_file, "Fail; page::%llu;" " invalid (fails none checksum)\n", cur_page_num); } #endif /* UNIV_INNOCHECKSUM */ + return(true); case SRV_CHECKSUM_ALGORITHM_NONE: diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 7917cbb528b..6e431a6ee0f 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -24,6 +24,7 @@ Modified Jan Lindström jan.lindstrom@mariadb.com *******************************************************/ #include "fil0fil.h" +#include "mtr0types.h" #include "mach0data.h" #include "page0size.h" #include "page0zip.h" diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 9d5f373f5de..67de1174d25 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -211,6 +211,7 @@ struct buf_pools_list_size_t { ulint unzip_LRU_bytes; /*!< unzip_LRU size in bytes */ ulint flush_list_bytes; /*!< flush_list size in bytes */ }; +#endif /* !UNIV_INNOCHECKSUM */ /** Page identifier. */ class page_id_t { @@ -333,6 +334,7 @@ operator<<( std::ostream& out, const page_id_t& page_id); +#ifndef UNIV_INNOCHECKSUM /********************************************************************//** Acquire mutex on all buffer pool instances */ UNIV_INLINE @@ -823,9 +825,16 @@ buf_page_is_corrupted( bool check_lsn, const byte* read_buf, const page_size_t& page_size, +#ifndef UNIV_INNOCHECKSUM const fil_space_t* space = NULL) +#else + const void* space = NULL) +#endif MY_ATTRIBUTE((warn_unused_result)); + + #ifndef UNIV_INNOCHECKSUM + /**********************************************************************//** Gets the space id, page offset, and byte offset within page of a pointer pointing to a buffer frame containing a file page. */ diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index 06785430c25..13b3ec4e37e 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -30,6 +30,7 @@ Created 04/01/2015 Jan Lindström #include "os0event.h" #include "my_crypt.h" +#endif /*! UNIV_INNOCHECKSUM */ /** * Magic pattern in start of crypt data on page 0 @@ -101,6 +102,8 @@ struct fil_space_rotate_state_t } scrubbing; }; +#ifndef UNIV_INNOCHECKSUM + struct fil_space_crypt_t : st_encryption_scheme { public: diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index bf231565657..097be6a5f96 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -28,8 +28,6 @@ Created 10/25/1995 Heikki Tuuri #define fil0fil_h #include "univ.i" -struct fil_space_t; - #ifndef UNIV_INNOCHECKSUM #include "log0recv.h" @@ -281,22 +279,23 @@ but in the MySQL Embedded Server Library and mysqlbackup it is not the default directory, and we must set the base file path explicitly */ extern const char* fil_path_to_mysql_datadir; -/** Initial size of a single-table tablespace in pages */ -#define FIL_IBD_FILE_INITIAL_SIZE 4 - -/** 'null' (undefined) page offset in the context of file spaces */ -#define FIL_NULL ULINT32_UNDEFINED - /* Space address data type; this is intended to be used when addresses accurate to a byte are stored in file pages. If the page part of the address is FIL_NULL, the address is considered undefined. */ typedef byte fil_faddr_t; /*!< 'type' definition in C: an address stored in a file page is a string of bytes */ +#endif /* !UNIV_INNOCHECKSUM */ + +/** Initial size of a single-table tablespace in pages */ +#define FIL_IBD_FILE_INITIAL_SIZE 4 + +/** 'null' (undefined) page offset in the context of file spaces */ +#define FIL_NULL ULINT32_UNDEFINED + #define FIL_ADDR_PAGE 0 /* first in address is the page offset */ #define FIL_ADDR_BYTE 4 /* then comes 2-byte byte offset within page*/ -#endif /* !UNIV_INNOCHECKSUM */ #define FIL_ADDR_SIZE 6 /* address size is 6 bytes */ #ifndef UNIV_INNOCHECKSUM @@ -431,8 +430,6 @@ index */ #define fil_page_index_page_check(page) \ fil_page_type_is_index(fil_page_get_type(page)) -#ifndef UNIV_INNOCHECKSUM - /** Enum values for encryption table option */ enum fil_encryption_t { /** Encrypted if innodb_encrypt_tables=ON (srv_encrypt_tables) */ @@ -454,6 +451,8 @@ extern ulint fil_n_pending_tablespace_flushes; /** Number of files currently open */ extern ulint fil_n_file_opened; +#ifndef UNIV_INNOCHECKSUM + /** Look up a tablespace. The caller should hold an InnoDB table lock or a MDL that prevents the tablespace from being dropped during the operation, @@ -1315,6 +1314,7 @@ fil_page_reset_type( byte* page, ulint type, mtr_t* mtr); + /** Get the file page type. @param[in] page file page @return page type */ @@ -1325,6 +1325,7 @@ fil_page_get_type( { return(mach_read_from_2(page + FIL_PAGE_TYPE)); } + /** Check (and if needed, reset) the page type. Data files created before MySQL 5.1 may contain garbage in the FIL_PAGE_TYPE field. diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index f9167e0462e..e5057b30501 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -158,8 +158,6 @@ descriptor page, but used only in the first. */ FSP_FREE_LIMIT at a time */ /* @} */ -#ifndef UNIV_INNOCHECKSUM - /* @defgroup File Segment Inode Constants (moved from fsp0fsp.c) @{ */ /* FILE SEGMENT INODE @@ -293,6 +291,7 @@ the extent are free and which contain old tuple version to clean. */ /** Offset of the descriptor array on a descriptor page */ #define XDES_ARR_OFFSET (FSP_HEADER_OFFSET + FSP_HEADER_SIZE) +#ifndef UNIV_INNOCHECKSUM /* @} */ /**********************************************************************//** diff --git a/storage/innobase/include/fsp0fsp.ic b/storage/innobase/include/fsp0fsp.ic index 6317a67a089..2da3320eef7 100644 --- a/storage/innobase/include/fsp0fsp.ic +++ b/storage/innobase/include/fsp0fsp.ic @@ -53,6 +53,7 @@ xdes_calc_descriptor_index( return(ut_2pow_remainder(offset, page_size.physical()) / FSP_EXTENT_SIZE); } +#endif /*!UNIV_INNOCHECKSUM */ /**********************************************************************//** Gets a descriptor bit of a page. @@ -80,6 +81,7 @@ xdes_get_bit( bit_index)); } +#ifndef UNIV_INNOCHECKSUM /** Calculates the page where the descriptor of a page resides. @param[in] page_size page size @param[in] offset page offset diff --git a/storage/innobase/include/mach0data.h b/storage/innobase/include/mach0data.h index 5a5574b2537..684a3155a16 100644 --- a/storage/innobase/include/mach0data.h +++ b/storage/innobase/include/mach0data.h @@ -63,6 +63,7 @@ mach_write_to_2( /*============*/ byte* b, /*!< in: pointer to two bytes where to store */ ulint n); /*!< in: ulint integer to be stored, >= 0, < 64k */ +#endif /* !UNIV_INNOCHECKSUM */ /** The following function is used to fetch data from 2 consecutive bytes. The most significant byte is at the lowest address. @param[in] b pointer to 2 bytes where to store @@ -72,6 +73,8 @@ uint16_t mach_read_from_2( const byte* b) MY_ATTRIBUTE((warn_unused_result)); + +#ifndef UNIV_INNOCHECKSUM /********************************************************//** The following function is used to convert a 16-bit data item to the canonical format, for fast bytewise equality test @@ -362,6 +365,8 @@ mach_write_ulonglong( ulint len, /*!< in: length of dest */ bool usign); /*!< in: signed or unsigned flag */ +#endif /* !UNIV_INNOCHECKSUM */ + /** Read 1 to 4 bytes from a file page buffered in the buffer pool. @param[in] ptr pointer where to read @param[in] type MLOG_1BYTE, MLOG_2BYTES, or MLOG_4BYTES @@ -373,8 +378,6 @@ mach_read_ulint( mlog_id_t type) MY_ATTRIBUTE((warn_unused_result)); -#endif /* !UNIV_INNOCHECKSUM */ - #include "mach0data.ic" #endif diff --git a/storage/innobase/include/mach0data.ic b/storage/innobase/include/mach0data.ic index c89e4960480..9c4760b6d3a 100644 --- a/storage/innobase/include/mach0data.ic +++ b/storage/innobase/include/mach0data.ic @@ -865,6 +865,8 @@ mach_write_ulonglong( } } +#endif /* !UNIV_INNOCHECKSUM */ + /** Read 1 to 4 bytes from a file page buffered in the buffer pool. @param[in] ptr pointer where to read @param[in] type MLOG_1BYTE, MLOG_2BYTES, or MLOG_4BYTES @@ -889,5 +891,3 @@ mach_read_ulint( ut_error; return(0); } - -#endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/include/page0page.h b/storage/innobase/include/page0page.h index a8c5f11c17b..0aca13ac159 100644 --- a/storage/innobase/include/page0page.h +++ b/storage/innobase/include/page0page.h @@ -85,8 +85,6 @@ Otherwise written as 0. @see PAGE_ROOT_AUTO_INC */ This field should not be written to after page creation. */ -#ifndef UNIV_INNOCHECKSUM - #define PAGE_BTR_SEG_LEAF 36 /* file segment header for the leaf pages in a B-tree: defined only on the root page of a B-tree, but not in the root of an ibuf tree */ @@ -141,6 +139,8 @@ Otherwise written as 0. @see PAGE_ROOT_AUTO_INC */ #define PAGE_SAME_PAGE 4 #define PAGE_NO_DIRECTION 5 +#ifndef UNIV_INNOCHECKSUM + /* PAGE DIRECTORY ============== */ diff --git a/storage/innobase/include/page0page.ic b/storage/innobase/include/page0page.ic index 1c680ce7cc4..2ad5f26dcc1 100644 --- a/storage/innobase/include/page0page.ic +++ b/storage/innobase/include/page0page.ic @@ -27,8 +27,8 @@ Created 2/2/1994 Heikki Tuuri #ifndef page0page_ic #define page0page_ic -#include "mach0data.h" #ifndef UNIV_INNOCHECKSUM +#include "mach0data.h" #ifdef UNIV_DEBUG # include "log0recv.h" #endif /* !UNIV_DEBUG */ @@ -40,6 +40,7 @@ Created 2/2/1994 Heikki Tuuri #undef UNIV_INLINE #define UNIV_INLINE #endif +#endif /* !UNIV_INNOCHECKSUM */ /************************************************************//** Gets the start of a page. @@ -53,6 +54,7 @@ page_align( return((page_t*) ut_align_down(ptr, UNIV_PAGE_SIZE)); } +#ifndef UNIV_INNOCHECKSUM /************************************************************//** Gets the offset within a page. @return offset from the start of the page */ @@ -181,7 +183,6 @@ page_header_get_field( } #ifndef UNIV_INNOCHECKSUM - /*************************************************************//** Sets the given header field. */ UNIV_INLINE @@ -285,6 +286,8 @@ page_header_reset_last_insert( } } +#endif /* !UNIV_INNOCHECKSUM */ + /************************************************************//** Determine whether the page is in new-style compact format. @return nonzero if the page is in compact format, zero if it is in @@ -298,6 +301,7 @@ page_is_comp( return(page[PAGE_HEADER + PAGE_N_HEAP] & 0x80); } +#ifndef UNIV_INNOCHECKSUM /************************************************************//** TRUE if the record is on a page in compact format. @return nonzero if in compact format */ @@ -326,6 +330,8 @@ page_rec_get_heap_no( } } +#endif /* !UNIV_INNOCHECKSUM */ + /************************************************************//** Determine whether the page is a B-tree leaf. @return true if the page is a B-tree leaf (PAGE_LEVEL = 0) */ @@ -338,6 +344,7 @@ page_is_leaf( return(!*(const uint16*) (page + (PAGE_HEADER + PAGE_LEVEL))); } +#ifndef UNIV_INNOCHECKSUM /************************************************************//** Determine whether the page is empty. @return true if the page is empty (PAGE_N_RECS = 0) */ @@ -627,6 +634,8 @@ page_get_middle_rec( return(page_rec_get_nth(page, middle)); } +#endif /* !UNIV_INNOCHECKSUM */ + /*************************************************************//** Gets the page number. @return page number */ @@ -640,6 +649,7 @@ page_get_page_no( return(mach_read_from_4(page + FIL_PAGE_OFFSET)); } +#ifndef UNIV_INNOCHECKSUM /*************************************************************//** Gets the tablespace identifier. @return space id */ @@ -653,6 +663,8 @@ page_get_space_id( return(mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID)); } +#endif /* !UNIV_INNOCHECKSUM */ + /*************************************************************//** Gets the number of user records on page (infimum and supremum records are not user records). @@ -666,6 +678,7 @@ page_get_n_recs( return(page_header_get_field(page, PAGE_N_RECS)); } +#ifndef UNIV_INNOCHECKSUM /*************************************************************//** Gets the number of dir slots in directory. @return number of slots */ @@ -1054,6 +1067,8 @@ page_rec_get_base_extra_size( return(REC_N_NEW_EXTRA_BYTES + (ulint) !page_rec_is_comp(rec)); } +#endif /* UNIV_INNOCHECKSUM */ + /************************************************************//** Returns the sum of the sizes of the records in the record list, excluding the infimum and supremum records. @@ -1077,6 +1092,7 @@ page_get_data_size( return(ret); } +#ifndef UNIV_INNOCHECKSUM /************************************************************//** Allocates a block of memory from the free list of an index page. */ UNIV_INLINE diff --git a/storage/innobase/include/rem0rec.h b/storage/innobase/include/rem0rec.h index 96d3bdfab8f..8d3f87450f8 100644 --- a/storage/innobase/include/rem0rec.h +++ b/storage/innobase/include/rem0rec.h @@ -27,12 +27,14 @@ Created 5/30/1994 Heikki Tuuri #ifndef rem0rec_h #define rem0rec_h +#ifndef UNIV_INNOCHECKSUM #include "univ.i" #include "data0data.h" #include "rem0types.h" #include "mtr0types.h" #include "page0types.h" #include "trx0types.h" +#endif /*! UNIV_INNOCHECKSUM */ #include <ostream> #include <sstream> @@ -92,6 +94,7 @@ offsets[] array, first passed to rec_get_offsets() */ #define REC_OFFS_NORMAL_SIZE OFFS_IN_REC_NORMAL_SIZE #define REC_OFFS_SMALL_SIZE 10 +#ifndef UNIV_INNOCHECKSUM /******************************************************//** The following function is used to get the pointer of the next chained record on the same page. @@ -1112,4 +1115,5 @@ int wsrep_rec_get_foreign_key( #include "rem0rec.ic" +#endif /* !UNIV_INNOCHECKSUM */ #endif /* rem0rec_h */ diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index da4db5b9435..a5635fc4ecd 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -108,13 +108,8 @@ support cross-platform development and expose comonly used SQL names. */ #include <unistd.h> #endif -#ifdef UNIV_INNOCHECKSUM -extern bool strict_verify; -extern FILE* log_file; -extern uintmax_t cur_page_num; -#endif /* UNIV_INNOCHECKSUM */ - #include "my_pthread.h" + /* Following defines are to enable performance schema instrumentation in each of five InnoDB modules if HAVE_PSI_INTERFACE is defined. */ @@ -487,6 +482,12 @@ in both 32-bit and 64-bit environments. */ # define UINT64PFx "%016" PRIx64 #endif +#ifdef UNIV_INNOCHECKSUM +extern bool strict_verify; +extern FILE* log_file; +extern unsigned long long cur_page_num; +#endif /* UNIV_INNOCHECKSUM */ + typedef int64_t ib_int64_t; typedef uint64_t ib_uint64_t; typedef uint32_t ib_uint32_t; diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index ffb8808d6b4..b6e0409459b 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -4980,24 +4980,19 @@ page_zip_verify_checksum( const void* data, /*!< in: compressed page */ ulint size) /*!< in: size of compressed page */ { - const unsigned char* p = static_cast<const unsigned char*>(data) - + FIL_PAGE_SPACE_OR_CHKSUM; - - const uint32_t stored = static_cast<uint32_t>( - mach_read_from_4(p)); - -#ifdef UNIV_INNOCHECKSUM - p = static_cast<const unsigned char*>(data) + FIL_PAGE_TYPE; - bool no_checksum = (mach_read_from_2(p) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); - p = static_cast<const unsigned char*>(data) + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION; - bool encrypted = (mach_read_from_4(p) != 0); - p = static_cast<const unsigned char*>(data) + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4; - const uint32_t checksum = static_cast<uint32_t>(mach_read_from_4(p)); - - if (no_checksum) { - return (TRUE); - } -#endif + ib_uint32_t stored; + ib_uint32_t calc; + + stored = static_cast<ib_uint32_t>(mach_read_from_4( + static_cast<const unsigned char*>(data) + FIL_PAGE_SPACE_OR_CHKSUM)); + + ulint page_no MY_ATTRIBUTE((unused)) = + mach_read_from_4(static_cast<const unsigned char*> + (data) + FIL_PAGE_OFFSET); + ulint space_id MY_ATTRIBUTE((unused)) = + mach_read_from_4(static_cast<const unsigned char*> + (data) + FIL_PAGE_SPACE_ID); + const page_id_t page_id(space_id, page_no); #if FIL_PAGE_LSN % 8 #error "FIL_PAGE_LSN must be 64 bit aligned" @@ -5017,7 +5012,7 @@ page_zip_verify_checksum( } if (i >= size) { if (log_file) { - fprintf(log_file, "Page::%lu is empty and" + fprintf(log_file, "Page::%llu is empty and" " uncorrupted\n", cur_page_num); } @@ -5041,21 +5036,12 @@ page_zip_verify_checksum( return(TRUE); } -#ifndef UNIV_INNOCHECKSUM - ulint page_no = mach_read_from_4(static_cast< - const unsigned char*> - (data) + FIL_PAGE_OFFSET); - ulint space_id = mach_read_from_4(static_cast< - const unsigned char*> - (data) + FIL_PAGE_SPACE_ID); - const page_id_t page_id(space_id, page_no); -#endif /* UNIV_INNOCHECKSUM */ - - const uint32_t calc = page_zip_calc_checksum(data, size, curr_algo); + calc = static_cast<ib_uint32_t>(page_zip_calc_checksum( + data, size, curr_algo)); #ifdef UNIV_INNOCHECKSUM if (log_file) { - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " %s checksum: calculated = %u;" " recorded = %u\n", cur_page_num, buf_checksum_algorithm_name( @@ -5070,21 +5056,17 @@ page_zip_verify_checksum( data, size, SRV_CHECKSUM_ALGORITHM_CRC32); if (log_file) { - fprintf(log_file, "page::%lu: crc32 checksum:" + fprintf(log_file, "page::%llu: crc32 checksum:" " calculated = %u; recorded = %u\n", cur_page_num, crc32, stored); - fprintf(log_file, "page::%lu: none checksum:" + fprintf(log_file, "page::%llu: none checksum:" " calculated = %lu; recorded = %u\n", cur_page_num, BUF_NO_CHECKSUM_MAGIC, stored); } } #endif /* UNIV_INNOCHECKSUM */ - if (stored == calc -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && stored == checksum) -#endif - ) { + if (stored == calc) { return(TRUE); } @@ -5116,11 +5098,7 @@ page_zip_verify_checksum( if (legacy_big_endian_checksum) { const uint32_t calculated = page_zip_calc_checksum(data, size, curr_algo, true); - if (stored == calculated -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && calculated == checksum) -#endif - ) { + if (stored == calculated) { return(TRUE); } @@ -5130,11 +5108,7 @@ page_zip_verify_checksum( uint32_t calculated = page_zip_calc_checksum(data, size, SRV_CHECKSUM_ALGORITHM_INNODB); - if (stored == calculated -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && stored == checksum) -#endif - ) { + if (stored == calculated) { #ifndef UNIV_INNOCHECKSUM if (curr_algo @@ -5154,11 +5128,7 @@ page_zip_verify_checksum( /* If legacy checksum is not checked, do it now. */ if ((legacy_checksum_checked - && stored == calculated) -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && calculated == checksum) -#endif - ) { + && stored == calculated)) { legacy_big_endian_checksum = true; return(TRUE); } @@ -5168,11 +5138,7 @@ page_zip_verify_checksum( case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: case SRV_CHECKSUM_ALGORITHM_INNODB: { - if (stored == BUF_NO_CHECKSUM_MAGIC -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && checksum == BUF_NO_CHECKSUM_MAGIC) -#endif - ) { + if (stored == BUF_NO_CHECKSUM_MAGIC) { #ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { @@ -5191,14 +5157,8 @@ page_zip_verify_checksum( uint32_t calculated1; if (stored == calculated -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && checksum == calculated) -#endif - || stored == (calculated1 = + || stored == (calculated1 = page_zip_calc_checksum(data, size, SRV_CHECKSUM_ALGORITHM_CRC32, true)) -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && checksum == calculated1) -#endif ) { #ifndef UNIV_INNOCHECKSUM if (curr_algo @@ -5222,12 +5182,7 @@ page_zip_verify_checksum( data, size, SRV_CHECKSUM_ALGORITHM_CRC32, true); if (stored == calculated - || stored == calculated1 -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && checksum == calculated) - || ( encrypted == true && checksum == calculated1) -#endif - ) { + || stored == calculated1) { #ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, @@ -5240,11 +5195,7 @@ page_zip_verify_checksum( calculated = page_zip_calc_checksum( data, size, SRV_CHECKSUM_ALGORITHM_INNODB); - if (stored == calculated -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && checksum == calculated) -#endif - ) { + if (stored == calculated) { #ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( |