diff options
author | Marko Mäkelä <marko.makela@mariadb.com> | 2017-08-08 09:47:00 +0300 |
---|---|---|
committer | Marko Mäkelä <marko.makela@mariadb.com> | 2017-08-08 09:47:00 +0300 |
commit | 1f0a22acbd906f391ed7ead6cad5ef50b89236c1 (patch) | |
tree | f9c5899ee90150032e04b4243726e9777a74b510 | |
parent | 4649fb37e311a2b0e5d1a81ffc2f4ac4443e42e5 (diff) | |
parent | 86e0a73eaa166f752d62b31e96925e29c4fe0c8c (diff) | |
download | mariadb-git-1f0a22acbd906f391ed7ead6cad5ef50b89236c1.tar.gz |
Merge 10.2 into bb-10.2-ext
41 files changed, 1114 insertions, 906 deletions
diff --git a/extra/CMakeLists.txt b/extra/CMakeLists.txt index 9cc5e7faa86..8c73e23d72b 100644 --- a/extra/CMakeLists.txt +++ b/extra/CMakeLists.txt @@ -79,7 +79,6 @@ IF(WITH_INNOBASE_STORAGE_ENGINE OR WITH_XTRADB_STORAGE_ENGINE) ../storage/innobase/ut/ut0ut.cc ../storage/innobase/buf/buf0buf.cc ../storage/innobase/page/page0zip.cc - ../storage/innobase/os/os0file.cc ../storage/innobase/fil/fil0crypt.cc ) diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc index fe828026b4b..a0976970db7 100644 --- a/extra/innochecksum.cc +++ b/extra/innochecksum.cc @@ -26,40 +26,58 @@ Published with a permission. */ -#include <my_config.h> #include <my_global.h> #include <stdio.h> #include <stdlib.h> #include <time.h> #include <sys/types.h> #include <sys/stat.h> -#ifdef HAVE_UNISTD_H +#ifndef __WIN__ # include <unistd.h> #endif #include <my_getopt.h> #include <m_string.h> -#include <welcome_copyright_notice.h> /* ORACLE_WELCOME_COPYRIGHT_NOTICE */ +#include <welcome_copyright_notice.h> /* ORACLE_WELCOME_COPYRIGHT_NOTICE */ /* Only parts of these files are included from the InnoDB codebase. The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */ -#include "univ.i" /* include all of this */ -#include "page0size.h" /* page_size_t */ -#include "page0zip.h" /* page_zip_calc_checksum() */ -#include "page0page.h" /* PAGE_* */ -#include "trx0undo.h" /* TRX_UNDO_* */ -#include "fut0lst.h" /* FLST_NODE_SIZE */ -#include "buf0checksum.h" /* buf_calc_page_*() */ -#include "fil0fil.h" /* FIL_* */ -#include "fil0crypt.h" -#include "os0file.h" -#include "fsp0fsp.h" /* fsp_flags_get_page_size() & - fsp_flags_get_zip_size() */ -#include "mach0data.h" /* mach_read_from_4() */ -#include "ut0crc32.h" /* ut_crc32_init() */ -#include "fsp0pagecompress.h" /* fil_get_compression_alg_name */ +typedef void fil_space_t; + +#include "univ.i" /* include all of this */ +#include "page0size.h" + +#define FLST_BASE_NODE_SIZE (4 + 2 * FIL_ADDR_SIZE) +#define FLST_NODE_SIZE (2 * FIL_ADDR_SIZE) +#define FSEG_PAGE_DATA FIL_PAGE_DATA +#define FSEG_HEADER_SIZE 10 +#define UT_BITS_IN_BYTES(b) (((b) + 7) / 8) + +#include "ut0ut.h" #include "ut0byte.h" +#include "mtr0types.h" #include "mach0data.h" +#include "fsp0types.h" +#include "rem0rec.h" +#include "buf0checksum.h" /* buf_calc_page_*() */ +#include "buf0buf.h" /* buf_page_is_corrupted */ +#include "fil0fil.h" /* FIL_* */ +#include "page0page.h" /* PAGE_* */ +#include "page0zip.h" /* page_zip_*() */ +#include "trx0undo.h" /* TRX_* */ +#include "fsp0fsp.h" /* fsp_flags_get_page_size() & + fsp_flags_get_zip_size() */ +#include "ut0crc32.h" /* ut_crc32_init() */ +#include "fsp0pagecompress.h" /* fil_get_compression_alg_name */ +#include "fil0crypt.h" /* fil_space_verify_crypt_checksum */ + +#include <string.h> + +#ifdef UNIV_NONINL +# include "fsp0fsp.ic" +# include "mach0data.ic" +# include "ut0rnd.ic" +#endif #ifndef PRIuMAX #define PRIuMAX "llu" @@ -68,25 +86,29 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */ /* Global variables */ static bool verbose; static bool just_count; -static uintmax_t start_page; -static uintmax_t end_page; -static uintmax_t do_page; +static unsigned long long start_page; +static unsigned long long end_page; +static unsigned long long do_page; static bool use_end_page; static bool do_one_page; -/* replaces declaration in srv0srv.c */ -ulong srv_page_size; -page_size_t univ_page_size(0, 0, false); +static my_bool do_leaf; +static my_bool per_page_details; +static ulong n_merge; extern ulong srv_checksum_algorithm; +static ulong physical_page_size; /* Page size in bytes on disk. */ +static ulong logical_page_size; /* Page size when uncompressed. */ +ulong srv_page_size; +page_size_t univ_page_size(0, 0, false); /* Current page number (0 based). */ -uintmax_t cur_page_num; +unsigned long long cur_page_num; /* Skip the checksum verification. */ static bool no_check; /* Enabled for strict checksum verification. */ -bool strict_verify; +bool strict_verify = 0; /* Enabled for rewrite checksum. */ static bool do_write; /* Mismatches count allowed (0 by default). */ -static uintmax_t allow_mismatches; +static unsigned long long allow_mismatches=0; static bool page_type_summary; static bool page_type_dump; /* Store filename for page-type-dump option. */ @@ -99,8 +121,7 @@ char* log_filename = NULL; FILE* log_file = NULL; /* Enabled for log write option. */ static bool is_log_enabled = false; -static my_bool do_leaf; -static ulong n_merge; + #ifndef _WIN32 /* advisory lock for non-window system. */ struct flock lk; @@ -198,75 +219,6 @@ struct per_index_stats { std::map<unsigned long long, per_index_stats> index_ids; -bool encrypted = false; - -ulint -page_is_comp( -/*=========*/ - const page_t* page) /*!< in: index page */ -{ - return(page_header_get_field(page, PAGE_N_HEAP) & 0x8000); -} - -bool -page_is_leaf( -/*=========*/ - const page_t* page) /*!< in: page */ -{ - return(!*(const uint16*) (page + (PAGE_HEADER + PAGE_LEVEL))); -} - -ulint -page_get_page_no( -/*=============*/ - const page_t* page) /*!< in: page */ -{ - return(mach_read_from_4(page + FIL_PAGE_OFFSET)); -} -#define FSEG_HEADER_SIZE 10 /*!< Length of the file system - header, in bytes */ -#define REC_N_NEW_EXTRA_BYTES 5 -#define REC_N_OLD_EXTRA_BYTES 6 -#define PAGE_DATA (PAGE_HEADER + 36 + 2 * FSEG_HEADER_SIZE) - /* start of data on the page */ -#define PAGE_NEW_SUPREMUM (PAGE_DATA + 2 * REC_N_NEW_EXTRA_BYTES + 8) - /* offset of the page supremum record on a - new-style compact page */ -#define PAGE_NEW_SUPREMUM_END (PAGE_NEW_SUPREMUM + 8) -#define PAGE_OLD_SUPREMUM (PAGE_DATA + 2 + 2 * REC_N_OLD_EXTRA_BYTES + 8) - /* offset of the page supremum record on an - old-style page */ -#define PAGE_OLD_SUPREMUM_END (PAGE_OLD_SUPREMUM + 9) - /* offset of the page supremum record end on - an old-style page */ -#define FLST_BASE_NODE_SIZE (4 + 2 * 6) -#define XDES_ARR_OFFSET (FSP_HEADER_OFFSET + FSP_HEADER_SIZE) -#define XDES_FREE_BIT 0 -#define XDES_BITMAP (FLST_NODE_SIZE + 12) -#define XDES_BITS_PER_PAGE 2 -#define UT_BITS_IN_BYTES(b) (((b) + 7) / 8) -#define XDES_SIZE \ - (XDES_BITMAP \ - + UT_BITS_IN_BYTES(FSP_EXTENT_SIZE * XDES_BITS_PER_PAGE)) - -ulint -page_get_data_size( -/*===============*/ - const page_t* page) /*!< in: index page */ -{ - ulint ret; - - ret = (ulint)(page_header_get_field(page, PAGE_HEAP_TOP) - - (page_is_comp(page) - ? PAGE_NEW_SUPREMUM_END - : PAGE_OLD_SUPREMUM_END) - - page_header_get_field(page, PAGE_GARBAGE)); - - ut_ad(ret < UNIV_PAGE_SIZE); - - return(ret); -} - void print_index_leaf_stats( unsigned long long id, const per_index_stats& index, @@ -280,8 +232,7 @@ void print_index_leaf_stats( fprintf(fil_out, "page_no\tdata_size\tn_recs\n"); while (it_page != index.leaves.end()) { const per_page_stats& stat = it_page->second; - fprintf(fil_out, "%llu\t" ULINTPF "\t" ULINTPF "\n", - it_page->first, stat.data_size, stat.n_recs); + fprintf(fil_out, "%llu\t" ULINTPF "\t" ULINTPF "\n", it_page->first, stat.data_size, stat.n_recs); page_no = stat.right_page_no; it_page = index.leaves.find(page_no); } @@ -307,6 +258,7 @@ void defrag_analysis( break; } } + if (index.max_data_size) { n_pages += data_size_total / index.max_data_size; if (data_size_total % index.max_data_size != 0) { @@ -316,15 +268,16 @@ void defrag_analysis( } if (index.leaf_pages) { - fprintf(fil_out, "count = " ULINTPF " free = " ULINTPF "\n", - index.count, index.free_pages); + fprintf(fil_out, "count = " ULINTPF " free = " ULINTPF "\n", index.count, index.free_pages); + } + + if (!n_leaf_pages) { + n_leaf_pages = 1; } - fprintf(fil_out, "%llu\t\t%llu\t\t" - ULINTPF "\t\t%lu\t\t" ULINTPF "\t\t%.2f\t" ULINTPF "\n", + fprintf(fil_out, "%llu\t\t%llu\t\t" ULINTPF "\t\t" ULINTPF "\t\t" ULINTPF "\t\t%.2f\t" ULINTPF "\n", id, index.leaf_pages, n_leaf_pages, n_merge, n_pages, - 1.0 - (double)n_pages / (double)n_leaf_pages, - index.max_data_size); + 1.0 - (double)n_pages / (double)n_leaf_pages, index.max_data_size); } void print_leaf_stats( @@ -372,29 +325,6 @@ get_page_size( return(page_size_t(flags)); } -#ifdef MYSQL_COMPRESSION -/** Decompress a page -@param[in,out] buf Page read from disk, uncompressed data will - also be copied to this page -@param[in, out] scratch Page to use for temporary decompress -@param[in] page_size scratch physical size -@return true if decompress succeeded */ -static -bool page_decompress( - byte* buf, - byte* scratch, - page_size_t page_size) -{ - dberr_t err; - - /* Set the dblwr recover flag to false. */ - err = os_file_decompress_page( - false, buf, scratch, page_size.physical()); - - return(err == DB_SUCCESS); -} -#endif - #ifdef _WIN32 /***********************************************//* @param [in] error error no. from the getLastError(). @@ -439,6 +369,7 @@ open_file( access = GENERIC_READ; flags = _O_RDONLY | _O_BINARY; } + /* CreateFile() also provide advisory lock with the usage of access and share mode of the file.*/ hFile = CreateFile( @@ -462,8 +393,7 @@ open_file( if (do_write) { create_flag = O_RDWR; lk.l_type = F_WRLCK; - } - else { + } else { create_flag = O_RDONLY; lk.l_type = F_RDLCK; } @@ -525,12 +455,18 @@ ulong read_file( /** Check if page is corrupted or not. @param[in] buf page frame @param[in] page_size page size +@param[in] is_encrypted true if page0 contained cryp_data + with crypt_scheme encrypted +@param[in] is_compressed true if page0 fsp_flags contained + page compression flag @retval true if page is corrupted otherwise false. */ static bool is_page_corrupted( - const byte* buf, - const page_size_t& page_size) + byte* buf, + const page_size_t& page_size, + bool is_encrypted, + bool is_compressed) { /* enable if page is corrupted. */ @@ -538,8 +474,22 @@ is_page_corrupted( /* use to store LSN values. */ ulint logseq; ulint logseqfield; + ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE); + uint key_version = mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + ulint space_id = mach_read_from_4( + buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + + /* We can't trust only a page type, thus we take account + also fsp_flags or crypt_data on page 0 */ + if ((page_type == FIL_PAGE_PAGE_COMPRESSED && is_compressed) || + (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED && + is_compressed && is_encrypted)) { + /* Page compressed tables do not contain post compression + checksum. */ + return (false); + } - if (!page_size.is_compressed()) { + if (page_size.is_compressed()) { /* check the stored log sequence numbers for uncompressed tablespace. */ logseq = mach_read_from_4(buf + FIL_PAGE_LSN + 4); @@ -549,35 +499,32 @@ is_page_corrupted( if (is_log_enabled) { fprintf(log_file, - "page::%" PRIuMAX + "space::" ULINTPF " page::%llu" "; log sequence number:first = " ULINTPF "; second = " ULINTPF "\n", - cur_page_num, logseq, logseqfield); + space_id, cur_page_num, logseq, logseqfield); if (logseq != logseqfield) { fprintf(log_file, - "Fail; page %" PRIuMAX + "Fail; space::" ULINTPF " page::%llu" " invalid (fails log " "sequence number check)\n", - cur_page_num); + space_id, cur_page_num); } } } - /* FIXME: Read the page number from the tablespace header, - and check that every page carries the same page number. */ + /* Again we can't trust only FIL_PAGE_FILE_FLUSH_LSN field + now repurposed as FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, + we need to check also crypt_data contents. - /* If page is encrypted, use different checksum calculation + If page is encrypted, use different checksum calculation as innochecksum can't decrypt pages. Note that some old InnoDB versions did not initialize FIL_PAGE_FILE_FLUSH_LSN field so if crypt checksum does not match we verify checksum using - normal method. - */ - if (mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0) { - is_corrupted = fil_space_verify_crypt_checksum( - const_cast<byte*>(buf), page_size, - mach_read_from_4(buf - + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID), - cur_page_num); + normal method. */ + if (is_encrypted && key_version != 0) { + is_corrupted = !fil_space_verify_crypt_checksum(buf, + page_size, space_id, cur_page_num); } else { is_corrupted = true; } @@ -628,8 +575,9 @@ is_page_empty( if (*page++) { return (false); } - } - return (true); + } + + return (true); } /********************************************************************//** @@ -682,7 +630,7 @@ update_checksum( mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); if (is_log_enabled) { - fprintf(log_file, "page::%" PRIuMAX "; Updated checksum =" + fprintf(log_file, "page::%llu; Updated checksum =" " %u\n", cur_page_num, checksum); } @@ -713,7 +661,7 @@ update_checksum( mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); if (is_log_enabled) { - fprintf(log_file, "page::%" PRIuMAX "; Updated checksum field1" + fprintf(log_file, "page::%llu; Updated checksum field1" " = %u\n", cur_page_num, checksum); } @@ -727,7 +675,7 @@ update_checksum( FIL_PAGE_END_LSN_OLD_CHKSUM,checksum); if (is_log_enabled) { - fprintf(log_file, "page::%" PRIuMAX "; Updated checksum " + fprintf(log_file, "page::%llu; Updated checksum " "field2 = %u\n", cur_page_num, checksum); } @@ -802,7 +750,7 @@ write_file( if (page_size != fwrite(buf, 1, page_size, file == stdin ? stdout : file)) { - fprintf(stderr, "Failed to write page %" PRIuMAX " to %s: %s\n", + fprintf(stderr, "Failed to write page::%llu to %s: %s\n", cur_page_num, filename, strerror(errno)); return(false); @@ -819,93 +767,32 @@ write_file( return(true); } -/********************************************************//** -Gets the next index page number. -@return next page number */ -ulint -btr_page_get_next( -/*==============*/ - const page_t* page) /*!< in: index page */ -{ - return(mach_read_from_4(page + FIL_PAGE_NEXT)); -} - -/********************************************************//** -Gets the previous index page number. -@return prev page number */ -ulint -btr_page_get_prev( -/*==============*/ - const page_t* page) /*!< in: index page */ -{ - return(mach_read_from_4(page + FIL_PAGE_PREV)); -} - -ulint -mach_read_ulint( - const byte* ptr, - mlog_id_t type) -{ - switch (type) { - case MLOG_1BYTE: - return(mach_read_from_1(ptr)); - case MLOG_2BYTES: - return(mach_read_from_2(ptr)); - case MLOG_4BYTES: - return(mach_read_from_4(ptr)); - default: - break; - } - - ut_error; - return(0); -} -ibool -xdes_get_bit( -/*=========*/ - const xdes_t* descr, /*!< in: descriptor */ - ulint bit, /*!< in: XDES_FREE_BIT or XDES_CLEAN_BIT */ - ulint offset) /*!< in: page offset within extent: - 0 ... FSP_EXTENT_SIZE - 1 */ -{ - ulint index = bit + XDES_BITS_PER_PAGE * offset; - - ulint bit_index = index % 8; - ulint byte_index = index / 8; - - return(ut_bit_get_nth( - mach_read_ulint(descr + XDES_BITMAP + byte_index, - MLOG_1BYTE), - bit_index)); -} - /* Parse the page and collect/dump the information about page type @param [in] page buffer page -@param [in] xdes xdes page +@param [out] xdes extend descriptor page @param [in] file file for diagnosis. -@param [in] page_size page size +@param [in] page_size page_size +@param [in] is_encrypted tablespace is encrypted */ void parse_page( const byte* page, - const byte* xdes, + byte* xdes, FILE* file, - page_size_t page_size) + const page_size_t& page_size, + bool is_encrypted) { - unsigned long long id=0; - ulint undo_page_type=0; - ulint n_recs; - ulint page_no=0; - ulint right_page_no=0; - ulint left_page_no=0; - ulint data_bytes=0; - bool is_leaf=false; - ulint size_range_id=0; - ulint data_types=0; - ulint key_version = 0; - + unsigned long long id; + ulint undo_page_type; char str[20]={'\0'}; + ulint n_recs; + ulint page_no; + ulint left_page_no; + ulint right_page_no; + ulint data_bytes; + bool is_leaf; + int size_range_id; /* Check whether page is doublewrite buffer. */ if(skip_page) { @@ -916,102 +803,105 @@ parse_page( switch (mach_read_from_2(page + FIL_PAGE_TYPE)) { - case FIL_PAGE_INDEX: + case FIL_PAGE_INDEX: { + uint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); page_type.n_fil_page_index++; - id = mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID); - n_recs = page_header_get_field(page, PAGE_N_RECS); - page_no = page_get_page_no(page); - left_page_no = btr_page_get_prev(page); - right_page_no = btr_page_get_next(page); - key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); /* If page is encrypted we can't read index header */ - if (!key_version) { - data_bytes = page_get_data_size(page); - } else { - data_bytes = 0; - } + if (!is_encrypted) { + id = mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID); + n_recs = mach_read_from_2(page + PAGE_HEADER + PAGE_N_RECS); + page_no = mach_read_from_4(page + FIL_PAGE_OFFSET); + left_page_no = mach_read_from_4(page + FIL_PAGE_PREV); + right_page_no = mach_read_from_4(page + FIL_PAGE_NEXT); + ulint is_comp = mach_read_from_2(page + PAGE_HEADER + PAGE_N_HEAP) & 0x8000; + ulint level = mach_read_from_2(page + PAGE_HEADER + PAGE_LEVEL); + ulint garbage = mach_read_from_2(page + PAGE_HEADER + PAGE_GARBAGE); - is_leaf = page_is_leaf(page); - if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tIndex page\t\t\t|" - "\tindex id=%llu,", cur_page_num, id); - - fprintf(file, - " page level=" ULINTPF " leaf %u" - ", No. of records=" ULINTPF - ", garbage=" ULINTPF - ", n_recs=" ULINTPF - ", %s\n", - page_header_get_field(page, PAGE_LEVEL), - is_leaf, - n_recs, - page_header_get_field(page, PAGE_GARBAGE), - data_types, - str); - } + data_bytes = (ulint)(mach_read_from_2(page + PAGE_HEADER + PAGE_HEAP_TOP) + - (is_comp + ? PAGE_NEW_SUPREMUM_END + : PAGE_OLD_SUPREMUM_END) + - garbage); - size_range_id = (data_bytes * SIZE_RANGES_FOR_PAGE - + page_size.logical() - 1) / - page_size.logical(); - - if (size_range_id > SIZE_RANGES_FOR_PAGE + 1) { - /* data_bytes is bigger than logical_page_size */ - size_range_id = SIZE_RANGES_FOR_PAGE + 1; - } + is_leaf = (!*(const uint16*) (page + (PAGE_HEADER + PAGE_LEVEL))); - /* update per-index statistics */ - { - if (index_ids.count(id) == 0) { - index_ids[id] = per_index_stats(); - } - - std::map<unsigned long long, per_index_stats>::iterator it; - it = index_ids.find(id); - per_index_stats &index = (it->second); - uchar* des = (uchar *)(xdes + XDES_ARR_OFFSET - + XDES_SIZE * ((page_no & (page_size.physical() - 1)) - / FSP_EXTENT_SIZE)); - - if (xdes_get_bit(des, XDES_FREE_BIT, - page_no % FSP_EXTENT_SIZE)) { - index.free_pages++; - return; + if (page_type_dump) { + fprintf(file, "#::%llu\t\t|\t\tIndex page\t\t\t|" + "\tindex id=%llu,", cur_page_num, id); + + fprintf(file, + " page level=" ULINTPF + ", No. of records=" ULINTPF + ", garbage=" ULINTPF ", %s\n", + level, n_recs, garbage, str); } - index.pages++; + size_range_id = (data_bytes * SIZE_RANGES_FOR_PAGE + + page_size.logical() - 1) / + page_size.logical(); - if (is_leaf) { - index.leaf_pages++; - if (data_bytes > index.max_data_size) { - index.max_data_size = data_bytes; + if (size_range_id > SIZE_RANGES_FOR_PAGE + 1) { + /* data_bytes is bigger than logical_page_size */ + size_range_id = SIZE_RANGES_FOR_PAGE + 1; + } + if (per_page_details) { + printf("index id=%llu page " ULINTPF " leaf %d n_recs " ULINTPF " data_bytes " ULINTPF + "\n", id, page_no, is_leaf, n_recs, data_bytes); + } + /* update per-index statistics */ + { + if (index_ids.count(id) == 0) { + index_ids[id] = per_index_stats(); + } + std::map<unsigned long long, per_index_stats>::iterator it; + it = index_ids.find(id); + per_index_stats &index = (it->second); + const byte* des = xdes + XDES_ARR_OFFSET + + XDES_SIZE * ((page_no & (page_size.physical() - 1)) + / FSP_EXTENT_SIZE); + if (xdes_get_bit(des, XDES_FREE_BIT, + page_no % FSP_EXTENT_SIZE)) { + index.free_pages++; + return; } - struct per_page_stats pp(n_recs, data_bytes, - left_page_no, right_page_no); + index.pages++; - index.leaves[page_no] = pp; + if (is_leaf) { + index.leaf_pages++; + if (data_bytes > index.max_data_size) { + index.max_data_size = data_bytes; + } + struct per_page_stats pp(n_recs, data_bytes, + left_page_no, right_page_no); - if (left_page_no == ULINT32_UNDEFINED) { - index.first_leaf_page = page_no; - index.count++; + index.leaves[page_no] = pp; + + if (left_page_no == ULINT32_UNDEFINED) { + index.first_leaf_page = page_no; + index.count++; + } } - } - index.total_n_recs += n_recs; - index.total_data_bytes += data_bytes; - index.pages_in_size_range[size_range_id] ++; + index.total_n_recs += n_recs; + index.total_data_bytes += data_bytes; + index.pages_in_size_range[size_range_id] ++; + } + } else { + fprintf(file, "#::%llu\t\t|\t\tEncrypted Index page\t\t\t|" + "\tkey_version %u,%s\n", cur_page_num, key_version, str); } break; - + } case FIL_PAGE_UNDO_LOG: page_type.n_fil_page_undo_log++; undo_page_type = mach_read_from_2(page + TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_TYPE); if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tUndo log page\t\t\t|", + fprintf(file, "#::%llu\t\t|\t\tUndo log page\t\t\t|", cur_page_num); } if (undo_page_type == TRX_UNDO_INSERT) { @@ -1085,7 +975,7 @@ parse_page( case FIL_PAGE_INODE: page_type.n_fil_page_inode++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInode page\t\t\t|" + fprintf(file, "#::%llu\t\t|\t\tInode page\t\t\t|" "\t%s\n",cur_page_num, str); } break; @@ -1093,7 +983,7 @@ parse_page( case FIL_PAGE_IBUF_FREE_LIST: page_type.n_fil_page_ibuf_free_list++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInsert buffer free list" + fprintf(file, "#::%llu\t\t|\t\tInsert buffer free list" " page\t|\t%s\n", cur_page_num, str); } break; @@ -1101,7 +991,7 @@ parse_page( case FIL_PAGE_TYPE_ALLOCATED: page_type.n_fil_page_type_allocated++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tFreshly allocated " + fprintf(file, "#::%llu\t\t|\t\tFreshly allocated " "page\t\t|\t%s\n", cur_page_num, str); } break; @@ -1109,7 +999,7 @@ parse_page( case FIL_PAGE_IBUF_BITMAP: page_type.n_fil_page_ibuf_bitmap++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInsert Buffer " + fprintf(file, "#::%llu\t\t|\t\tInsert Buffer " "Bitmap\t\t|\t%s\n", cur_page_num, str); } break; @@ -1117,7 +1007,7 @@ parse_page( case FIL_PAGE_TYPE_SYS: page_type.n_fil_page_type_sys++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tSystem page\t\t\t|" + fprintf(file, "#::%llu\t\t|\t\tSystem page\t\t\t|" "\t%s\n",cur_page_num, str); } break; @@ -1125,25 +1015,25 @@ parse_page( case FIL_PAGE_TYPE_TRX_SYS: page_type.n_fil_page_type_trx_sys++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tTransaction system " + fprintf(file, "#::%llu\t\t|\t\tTransaction system " "page\t\t|\t%s\n", cur_page_num, str); } break; case FIL_PAGE_TYPE_FSP_HDR: page_type.n_fil_page_type_fsp_hdr++; - memcpy((void *)xdes, (void *)page, page_size.physical()); + memcpy(xdes, page, page_size.physical()); if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tFile Space " + fprintf(file, "#::%llu\t\t|\t\tFile Space " "Header\t\t|\t%s\n", cur_page_num, str); } break; case FIL_PAGE_TYPE_XDES: page_type.n_fil_page_type_xdes++; - memcpy((void *)xdes, (void *)page, page_size.physical()); + memcpy(xdes, page, page_size.physical()); if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tExtent descriptor " + fprintf(file, "#::%llu\t\t|\t\tExtent descriptor " "page\t\t|\t%s\n", cur_page_num, str); } break; @@ -1151,7 +1041,7 @@ parse_page( case FIL_PAGE_TYPE_BLOB: page_type.n_fil_page_type_blob++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tBLOB page\t\t\t|\t%s\n", + fprintf(file, "#::%llu\t\t|\t\tBLOB page\t\t\t|\t%s\n", cur_page_num, str); } break; @@ -1159,7 +1049,7 @@ parse_page( case FIL_PAGE_TYPE_ZBLOB: page_type.n_fil_page_type_zblob++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tCompressed BLOB " + fprintf(file, "#::%llu\t\t|\t\tCompressed BLOB " "page\t\t|\t%s\n", cur_page_num, str); } break; @@ -1167,25 +1057,26 @@ parse_page( case FIL_PAGE_TYPE_ZBLOB2: page_type.n_fil_page_type_zblob2++; if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tSubsequent Compressed " + fprintf(file, "#::%llu\t\t|\t\tSubsequent Compressed " "BLOB page\t|\t%s\n", cur_page_num, str); } break; - case FIL_PAGE_PAGE_COMPRESSED: - page_type.n_fil_page_type_page_compressed++; - if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tPage compressed " + case FIL_PAGE_PAGE_COMPRESSED: + page_type.n_fil_page_type_page_compressed++; + if (page_type_dump) { + fprintf(file, "#::%llu\t\t|\t\tPage compressed " "page\t|\t%s\n", cur_page_num, str); - } - break; - case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED: - page_type.n_fil_page_type_page_compressed_encrypted++; - if (page_type_dump) { - fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tPage compressed encrypted " + } + break; + + case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED: + page_type.n_fil_page_type_page_compressed_encrypted++; + if (page_type_dump) { + fprintf(file, "#::%llu\t\t|\t\tPage compressed encrypted " "page\t|\t%s\n", cur_page_num, str); - } - break; + } + break; default: page_type.n_fil_page_type_other++; break; @@ -1273,7 +1164,7 @@ print_summary( page_type.n_fil_page_type_page_compressed); fprintf(fil_out, "%8d\tPage compressed encrypted page\n", page_type.n_fil_page_type_page_compressed_encrypted); - fprintf(fil_out, "%8d\tOther type of page", + fprintf(fil_out, "%8d\tOther type of page\n", page_type.n_fil_page_type_other); fprintf(fil_out, "\n===============================================\n"); @@ -1331,8 +1222,10 @@ static struct my_option innochecksum_options[] = { 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, {"verbose", 'v', "Verbose (prints progress every 5 seconds).", &verbose, &verbose, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, +#ifndef DBUG_OFF {"debug", '#', "Output debug log. See " REFMAN "dbug-package.html", &dbug_setting, &dbug_setting, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, +#endif /* !DBUG_OFF */ {"count", 'c', "Print the count of pages in the file and exits.", &just_count, &just_count, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"start_page", 's', "Start on this page number (0 based).", @@ -1361,10 +1254,12 @@ static struct my_option innochecksum_options[] = { {"page-type-dump", 'D', "Dump the page type info for each page in a " "tablespace.", &page_dump_filename, &page_dump_filename, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"per-page-details", 'i', "Print out per-page detail information.", + &per_page_details, &per_page_details, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"log", 'l', "log output.", &log_filename, &log_filename, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, - {"leaf", 'e', "Examine leaf index pages", + {"leaf", 'f', "Examine leaf index pages", &do_leaf, &do_leaf, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"merge", 'm', "leaf page count if merge given number of consecutive pages", &n_merge, &n_merge, 0, GET_ULONG, REQUIRED_ARG, 0, 0, (longlong)10L, 0, 1, 0}, @@ -1392,9 +1287,9 @@ static void usage(void) puts(ORACLE_WELCOME_COPYRIGHT_NOTICE("2000")); printf("InnoDB offline file checksum utility.\n"); printf("Usage: %s [-c] [-s <start page>] [-e <end page>] " - "[-p <page>] [-v] [-a <allow mismatches>] [-n] " + "[-p <page>] [-i] [-v] [-a <allow mismatches>] [-n] " "[-C <strict-check>] [-w <write>] [-S] [-D <page type dump>] " - "[-l <log>] [-e] <filename or [-]>\n", my_progname); + "[-l <log>] [-l] [-m <merge pages>] <filename or [-]>\n", my_progname); printf("See " REFMAN "innochecksum.html for usage hints.\n"); my_print_help(innochecksum_options); my_print_variables(innochecksum_options); @@ -1487,7 +1382,7 @@ get_options( char ***argv) { if (handle_options(argc, argv, innochecksum_options, - innochecksum_get_one_option)) { + innochecksum_get_one_option)) { my_end(0); exit(true); } @@ -1502,6 +1397,133 @@ get_options( return (false); } +/** Check from page 0 if table is encrypted. +@param[in] filename Filename +@param[in] page_size page size +@param[in] page Page 0 +@retval true if tablespace is encrypted, false if not +*/ +static +bool check_encryption( + const char* filename, + const page_size_t& page_size, + byte * page) +{ + ulint offset = (FSP_HEADER_OFFSET + (XDES_ARR_OFFSET + XDES_SIZE * + (page_size.physical()) / FSP_EXTENT_SIZE)); + + if (memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) { + return false; + } + + ulint type = mach_read_from_1(page + offset + MAGIC_SZ + 0); + + if (! (type == CRYPT_SCHEME_UNENCRYPTED || + type == CRYPT_SCHEME_1)) { + return false; + } + + ulint iv_length = mach_read_from_1(page + offset + MAGIC_SZ + 1); + + if (iv_length != CRYPT_SCHEME_1_IV_LEN) { + return false; + } + + uint min_key_version = mach_read_from_4 + (page + offset + MAGIC_SZ + 2 + iv_length); + + uint key_id = mach_read_from_4 + (page + offset + MAGIC_SZ + 2 + iv_length + 4); + + if (type == CRYPT_SCHEME_1 && is_log_enabled) { + fprintf(log_file,"Tablespace %s encrypted key_version %u key_id %u\n", + filename, min_key_version, key_id); + } + + return (type == CRYPT_SCHEME_1); +} + +/** +Verify page checksum. +@param[in] buf page to verify +@param[in] page_size page size +@param[in] is_encrypted true if tablespace is encrypted +@param[in] is_compressed true if tablespace is page compressed +@param[in,out] mismatch_count Number of pages failed in checksum verify +@retval 0 if page checksum matches or 1 if it does not match +*/ +static +int verify_checksum( + byte* buf, + const page_size_t& page_size, + bool is_encrypted, + bool is_compressed, + unsigned long long* mismatch_count) +{ + int exit_status = 0; + bool is_corrupted = false; + + is_corrupted = is_page_corrupted( + buf, page_size, is_encrypted, is_compressed); + + if (is_corrupted) { + fprintf(stderr, "Fail: page::%llu invalid\n", + cur_page_num); + + (*mismatch_count)++; + + if (*mismatch_count > allow_mismatches) { + fprintf(stderr, + "Exceeded the " + "maximum allowed " + "checksum mismatch " + "count::%llu current::%llu\n", + *mismatch_count, + allow_mismatches); + + exit_status = 1; + } + } + + return (exit_status); +} + +/** Rewrite page checksum if needed. +@param[in] filename File name +@param[in] fil_in File pointer +@param[in] buf page +@param[in] page_size page size +@param[in] pos File position +@param[in] is_encrypted true if tablespace is encrypted +@param[in] is_compressed true if tablespace is page compressed +@retval 0 if checksum rewrite was successful, 1 if error was detected */ +static +int +rewrite_checksum( + const char* filename, + FILE* fil_in, + byte* buf, + const page_size_t& page_size, + fpos_t* pos, + bool is_encrypted, + bool is_compressed) +{ + int exit_status = 0; + /* Rewrite checksum. Note that for encrypted and + page compressed tables this is not currently supported. */ + if (do_write && + !is_encrypted && + !is_compressed + && !write_file(filename, fil_in, buf, + page_size.is_compressed(), pos, + static_cast<ulong>(page_size.physical()))) { + + exit_status = 1; + } + + return (exit_status); +} + int main( int argc, char **argv) @@ -1511,15 +1533,12 @@ int main( /* our input filename. */ char* filename; /* Buffer to store pages read. */ + byte* buf_ptr = NULL; + byte* xdes_ptr = NULL; byte* buf = NULL; - /* Buffer for xdes */ byte* xdes = NULL; /* bytes read count */ ulong bytes; - /* Buffer to decompress page.*/ -#ifdef MYSQL_COMPRESSION - byte* tbuf = NULL; -#endif /* current time */ time_t now; /* last time */ @@ -1531,6 +1550,8 @@ int main( struct stat st; #endif /* _WIN32 */ + int exit_status = 0; + /* size of file (has to be 64 bits) */ unsigned long long int size = 0; /* number of pages in file */ @@ -1538,9 +1559,7 @@ int main( off_t offset = 0; /* count the no. of page corrupted. */ - ulint mismatch_count = 0; - /* Variable to ack the page is corrupted or not. */ - bool is_corrupted = false; + unsigned long long mismatch_count = 0; bool partial_page_read = false; /* Enabled when read from stdin is done. */ @@ -1560,32 +1579,37 @@ int main( DBUG_PROCESS(argv[0]); if (get_options(&argc,&argv)) { - DBUG_RETURN(1); + exit_status = 1; + goto my_exit; } if (strict_verify && no_check) { fprintf(stderr, "Error: --strict-check option cannot be used " "together with --no-check option.\n"); - DBUG_RETURN(1); + exit_status = 1; + goto my_exit; } if (no_check && !do_write) { fprintf(stderr, "Error: --no-check must be associated with " "--write option.\n"); - DBUG_RETURN(1); + exit_status = 1; + goto my_exit; } if (page_type_dump) { fil_page_type = create_file(page_dump_filename); if (!fil_page_type) { - DBUG_RETURN(1); + exit_status = 1; + goto my_exit; } } if (is_log_enabled) { log_file = create_file(log_filename); if (!log_file) { - DBUG_RETURN(1); + exit_status = 1; + goto my_exit; } fprintf(log_file, "InnoDB File Checksum Utility.\n"); } @@ -1595,17 +1619,17 @@ int main( } - buf = (byte*) malloc(UNIV_PAGE_SIZE_MAX * 2); - xdes = (byte *) malloc(UNIV_PAGE_SIZE_MAX *2); -#ifdef MYSQL_COMPRESSION - tbuf = buf + UNIV_PAGE_SIZE_MAX; -#endif + buf_ptr = (byte*) malloc(UNIV_PAGE_SIZE_MAX * 2); + xdes_ptr = (byte*)malloc(UNIV_PAGE_SIZE_MAX * 2); + buf = (byte *) ut_align(buf_ptr, UNIV_PAGE_SIZE_MAX); + xdes = (byte *) ut_align(xdes_ptr, UNIV_PAGE_SIZE_MAX); + /* The file name is not optional. */ for (int i = 0; i < argc; ++i) { + /* Reset parameters for each file. */ filename = argv[i]; memset(&page_type, 0, sizeof(innodb_page_type)); - is_corrupted = false; partial_page_read = false; skip_page = false; @@ -1630,7 +1654,8 @@ int main( fprintf(stderr, "Error: %s cannot be found\n", filename); - goto err_exit; + exit_status = 1; + goto my_exit; } if (!read_from_stdin) { @@ -1638,35 +1663,17 @@ int main( fil_in = open_file(filename); /*If fil_in is NULL, terminate as some error encountered */ if(fil_in == NULL) { - goto err_exit; + exit_status = 1; + goto my_exit; } /* Save the current file pointer in pos variable.*/ if (0 != fgetpos(fil_in, &pos)) { perror("fgetpos"); - goto err_exit; + exit_status = 1; + goto my_exit; } } - /* Testing for lock mechanism. The innochecksum - acquire lock on given file. So other tools accessing the same - file for processsing must fail. */ -#ifdef _WIN32 - DBUG_EXECUTE_IF("innochecksum_cause_mysqld_crash", - ut_ad(page_dump_filename); - while((_access( page_dump_filename, 0)) == 0) { - sleep(1); - } - DBUG_RETURN(0); ); -#else - DBUG_EXECUTE_IF("innochecksum_cause_mysqld_crash", - ut_ad(page_dump_filename); - struct stat status_buf; - while(stat(page_dump_filename, &status_buf) == 0) { - sleep(1); - } - DBUG_RETURN(0); ); -#endif /* _WIN32 */ - /* Read the minimum page size. */ bytes = ulong(fread(buf, 1, UNIV_ZIP_SIZE_MIN, fil_in)); partial_page_read = true; @@ -1674,10 +1681,11 @@ int main( if (bytes != UNIV_ZIP_SIZE_MIN) { fprintf(stderr, "Error: Was not able to read the " "minimum page size "); - fprintf(stderr, "of %d bytes. Bytes read was %lu\n", + fprintf(stderr, "of %d bytes. Bytes read was " ULINTPF "\n", UNIV_ZIP_SIZE_MIN, bytes); - goto err_exit; + exit_status = 1; + goto my_exit; } /* enable variable is_system_tablespace when space_id of given @@ -1687,8 +1695,79 @@ int main( FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, 4)) ? true : false; + /* Determine page size, zip_size and page compression + from fsp_flags and encryption metadata from page 0 */ const page_size_t& page_size = get_page_size(buf); + ulint flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf); + ulint zip_size = page_size.is_compressed() ? page_size.logical() : 0; + logical_page_size = page_size.is_compressed() ? zip_size : 0; + physical_page_size = page_size.physical(); + srv_page_size = page_size.logical(); + bool is_compressed = FSP_FLAGS_HAS_PAGE_COMPRESSION(flags); + + if (page_size.physical() > UNIV_ZIP_SIZE_MIN) { + /* Read rest of the page 0 to determine crypt_data */ + bytes = ulong(read_file(buf, partial_page_read, page_size.physical(), fil_in)); + + if (bytes != page_size.physical()) { + fprintf(stderr, "Error: Was not able to read the " + "rest of the page "); + fprintf(stderr, "of " ULINTPF " bytes. Bytes read was " ULINTPF "\n", + page_size.physical() - UNIV_ZIP_SIZE_MIN, bytes); + + exit_status = 1; + goto my_exit; + } + partial_page_read = false; + } + + /* Now that we have full page 0 in buffer, check encryption */ + bool is_encrypted = check_encryption(filename, page_size, buf); + + /* Verify page 0 contents. Note that we can't allow + checksum mismatch on page 0, because that would mean we + could not trust it content. */ + if (!no_check) { + unsigned long long tmp_allow_mismatches = allow_mismatches; + allow_mismatches = 0; + + exit_status = verify_checksum(buf, page_size, is_encrypted, is_compressed, &mismatch_count); + + if (exit_status) { + fprintf(stderr, "Error: Page 0 checksum mismatch, can't continue. \n"); + goto my_exit; + } + allow_mismatches = tmp_allow_mismatches; + } + + if ((exit_status = rewrite_checksum(filename, fil_in, buf, + page_size, &pos, is_encrypted, is_compressed))) { + goto my_exit; + } + + if (page_type_dump) { + fprintf(fil_page_type, + "\n\nFilename::%s\n", filename); + fprintf(fil_page_type, + "========================================" + "======================================\n"); + fprintf(fil_page_type, + "\tPAGE_NO\t\t|\t\tPAGE_TYPE\t\t" + "\t|\tEXTRA INFO\n"); + fprintf(fil_page_type, + "========================================" + "======================================\n"); + } + + if (per_page_details) { + printf("page %llu ", cur_page_num); + } + + if (page_type_summary || page_type_dump) { + parse_page(buf, xdes, fil_page_type, page_size, is_encrypted); + } + pages = (ulint) (size / page_size.physical()); if (just_count) { @@ -1704,14 +1783,14 @@ int main( "(" ULINTPF " pages)\n", filename, size, pages); if (do_one_page) { fprintf(log_file, "Innochecksum: " - "checking page %" PRIuMAX "\n", + "checking page::%llu;\n", do_page); } } } else { if (is_log_enabled) { fprintf(log_file, "Innochecksum: checking " - "pages in range %" PRIuMAX " to %" PRIuMAX "\n", + "pages in range::%llu to %llu\n", start_page, use_end_page ? end_page : (pages - 1)); } @@ -1735,14 +1814,16 @@ int main( perror("Error: Unable to seek to " "necessary offset"); - goto err_exit; + exit_status = 1; + goto my_exit; } /* Save the current file pointer in pos variable. */ if (0 != fgetpos(fil_in, &pos)) { perror("fgetpos"); - goto err_exit; + exit_status = 1; + goto my_exit; } } else { @@ -1762,7 +1843,7 @@ int main( bytes = read_file(buf, partial_page_read, static_cast<ulong>( - page_size.physical()), + page_size.physical()), fil_in); partial_page_read = false; @@ -1773,34 +1854,22 @@ int main( "to seek to necessary " "offset"); - goto err_exit; + exit_status = 1; + goto my_exit; } } } } - if (page_type_dump) { - fprintf(fil_page_type, - "\n\nFilename::%s\n", filename); - fprintf(fil_page_type, - "========================================" - "======================================\n"); - fprintf(fil_page_type, - "\tPAGE_NO\t\t|\t\tPAGE_TYPE\t\t" - "\t|\tEXTRA INFO\n"); - fprintf(fil_page_type, - "========================================" - "======================================\n"); - } - /* main checksumming loop */ - cur_page_num = start_page; + cur_page_num = start_page ? start_page : cur_page_num + 1; + lastt = 0; while (!feof(fil_in)) { bytes = read_file(buf, partial_page_read, static_cast<ulong>( - page_size.physical()), fil_in); + page_size.physical()), fil_in); partial_page_read = false; if (!bytes && feof(fil_in)) { @@ -1812,14 +1881,16 @@ int main( page_size.physical()); perror(" "); - goto err_exit; + exit_status = 1; + goto my_exit; } if (bytes != page_size.physical()) { - fprintf(stderr, "Error: bytes read (%lu) " + fprintf(stderr, "Error: bytes read (" ULINTPF ") " "doesn't match page size (" ULINTPF ")\n", bytes, page_size.physical()); - goto err_exit; + exit_status = 1; + goto my_exit; } if (is_system_tablespace) { @@ -1827,60 +1898,29 @@ int main( skip_page = is_page_doublewritebuffer(buf); } else { skip_page = false; -#ifdef MYSQL_COMPRESSION - if (!page_decompress(buf, tbuf, page_size)) { - - fprintf(stderr, - "Page decompress failed"); - - goto err_exit; - } -#endif } - ulint page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); + ulint cur_page_type = mach_read_from_2(buf+FIL_PAGE_TYPE); - if (page_type == FIL_PAGE_PAGE_COMPRESSED || - page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + /* FIXME: Page compressed or Page compressed and encrypted + pages do not contain checksum. */ + if (cur_page_type == FIL_PAGE_PAGE_COMPRESSED || + cur_page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { skip_page = true; } /* If no-check is enabled, skip the checksum verification.*/ - if (!no_check) { - /* Checksum verification */ - if (!skip_page) { - is_corrupted = is_page_corrupted( - buf, page_size); - - if (is_corrupted) { - fprintf(stderr, "Fail: page " - "%" PRIuMAX " invalid\n", - cur_page_num); - - mismatch_count++; - - if(mismatch_count > allow_mismatches) { - fprintf(stderr, - "Exceeded the " - "maximum allowed " - "checksum mismatch " - "count::%" PRIuMAX "\n", - allow_mismatches); - - goto err_exit; - } - } - } + if (!no_check + && !skip_page + && (exit_status = verify_checksum(buf, page_size, + is_encrypted, is_compressed, &mismatch_count))) { + goto my_exit; } - /* Rewrite checksum */ - if (do_write - && !write_file(filename, fil_in, buf, - page_size.is_compressed(), &pos, - static_cast<ulong>(page_size.physical()))) { - - goto err_exit; + if ((exit_status = rewrite_checksum(filename, fil_in, buf, + page_size, &pos, is_encrypted, is_compressed))) { + goto my_exit; } /* end if this was the last page we were supposed to check */ @@ -1888,12 +1928,17 @@ int main( break; } + if (per_page_details) { + printf("page %llu ", cur_page_num); + } + if (page_type_summary || page_type_dump) { - parse_page(buf, xdes , fil_page_type, page_size); + parse_page(buf, xdes, fil_page_type, page_size, is_encrypted); } /* do counter increase and progress printing */ cur_page_num++; + if (verbose && !read_from_stdin) { if ((cur_page_num % 64) == 0) { now = time(0); @@ -1902,7 +1947,7 @@ int main( } if (now - lastt >= 1 && is_log_enabled) { - fprintf(log_file, "page %" PRIuMAX " " + fprintf(log_file, "page::%llu " "okay: %.3f%% done\n", (cur_page_num - 1), (float) cur_page_num / pages * 100); @@ -1933,19 +1978,19 @@ int main( fclose(log_file); } - free(buf); - free(xdes); - my_end(0); + free(buf_ptr); + free(xdes_ptr); - DBUG_RETURN(0); + my_end(exit_status); + DBUG_RETURN(exit_status); -err_exit: - if (buf) { - free(buf); +my_exit: + if (buf_ptr) { + free(buf_ptr); } - if (xdes) { - free(xdes); + if (xdes_ptr) { + free(xdes_ptr); } if (!read_from_stdin && fil_in) { @@ -1956,7 +2001,6 @@ err_exit: fclose(log_file); } - my_end(1); - - DBUG_RETURN(1); + my_end(exit_status); + DBUG_RETURN(exit_status); } diff --git a/mysql-test/include/wait_innodb_all_purged.inc b/mysql-test/include/wait_innodb_all_purged.inc deleted file mode 100644 index a77e6ec1573..00000000000 --- a/mysql-test/include/wait_innodb_all_purged.inc +++ /dev/null @@ -1,60 +0,0 @@ -# include/wait_innodb_all_purged.inc -# -# SUMMARY -# -# Waits until purged all undo records of innodb, or operation times out. -# -# USAGE -# -# --source include/wait_innodb_all_purged.inc -# ---source include/have_innodb.inc - -if (`select version() like '%debug%'`) { ---disable_query_log - -let $wait_counter_init= 300; -if ($wait_timeout) -{ - let $wait_counter_init= `SELECT $wait_timeout * 10`; -} -# Reset $wait_timeout so that its value won't be used on subsequent -# calls, and default will be used instead. -let $wait_timeout= 0; - -let $wait_counter= $wait_counter_init; - -# Keep track of how many times the wait condition is tested -let $wait_condition_reps= 0; -let $prev_trx_age= 0; -while ($wait_counter) -{ - let $trx_age = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS -WHERE VARIABLE_NAME = 'INNODB_PURGE_TRX_ID_AGE';`; - - if ($trx_age != $prev_trx_age) - { - let $wait_counter= $wait_counter_init; - let $prev_trx_age= $trx_age; - } - - let $success= `SELECT $trx_age < 1`; - inc $wait_condition_reps; - if ($success) - { - let $wait_counter= 0; - } - if (!$success) - { - set global innodb_purge_run_now=ON; - real_sleep 0.1; - dec $wait_counter; - } -} -if (!$success) -{ - echo Timeout in wait_innodb_all_purged.inc for INNODB_PURGE_TRX_ID_AGE = $trx_age; -} - ---enable_query_log -} diff --git a/mysql-test/suite/binlog/r/binlog_innodb.result b/mysql-test/suite/binlog/r/binlog_innodb.result index 2896706d407..233dda00075 100644 --- a/mysql-test/suite/binlog/r/binlog_innodb.result +++ b/mysql-test/suite/binlog/r/binlog_innodb.result @@ -176,4 +176,14 @@ ERROR 23000: Duplicate entry '4' for key 'PRIMARY' # There must be no UPDATE query event; include/show_binlog_events.inc drop table t1, t2; +*** MDEV-11937: InnoDB flushes redo log too often *** +CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB; +SET @old_flush = @@GLOBAL.innodb_flush_log_at_trx_commit; +SET GLOBAL innodb_flush_log_at_trx_commit=1; +SELECT IF(@num_sync < 100*1.5, "OK", +CONCAT("ERROR: More than 1 fsync per commit (saw ", @num_sync/100, ")")) AS status; +status +OK +DROP TABLE t1; +SET GLOBAL innodb_flush_log_at_trx_commit=@old_flush; End of tests diff --git a/mysql-test/suite/binlog/t/binlog_innodb.test b/mysql-test/suite/binlog/t/binlog_innodb.test index 8191b72d5a9..153dcdd155a 100644 --- a/mysql-test/suite/binlog/t/binlog_innodb.test +++ b/mysql-test/suite/binlog/t/binlog_innodb.test @@ -172,4 +172,33 @@ source include/show_binlog_events.inc; # cleanup bug#27716 drop table t1, t2; +--echo *** MDEV-11937: InnoDB flushes redo log too often *** + +# Count number of log fsyncs reported by InnoDB per commit. +CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB; + +SET @old_flush = @@GLOBAL.innodb_flush_log_at_trx_commit; +SET GLOBAL innodb_flush_log_at_trx_commit=1; + +--let $syncs1 = query_get_value(SHOW STATUS LIKE 'Innodb_os_log_fsyncs', Value, 1) +--let $ROWS = 100 +--disable_query_log +let $count = $ROWS; +while ($count) { + eval INSERT INTO t1 VALUES ($count); + dec $count; +} +--let $syncs2 = query_get_value(SHOW STATUS LIKE 'Innodb_os_log_fsyncs', Value, 1) +eval SET @num_sync = $syncs2 - $syncs1; +--enable_query_log + +# Allow a bit of slack, in case some background process or something +# is introducing a few more syncs. +eval SELECT IF(@num_sync < $ROWS*1.5, "OK", + CONCAT("ERROR: More than 1 fsync per commit (saw ", @num_sync/$ROWS, ")")) AS status; + +DROP TABLE t1; +SET GLOBAL innodb_flush_log_at_trx_commit=@old_flush; + + --echo End of tests diff --git a/mysql-test/suite/encryption/r/innochecksum.result b/mysql-test/suite/encryption/r/innochecksum.result index 7cd7af7b93b..6ea54f3d053 100644 --- a/mysql-test/suite/encryption/r/innochecksum.result +++ b/mysql-test/suite/encryption/r/innochecksum.result @@ -1,3 +1,5 @@ +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; set global innodb_compression_algorithm = 1; # Create and populate a tables CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; @@ -5,6 +7,7 @@ CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FOR CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO; CREATE TABLE t4 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1; CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4; +CREATE TABLE t6 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB; # Write file to make mysql-test-run.pl expect the "crash", but don't # start it until it's told to # We give 30 seconds to do a clean shutdown because we do not want @@ -16,6 +19,27 @@ CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_CO # Run innochecksum on t3 # Run innochecksum on t4 # Run innochecksum on t4 +# Run innochecksum on t5 +# Run innochecksum on t6 +# Backup tables before corrupting +# Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION +# Run innochecksum on t2 +# Run innochecksum on t3 +# no encryption corrupting the field should not have effect +# Run innochecksum on t6 +# no encryption corrupting the field should not have effect +# Restore the original tables +# Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4 (post encryption checksum) +# Run innochecksum on t2 +# Run innochecksum on t3 +# Run innochecksum on t6 +# no encryption corrupting the field should not have effect +# Restore the original tables +# Corrupt FIL_DATA+10 (data) +# Run innochecksum on t2 +# Run innochecksum on t3 +# Run innochecksum on t6 +# Restore the original tables # Write file to make mysql-test-run.pl start up the server again # Cleanup -DROP TABLE t1, t2, t3, t4, t5; +DROP TABLE t1, t2, t3, t4, t5, t6; diff --git a/mysql-test/suite/encryption/t/innochecksum.test b/mysql-test/suite/encryption/t/innochecksum.test index cb1b97ebfb3..6e3962c462f 100644 --- a/mysql-test/suite/encryption/t/innochecksum.test +++ b/mysql-test/suite/encryption/t/innochecksum.test @@ -2,21 +2,25 @@ # MDEV-8773: InnoDB innochecksum does not work with encrypted or page compressed tables # -# Don't test under embedded +--source include/innodb_page_size_small.inc +# Don't test under embedded as we restart server -- source include/not_embedded.inc # Require InnoDB -- source include/have_innodb.inc -- source include/have_file_key_management_plugin.inc +-- source include/innodb_page_size_small.inc if (!$INNOCHECKSUM) { --echo Need innochecksum binary --die Need innochecksum binary } -let $innodb_compression_algorithm_orig=`SELECT @@innodb_compression_algorithm`; - +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; # zlib set global innodb_compression_algorithm = 1; +--enable_warnings --echo # Create and populate a tables CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4; @@ -24,9 +28,11 @@ CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FOR CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO; CREATE TABLE t4 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1; CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4; +CREATE TABLE t6 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB; --disable_query_log --let $i = 1000 +begin; while ($i) { INSERT INTO t1 (b) VALUES (REPEAT('abcdefghijklmnopqrstuvwxyz', 100)); @@ -36,6 +42,8 @@ INSERT INTO t2 SELECT * FROM t1; INSERT INTO t3 SELECT * FROM t1; INSERT INTO t4 SELECT * FROM t1; INSERT INTO t5 SELECT * FROM t1; +INSERT INTO t6 SELECT * FROM t1; +commit; --enable_query_log let $MYSQLD_DATADIR=`select @@datadir`; @@ -44,6 +52,10 @@ let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd; let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd; let t4_IBD = $MYSQLD_DATADIR/test/t4.ibd; let t5_IBD = $MYSQLD_DATADIR/test/t5.ibd; +let t6_IBD = $MYSQLD_DATADIR/test/t6.ibd; + +let INNODB_PAGE_SIZE=`select @@innodb_page_size`; +let MYSQLD_DATADIR=`select @@datadir`; --echo # Write file to make mysql-test-run.pl expect the "crash", but don't --echo # start it until it's told to @@ -77,17 +89,198 @@ shutdown_server 30; --exec $INNOCHECKSUM $t4_IBD +--echo # Run innochecksum on t5 + +--exec $INNOCHECKSUM $t5_IBD + +--echo # Run innochecksum on t6 + +--exec $INNOCHECKSUM $t6_IBD + +--enable_result_log + +--echo # Backup tables before corrupting +--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_DATADIR/test/t1.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t2.ibd $MYSQLD_DATADIR/test/t2.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t3.ibd $MYSQLD_DATADIR/test/t3.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t4.ibd $MYSQLD_DATADIR/test/t4.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t5.ibd $MYSQLD_DATADIR/test/t5.ibd.backup +--copy_file $MYSQLD_DATADIR/test/t6.ibd $MYSQLD_DATADIR/test/t6.ibd.backup + +# +# MDEV-11939: innochecksum mistakes a file for an encrypted one +# + +--echo # Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + +perl; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +EOF + +-- disable_result_log +--error 1 +--exec $INNOCHECKSUM $t1_IBD + +--echo # Run innochecksum on t2 + +--error 1 +--exec $INNOCHECKSUM $t2_IBD + +--echo # Run innochecksum on t3 +--echo # no encryption corrupting the field should not have effect +--exec $INNOCHECKSUM $t3_IBD + +--echo # Run innochecksum on t6 +--echo # no encryption corrupting the field should not have effect +--exec $INNOCHECKSUM $t6_IBD + --enable_result_log +--echo # Restore the original tables +--remove_file $MYSQLD_DATADIR/test/t1.ibd +--remove_file $MYSQLD_DATADIR/test/t2.ibd +--remove_file $MYSQLD_DATADIR/test/t3.ibd +--remove_file $MYSQLD_DATADIR/test/t4.ibd +--remove_file $MYSQLD_DATADIR/test/t5.ibd +--remove_file $MYSQLD_DATADIR/test/t6.ibd +--copy_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd +--copy_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd +--copy_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd +--copy_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd +--copy_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd +--copy_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd + +--echo # Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4 (post encryption checksum) + +perl; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +EOF + +-- disable_result_log +--error 1 +--exec $INNOCHECKSUM $t1_IBD + +--echo # Run innochecksum on t2 +--error 1 +--exec $INNOCHECKSUM $t2_IBD + +--echo # Run innochecksum on t3 +--error 1 +--exec $INNOCHECKSUM $t3_IBD + +--echo # Run innochecksum on t6 +--echo # no encryption corrupting the field should not have effect +--exec $INNOCHECKSUM $t6_IBD + +--enable_result_log + +--echo # Restore the original tables +--remove_file $MYSQLD_DATADIR/test/t1.ibd +--remove_file $MYSQLD_DATADIR/test/t2.ibd +--remove_file $MYSQLD_DATADIR/test/t3.ibd +--remove_file $MYSQLD_DATADIR/test/t4.ibd +--remove_file $MYSQLD_DATADIR/test/t5.ibd +--remove_file $MYSQLD_DATADIR/test/t6.ibd +--copy_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd +--copy_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd +--copy_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd +--copy_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd +--copy_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd +--copy_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd + +--echo # Corrupt FIL_DATA+10 (data) + +perl; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open"; +binmode FILE; +seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek"; +print FILE pack("H*", "c00lcafedeadb017"); +close FILE or die "close"; +EOF + +-- disable_result_log +--error 1 +--exec $INNOCHECKSUM $t1_IBD + +--echo # Run innochecksum on t2 +--error 1 +--exec $INNOCHECKSUM $t2_IBD + +--echo # Run innochecksum on t3 +--error 1 +--exec $INNOCHECKSUM $t3_IBD + +--echo # Run innochecksum on t6 +--error 1 +--exec $INNOCHECKSUM $t6_IBD + +--enable_result_log + +--echo # Restore the original tables +--move_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd +--move_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd +--move_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd +--move_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd +--move_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd +--move_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd + --echo # Write file to make mysql-test-run.pl start up the server again --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect --enable_reconnect --source include/wait_until_connected_again.inc --echo # Cleanup -DROP TABLE t1, t2, t3, t4, t5; - -# reset system ---disable_query_log -EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig; ---enable_query_log +DROP TABLE t1, t2, t3, t4, t5, t6; diff --git a/mysql-test/suite/gcol/r/innodb_virtual_debug_purge.result b/mysql-test/suite/gcol/r/innodb_virtual_debug_purge.result index e2d9be445da..7d1d1d30198 100644 --- a/mysql-test/suite/gcol/r/innodb_virtual_debug_purge.result +++ b/mysql-test/suite/gcol/r/innodb_virtual_debug_purge.result @@ -1,5 +1,7 @@ set default_storage_engine=innodb; set @old_dbug=@@global.debug_dbug; +SET @saved_frequency = @@GLOBAL.innodb_purge_rseg_truncate_frequency; +SET GLOBAL innodb_purge_rseg_truncate_frequency = 1; CREATE TABLE `t` ( `a` BLOB, `b` BLOB, @@ -12,9 +14,7 @@ INSERT INTO t VALUES (REPEAT('a', 16000), REPEAT('b', 16000), DEFAULT, "mm", 2); CREATE INDEX idx ON t(c(100)); SET global debug_dbug="+d,ib_purge_virtual_index_callback"; UPDATE t SET a = REPEAT('m', 16000) WHERE a like "aaa%"; -select sleep(3); -sleep(3) -0 +InnoDB 0 transactions not purged SET global debug_dbug=@old_dbug; DROP TABLE t; CREATE TABLE t ( @@ -29,9 +29,7 @@ INSERT INTO t VALUES (REPEAT('a', 100), REPEAT('b', 100), DEFAULT, "mm", 2); CREATE INDEX idx ON t(c(100)); SET global debug_dbug="+d,ib_purge_virtual_index_callback"; UPDATE t SET a = REPEAT('m', 100) WHERE a like "aaa%"; -select sleep(3); -sleep(3) -0 +InnoDB 0 transactions not purged SET global debug_dbug=@old_dbug; DROP TABLE t; CREATE TABLE t1 ( @@ -52,9 +50,7 @@ insert into t1 values(4, 18, default); CREATE INDEX idx ON t1(x); SET global debug_dbug="+d,ib_purge_virtual_index_callback"; UPDATE t1 SET id = 10 WHERE id = 1; -select sleep(3); -sleep(3) -0 +InnoDB 0 transactions not purged SET global debug_dbug=@old_dbug; DROP TABLE t1; connect con1,localhost,root,,; @@ -80,7 +76,7 @@ SET DEBUG_SYNC= 'now WAIT_FOR uncommitted'; # enable purge COMMIT; # wait for purge to process the deleted records. -Timeout in wait_innodb_all_purged.inc for INNODB_PURGE_TRX_ID_AGE = 4 +InnoDB 0 transactions not purged SET DEBUG_SYNC= 'now SIGNAL purged'; connection default; /* connection default */ ALTER TABLE t1 ADD COLUMN c INT GENERATED ALWAYS AS(a+b), ADD INDEX idx (c), ALGORITHM=INPLACE, LOCK=SHARED; @@ -119,6 +115,7 @@ INSERT INTO t1(a, b) VALUES (8, 8); # enable purge COMMIT; # wait for purge to process the deleted/updated records. +InnoDB 1 transactions not purged SET DEBUG_SYNC= 'now SIGNAL purged'; disconnect con1; connection default; @@ -141,20 +138,26 @@ DROP TABLE t0, t1; create table t (a blob, b blob, c blob as (concat(a,b)), h varchar(10), index (c(100))); insert t(a,b,h) values (repeat('g', 16000), repeat('x', 16000), "kk"); insert t(a,b,h) values (repeat('a', 16000), repeat('b', 16000), "mm"); -set global innodb_purge_stop_now = 1; set global debug_dbug="+d,ib_purge_virtual_index_callback"; +connect prevent_purge, localhost, root; +start transaction with consistent snapshot; +connection default; update t set a = repeat('m', 16000) where a like "aaa%"; connect con1, localhost, root; lock table t write; +disconnect prevent_purge; connection default; -set global innodb_purge_run_now=1; select variable_value>1 from information_schema.global_status where variable_name='innodb_purge_trx_id_age'; variable_value>1 1 disconnect con1; +start transaction with consistent snapshot; +commit; +InnoDB 0 transactions not purged select variable_value>1 from information_schema.global_status where variable_name='innodb_purge_trx_id_age'; variable_value>1 0 set global debug_dbug=@old_dbug; drop table t; set debug_sync=reset; +SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency; diff --git a/mysql-test/suite/gcol/r/innodb_virtual_purge.result b/mysql-test/suite/gcol/r/innodb_virtual_purge.result index 658f49b4b31..308b01ded25 100644 --- a/mysql-test/suite/gcol/r/innodb_virtual_purge.result +++ b/mysql-test/suite/gcol/r/innodb_virtual_purge.result @@ -1,3 +1,5 @@ +SET @saved_frequency = @@GLOBAL.innodb_purge_rseg_truncate_frequency; +SET GLOBAL innodb_purge_rseg_truncate_frequency = 1; # # Bug#21869656 UNDO LOG DOES NOT CONTAIN ENOUGH INFORMATION # ON INDEXED VIRTUAL COLUMNS @@ -21,6 +23,7 @@ connection con1; COMMIT; UPDATE t1 SET a=1; connection default; +InnoDB 0 transactions not purged CHECK TABLE t1; Table Op Msg_type Msg_text test.t1 check status OK @@ -119,6 +122,7 @@ connection con1; COMMIT; disconnect con1; connection default; +InnoDB 0 transactions not purged CHECK TABLE t1; Table Op Msg_type Msg_text test.t1 check status OK @@ -138,3 +142,4 @@ CREATE TABLE t1 (a VARCHAR(30), b INT, a2 VARCHAR(30) GENERATED ALWAYS AS (a) VI CREATE INDEX idx ON t1(a2(10), b, a2(20)); ERROR 42S21: Duplicate column name 'a2' DROP TABLE t1; +SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency; diff --git a/mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test b/mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test index 1e6cd44d0aa..2668e26c976 100644 --- a/mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test +++ b/mysql-test/suite/gcol/t/innodb_virtual_debug_purge.test @@ -5,6 +5,9 @@ set default_storage_engine=innodb; set @old_dbug=@@global.debug_dbug; +# Ensure that the history list length will actually be decremented by purge. +SET @saved_frequency = @@GLOBAL.innodb_purge_rseg_truncate_frequency; +SET GLOBAL innodb_purge_rseg_truncate_frequency = 1; CREATE TABLE `t` ( `a` BLOB, @@ -21,11 +24,10 @@ CREATE INDEX idx ON t(c(100)); SET global debug_dbug="+d,ib_purge_virtual_index_callback"; UPDATE t SET a = REPEAT('m', 16000) WHERE a like "aaa%"; -select sleep(3); +--source ../../innodb/include/wait_all_purged.inc SET global debug_dbug=@old_dbug; DROP TABLE t; - CREATE TABLE t ( a TINYBLOB, b TINYBLOB, @@ -41,7 +43,7 @@ CREATE INDEX idx ON t(c(100)); SET global debug_dbug="+d,ib_purge_virtual_index_callback"; UPDATE t SET a = REPEAT('m', 100) WHERE a like "aaa%"; -select sleep(3); +--source ../../innodb/include/wait_all_purged.inc SET global debug_dbug=@old_dbug; DROP TABLE t; @@ -68,7 +70,7 @@ CREATE INDEX idx ON t1(x); SET global debug_dbug="+d,ib_purge_virtual_index_callback"; UPDATE t1 SET id = 10 WHERE id = 1; -select sleep(3); +--source ../../innodb/include/wait_all_purged.inc SET global debug_dbug=@old_dbug; DROP TABLE t1; @@ -109,7 +111,7 @@ SET DEBUG_SYNC= 'now WAIT_FOR uncommitted'; COMMIT; --echo # wait for purge to process the deleted records. ---source include/wait_innodb_all_purged.inc +--source ../../innodb/include/wait_all_purged.inc SET DEBUG_SYNC= 'now SIGNAL purged'; @@ -154,7 +156,7 @@ INSERT INTO t1(a, b) VALUES (8, 8); COMMIT; --echo # wait for purge to process the deleted/updated records. ---source include/wait_innodb_all_purged.inc +--source ../../innodb/include/wait_all_purged.inc SET DEBUG_SYNC= 'now SIGNAL purged'; @@ -175,20 +177,24 @@ DROP TABLE t0, t1; create table t (a blob, b blob, c blob as (concat(a,b)), h varchar(10), index (c(100))); insert t(a,b,h) values (repeat('g', 16000), repeat('x', 16000), "kk"); insert t(a,b,h) values (repeat('a', 16000), repeat('b', 16000), "mm"); -set global innodb_purge_stop_now = 1; set global debug_dbug="+d,ib_purge_virtual_index_callback"; +connect(prevent_purge, localhost, root); +start transaction with consistent snapshot; +connection default; update t set a = repeat('m', 16000) where a like "aaa%"; connect(con1, localhost, root); lock table t write; +disconnect prevent_purge; connection default; -set global innodb_purge_run_now=1; -sleep 3; select variable_value>1 from information_schema.global_status where variable_name='innodb_purge_trx_id_age'; disconnect con1; -sleep 3; +start transaction with consistent snapshot; +commit; +--source ../../innodb/include/wait_all_purged.inc select variable_value>1 from information_schema.global_status where variable_name='innodb_purge_trx_id_age'; set global debug_dbug=@old_dbug; drop table t; --source include/wait_until_count_sessions.inc set debug_sync=reset; +SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency; diff --git a/mysql-test/suite/gcol/t/innodb_virtual_purge.test b/mysql-test/suite/gcol/t/innodb_virtual_purge.test index ab6ba52c2c3..4eb5d8c65b8 100644 --- a/mysql-test/suite/gcol/t/innodb_virtual_purge.test +++ b/mysql-test/suite/gcol/t/innodb_virtual_purge.test @@ -1,6 +1,10 @@ --source include/have_innodb.inc --source include/count_sessions.inc +# Ensure that the history list length will actually be decremented by purge. +SET @saved_frequency = @@GLOBAL.innodb_purge_rseg_truncate_frequency; +SET GLOBAL innodb_purge_rseg_truncate_frequency = 1; + --echo # --echo # Bug#21869656 UNDO LOG DOES NOT CONTAIN ENOUGH INFORMATION --echo # ON INDEXED VIRTUAL COLUMNS @@ -32,8 +36,7 @@ COMMIT; UPDATE t1 SET a=1; connection default; -# wait for purge to process the update_undo record (in debug builds) ---source include/wait_innodb_all_purged.inc +--source ../../innodb/include/wait_all_purged.inc CHECK TABLE t1; SELECT b1 FROM t1; @@ -118,8 +121,7 @@ COMMIT; disconnect con1; connection default; -# wait for purge to process the update_undo record (in debug builds) ---source include/wait_innodb_all_purged.inc +--source ../../innodb/include/wait_all_purged.inc CHECK TABLE t1; SELECT b1 FROM t1; @@ -136,3 +138,4 @@ CREATE INDEX idx ON t1(a2(10), b, a2(20)); DROP TABLE t1; --source include/wait_until_count_sessions.inc +SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency; diff --git a/mysql-test/suite/innodb/include/innodb_merge_threshold_delete.inc b/mysql-test/suite/innodb/include/innodb_merge_threshold_delete.inc index 6aad7afa878..8c60cd6e230 100644 --- a/mysql-test/suite/innodb/include/innodb_merge_threshold_delete.inc +++ b/mysql-test/suite/innodb/include/innodb_merge_threshold_delete.inc @@ -6,7 +6,6 @@ # --source include/have_innodb.inc ---source include/have_debug.inc --source include/have_innodb_16k.inc # turn on flags @@ -47,17 +46,17 @@ INFORMATION_SCHEMA.INNODB_BUFFER_PAGE s2 where s1.SPACE = s2.SPACE AND NAME like 'test/tab1%' and PAGE_TYPE = "INDEX" order by PAGE_NUMBER, NUMBER_RECORDS; -set global innodb_purge_stop_now=ON; +begin; delete from tab1 where a = 12; delete from tab1 where a = 13; delete from tab1 where a = 14; delete from tab1 where a = 5; delete from tab1 where a = 6; delete from tab1 where a = 7; -set global innodb_purge_run_now=ON; +commit; # wait for purge view progress (records are deleted actually by purge) ---source include/wait_innodb_all_purged.inc +--source include/wait_all_purged.inc # not merged yet # | 1,2,3,4 | 8,9,10,11 | @@ -72,34 +71,27 @@ INFORMATION_SCHEMA.INNODB_BUFFER_PAGE s2 where s1.SPACE = s2.SPACE AND NAME like 'test/tab1%' and PAGE_TYPE = "INDEX" order by PAGE_NUMBER, NUMBER_RECORDS; - -set global innodb_purge_stop_now=ON; delete from tab1 where a = 11; -set global innodb_purge_run_now=ON; # wait for purge view progress (records are deleted actually by purge) ---source include/wait_innodb_all_purged.inc +--source include/wait_all_purged.inc --echo # check page merge happens (MERGE_THRESHOLD=50 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; -set global innodb_purge_stop_now=ON; delete from tab1 where a = 10; -set global innodb_purge_run_now=ON; # wait for purge view progress (records are deleted actually by purge) ---source include/wait_innodb_all_purged.inc +--source include/wait_all_purged.inc --echo # check page merge happens (MERGE_THRESHOLD=35 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; -set global innodb_purge_stop_now=ON; delete from tab1 where a = 9; -set global innodb_purge_run_now=ON; # wait for purge view progress (records are deleted actually by purge) ---source include/wait_innodb_all_purged.inc +--source include/wait_all_purged.inc --echo # check page merge happens (MERGE_THRESHOLD=25 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics diff --git a/mysql-test/suite/innodb/include/innodb_merge_threshold_secondary.inc b/mysql-test/suite/innodb/include/innodb_merge_threshold_secondary.inc index 72242ea2e1b..8e821365e3f 100644 --- a/mysql-test/suite/innodb/include/innodb_merge_threshold_secondary.inc +++ b/mysql-test/suite/innodb/include/innodb_merge_threshold_secondary.inc @@ -7,7 +7,6 @@ # --source include/have_innodb.inc ---source include/have_debug.inc --source include/have_innodb_16k.inc # turn on flags @@ -79,7 +78,7 @@ INFORMATION_SCHEMA.INNODB_BUFFER_PAGE s2 where s1.SPACE = s2.SPACE AND NAME like 'test/tab1%' and PAGE_TYPE = "INDEX" order by PAGE_NUMBER, NUMBER_RECORDS; -set global innodb_purge_stop_now=ON; +begin; delete from tab1 where a = 33; delete from tab1 where a = 34; delete from tab1 where a = 35; @@ -100,10 +99,10 @@ delete from tab1 where a = 18; delete from tab1 where a = 19; delete from tab1 where a = 20; delete from tab1 where a = 21; -set global innodb_purge_run_now=ON; +commit; # wait for purge view progress (records are deleted actually by purge) ---source include/wait_innodb_all_purged.inc +--source include/wait_all_purged.inc # secondary index is not merged yet # | 1,..,11 | 22,..,32 | @@ -119,33 +118,27 @@ where s1.SPACE = s2.SPACE AND NAME like 'test/tab1%' and PAGE_TYPE = "INDEX" order by PAGE_NUMBER, NUMBER_RECORDS; -set global innodb_purge_stop_now=ON; delete from tab1 where a = 32; -set global innodb_purge_run_now=ON; # wait for purge view progress (records are deleted actually by purge) ---source include/wait_innodb_all_purged.inc +--source include/wait_all_purged.inc --echo # check page merge happens (MERGE_THRESHOLD=50 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; -set global innodb_purge_stop_now=ON; delete from tab1 where a = 31; -set global innodb_purge_run_now=ON; # wait for purge view progress (records are deleted actually by purge) ---source include/wait_innodb_all_purged.inc +--source include/wait_all_purged.inc --echo # check page merge happens (MERGE_THRESHOLD=45 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; -set global innodb_purge_stop_now=ON; delete from tab1 where a = 30; -set global innodb_purge_run_now=ON; # wait for purge view progress (records are deleted actually by purge) ---source include/wait_innodb_all_purged.inc +--source include/wait_all_purged.inc --echo # check page merge happens (MERGE_THRESHOLD=40 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics diff --git a/mysql-test/suite/innodb/include/innodb_merge_threshold_update.inc b/mysql-test/suite/innodb/include/innodb_merge_threshold_update.inc index ff34cafd24a..61e961ac6ae 100644 --- a/mysql-test/suite/innodb/include/innodb_merge_threshold_update.inc +++ b/mysql-test/suite/innodb/include/innodb_merge_threshold_update.inc @@ -6,7 +6,6 @@ # --source include/have_innodb.inc ---source include/have_debug.inc --source include/have_innodb_16k.inc # turn on flags diff --git a/mysql-test/suite/innodb/r/index_merge_threshold.result b/mysql-test/suite/innodb/r/index_merge_threshold.result index 092bb69aa65..35cb82e6f1e 100644 --- a/mysql-test/suite/innodb/r/index_merge_threshold.result +++ b/mysql-test/suite/innodb/r/index_merge_threshold.result @@ -1,3 +1,5 @@ +SET @saved_frequency = @@GLOBAL.innodb_purge_rseg_truncate_frequency; +SET GLOBAL innodb_purge_rseg_truncate_frequency = 1; CREATE TABLE tab(a BIGINT PRIMARY KEY,c1 TINYTEXT,c2 TEXT,c3 MEDIUMTEXT, c4 TINYBLOB,c5 BLOB,c6 MEDIUMBLOB,c7 LONGBLOB) ENGINE=InnoDB; CREATE INDEX index1 ON tab(c1(255)) COMMENT 'Check index level merge MERGE_THRESHOLD=51'; @@ -205,14 +207,15 @@ PAGE_NUMBER NUMBER_RECORDS 3 2 4 7 5 7 -set global innodb_purge_stop_now=ON; +begin; delete from tab1 where a = 12; delete from tab1 where a = 13; delete from tab1 where a = 14; delete from tab1 where a = 5; delete from tab1 where a = 6; delete from tab1 where a = 7; -set global innodb_purge_run_now=ON; +commit; +InnoDB 0 transactions not purged # check page merge happens (nothing is expected) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; @@ -228,27 +231,24 @@ PAGE_NUMBER NUMBER_RECORDS 3 2 4 4 5 4 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 11; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=50 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; name count_reset index_page_merge_attempts 1 index_page_merge_successful 1 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 10; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=35 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; name count_reset index_page_merge_attempts 2 index_page_merge_successful 2 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 9; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=25 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; @@ -288,14 +288,15 @@ PAGE_NUMBER NUMBER_RECORDS 3 2 4 7 5 7 -set global innodb_purge_stop_now=ON; +begin; delete from tab1 where a = 12; delete from tab1 where a = 13; delete from tab1 where a = 14; delete from tab1 where a = 5; delete from tab1 where a = 6; delete from tab1 where a = 7; -set global innodb_purge_run_now=ON; +commit; +InnoDB 0 transactions not purged # check page merge happens (nothing is expected) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; @@ -311,27 +312,24 @@ PAGE_NUMBER NUMBER_RECORDS 3 2 4 4 5 4 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 11; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=50 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; name count_reset index_page_merge_attempts 0 index_page_merge_successful 0 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 10; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=35 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; name count_reset index_page_merge_attempts 1 index_page_merge_successful 1 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 9; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=25 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; @@ -371,14 +369,15 @@ PAGE_NUMBER NUMBER_RECORDS 3 2 4 7 5 7 -set global innodb_purge_stop_now=ON; +begin; delete from tab1 where a = 12; delete from tab1 where a = 13; delete from tab1 where a = 14; delete from tab1 where a = 5; delete from tab1 where a = 6; delete from tab1 where a = 7; -set global innodb_purge_run_now=ON; +commit; +InnoDB 0 transactions not purged # check page merge happens (nothing is expected) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; @@ -394,27 +393,24 @@ PAGE_NUMBER NUMBER_RECORDS 3 2 4 4 5 4 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 11; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=50 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; name count_reset index_page_merge_attempts 0 index_page_merge_successful 0 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 10; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=35 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; name count_reset index_page_merge_attempts 0 index_page_merge_successful 0 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 9; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=25 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; @@ -459,14 +455,15 @@ PAGE_NUMBER NUMBER_RECORDS 3 2 4 7 5 7 -set global innodb_purge_stop_now=ON; +begin; delete from tab1 where a = 12; delete from tab1 where a = 13; delete from tab1 where a = 14; delete from tab1 where a = 5; delete from tab1 where a = 6; delete from tab1 where a = 7; -set global innodb_purge_run_now=ON; +commit; +InnoDB 0 transactions not purged # check page merge happens (nothing is expected) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; @@ -483,27 +480,24 @@ PAGE_NUMBER NUMBER_RECORDS 3 2 4 4 5 4 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 11; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=50 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; name count_reset index_page_merge_attempts 0 index_page_merge_successful 0 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 10; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=35 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; name count_reset index_page_merge_attempts 1 index_page_merge_successful 1 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 9; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=25 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; @@ -872,7 +866,7 @@ PAGE_NUMBER NUMBER_RECORDS 4 2 27 21 28 21 -set global innodb_purge_stop_now=ON; +begin; delete from tab1 where a = 33; delete from tab1 where a = 34; delete from tab1 where a = 35; @@ -893,7 +887,8 @@ delete from tab1 where a = 18; delete from tab1 where a = 19; delete from tab1 where a = 20; delete from tab1 where a = 21; -set global innodb_purge_run_now=ON; +commit; +InnoDB 0 transactions not purged # check page merge happens (nothing is expected) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; @@ -910,27 +905,24 @@ PAGE_NUMBER NUMBER_RECORDS 4 2 27 11 28 11 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 32; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=50 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; name count_reset index_page_merge_attempts 1 index_page_merge_successful 1 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 31; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=45 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; name count_reset index_page_merge_attempts 2 index_page_merge_successful 2 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 30; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=40 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; @@ -1000,7 +992,7 @@ PAGE_NUMBER NUMBER_RECORDS 4 2 27 21 28 21 -set global innodb_purge_stop_now=ON; +begin; delete from tab1 where a = 33; delete from tab1 where a = 34; delete from tab1 where a = 35; @@ -1021,7 +1013,8 @@ delete from tab1 where a = 18; delete from tab1 where a = 19; delete from tab1 where a = 20; delete from tab1 where a = 21; -set global innodb_purge_run_now=ON; +commit; +InnoDB 0 transactions not purged # check page merge happens (nothing is expected) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; @@ -1038,27 +1031,24 @@ PAGE_NUMBER NUMBER_RECORDS 4 2 27 11 28 11 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 32; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=50 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; name count_reset index_page_merge_attempts 0 index_page_merge_successful 0 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 31; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=45 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; name count_reset index_page_merge_attempts 1 index_page_merge_successful 1 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 30; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=40 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; @@ -1128,7 +1118,7 @@ PAGE_NUMBER NUMBER_RECORDS 4 2 27 21 28 21 -set global innodb_purge_stop_now=ON; +begin; delete from tab1 where a = 33; delete from tab1 where a = 34; delete from tab1 where a = 35; @@ -1149,7 +1139,8 @@ delete from tab1 where a = 18; delete from tab1 where a = 19; delete from tab1 where a = 20; delete from tab1 where a = 21; -set global innodb_purge_run_now=ON; +commit; +InnoDB 0 transactions not purged # check page merge happens (nothing is expected) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; @@ -1166,27 +1157,24 @@ PAGE_NUMBER NUMBER_RECORDS 4 2 27 11 28 11 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 32; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=50 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; name count_reset index_page_merge_attempts 0 index_page_merge_successful 0 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 31; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=45 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; name count_reset index_page_merge_attempts 0 index_page_merge_successful 0 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 30; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=40 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; @@ -1257,7 +1245,7 @@ PAGE_NUMBER NUMBER_RECORDS 4 2 27 21 28 21 -set global innodb_purge_stop_now=ON; +begin; delete from tab1 where a = 33; delete from tab1 where a = 34; delete from tab1 where a = 35; @@ -1278,7 +1266,8 @@ delete from tab1 where a = 18; delete from tab1 where a = 19; delete from tab1 where a = 20; delete from tab1 where a = 21; -set global innodb_purge_run_now=ON; +commit; +InnoDB 0 transactions not purged # check page merge happens (nothing is expected) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; @@ -1295,27 +1284,24 @@ PAGE_NUMBER NUMBER_RECORDS 4 2 27 11 28 11 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 32; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=50 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; name count_reset index_page_merge_attempts 0 index_page_merge_successful 0 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 31; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=45 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; name count_reset index_page_merge_attempts 1 index_page_merge_successful 1 -set global innodb_purge_stop_now=ON; delete from tab1 where a = 30; -set global innodb_purge_run_now=ON; +InnoDB 0 transactions not purged # check page merge happens (MERGE_THRESHOLD=40 causes merge here) SELECT name,count_reset FROM information_schema.innodb_metrics WHERE name like 'index_page_merge_%'; @@ -1323,3 +1309,4 @@ name count_reset index_page_merge_attempts 2 index_page_merge_successful 2 DROP TABLE tab1; +SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency; diff --git a/mysql-test/suite/innodb/r/innodb_defragment_fill_factor.result b/mysql-test/suite/innodb/r/innodb_defragment_fill_factor.result index e1e616a7e6f..5809542db8a 100644 --- a/mysql-test/suite/innodb/r/innodb_defragment_fill_factor.result +++ b/mysql-test/suite/innodb/r/innodb_defragment_fill_factor.result @@ -1,5 +1,3 @@ -DROP TABLE if exists t1; -DROP TABLE if exists t2; Testing tables with large records CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARCHAR(256), c VARCHAR(256), KEY SECOND(a, b,c)) ENGINE=INNODB; INSERT INTO t1 VALUES (1, REPEAT('A', 256), REPEAT('B', 256)); @@ -13,6 +11,7 @@ INSERT INTO t1 (b) SELECT b from t1; INSERT INTO t1 (b) SELECT b from t1; INSERT INTO t1 (b) SELECT b from t1; INSERT INTO t1 (b) SELECT b from t1; +SET GLOBAL innodb_fast_shutdown = 0; optimize table t1; Table Op Msg_type Msg_text test.t1 optimize status OK @@ -57,6 +56,7 @@ insert into t1 values (204, REPEAT('A', 256), REPEAT('B', 256)); DROP TABLE t1; Testing table with small records CREATE TABLE t2 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARCHAR(16), c VARCHAR(32), KEY SECOND(a,b,c)) ENGINE=INNODB; +SET GLOBAL innodb_fast_shutdown = 0; optimize table t2; Table Op Msg_type Msg_text test.t2 optimize status OK diff --git a/mysql-test/suite/innodb/t/index_merge_threshold.test b/mysql-test/suite/innodb/t/index_merge_threshold.test index 7cbde0acf03..a587e10db6c 100644 --- a/mysql-test/suite/innodb/t/index_merge_threshold.test +++ b/mysql-test/suite/innodb/t/index_merge_threshold.test @@ -11,9 +11,11 @@ # Check actual behavior for table, partitioned table and temporary table # ############################################################# --source include/have_innodb_16k.inc ---source include/have_debug.inc --source include/have_partition.inc +SET @saved_frequency = @@GLOBAL.innodb_purge_rseg_truncate_frequency; +SET GLOBAL innodb_purge_rseg_truncate_frequency = 1; + # Check index merge threshold by create index on all datatypes CREATE TABLE tab(a BIGINT PRIMARY KEY,c1 TINYTEXT,c2 TEXT,c3 MEDIUMTEXT, @@ -187,3 +189,4 @@ CREATE INDEX index1 ON tab1(b(750)) COMMENT 'MERGE_THRESHOLD=45'; --source suite/innodb/include/innodb_merge_threshold_secondary.inc DROP TABLE tab1; +SET GLOBAL innodb_purge_rseg_truncate_frequency = @saved_frequency; diff --git a/mysql-test/suite/innodb/t/innodb_defragment_fill_factor.test b/mysql-test/suite/innodb/t/innodb_defragment_fill_factor.test index 2edc8a45c02..3a5897b9911 100644 --- a/mysql-test/suite/innodb/t/innodb_defragment_fill_factor.test +++ b/mysql-test/suite/innodb/t/innodb_defragment_fill_factor.test @@ -4,11 +4,6 @@ --source include/not_embedded.inc --source include/have_innodb_16k.inc ---disable_warnings -DROP TABLE if exists t1; -DROP TABLE if exists t2; ---enable_warnings - --echo Testing tables with large records # Create table. CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY AUTO_INCREMENT, b VARCHAR(256), c VARCHAR(256), KEY SECOND(a, b,c)) ENGINE=INNODB; @@ -36,7 +31,7 @@ while ($size) } --enable_query_log ---source include/wait_innodb_all_purged.inc +SET GLOBAL innodb_fast_shutdown = 0; --source include/restart_mysqld.inc optimize table t1; @@ -135,7 +130,7 @@ while ($size) } --enable_query_log ---source include/wait_innodb_all_purged.inc +SET GLOBAL innodb_fast_shutdown = 0; --source include/restart_mysqld.inc optimize table t2; @@ -206,5 +201,3 @@ if ($second_before == $second_after) { } DROP TABLE t2; - ---source include/wait_innodb_all_purged.inc diff --git a/mysql-test/suite/innodb_zip/r/innochecksum_2.result b/mysql-test/suite/innodb_zip/r/innochecksum_2.result index 78d8a1cbf90..582bb42f0cb 100644 --- a/mysql-test/suite/innodb_zip/r/innochecksum_2.result +++ b/mysql-test/suite/innodb_zip/r/innochecksum_2.result @@ -31,6 +31,7 @@ allow-mismatches 0 write crc32 page-type-summary FALSE page-type-dump MYSQLTEST_VARDIR/tmp/dump.txt +per-page-details FALSE log (No default value) leaf FALSE merge 0 @@ -41,7 +42,7 @@ innochecksum Ver #.#.# Copyright (c) YEAR, YEAR , Oracle, MariaDB Corporation Ab and others. InnoDB offline file checksum utility. -Usage: innochecksum [-c] [-s <start page>] [-e <end page>] [-p <page>] [-v] [-a <allow mismatches>] [-n] [-C <strict-check>] [-w <write>] [-S] [-D <page type dump>] [-l <log>] [-e] <filename or [-]> +Usage: innochecksum [-c] [-s <start page>] [-e <end page>] [-p <page>] [-i] [-v] [-a <allow mismatches>] [-n] [-C <strict-check>] [-w <write>] [-S] [-D <page type dump>] [-l <log>] [-l] [-m <merge pages>] <filename or [-]> -?, --help Displays this help and exits. -I, --info Synonym for --help. -V, --version Displays version information and exits. @@ -62,8 +63,10 @@ Usage: innochecksum [-c] [-s <start page>] [-e <end page>] [-p <page>] [-v] [-a Display a count of each page type in a tablespace. -D, --page-type-dump=name Dump the page type info for each page in a tablespace. + -i, --per-page-details + Print out per-page detail information. -l, --log=name log output. - -e, --leaf Examine leaf index pages + -f, --leaf Examine leaf index pages -m, --merge=# leaf page count if merge given number of consecutive pages @@ -81,6 +84,7 @@ allow-mismatches 0 write crc32 page-type-summary FALSE page-type-dump (No default value) +per-page-details FALSE log (No default value) leaf FALSE merge 0 diff --git a/mysql-test/suite/innodb_zip/r/innochecksum_3.result b/mysql-test/suite/innodb_zip/r/innochecksum_3.result index 800556c4ff3..aaab68b3df9 100644 --- a/mysql-test/suite/innodb_zip/r/innochecksum_3.result +++ b/mysql-test/suite/innodb_zip/r/innochecksum_3.result @@ -1,5 +1,6 @@ # Set the environmental variables call mtr.add_suppression("InnoDB: Unable to read tablespace .* page no .* into the buffer pool after 100 attempts"); +call mtr.add_suppression("InnoDB: innodb_checksum_algorithm is set to.*"); [1]: Further Test are for rewrite checksum (innodb|crc32|none) for all ibd file & start the server. CREATE TABLE tab1 (pk INTEGER NOT NULL PRIMARY KEY, linestring_key GEOMETRY NOT NULL, @@ -105,6 +106,7 @@ File::tab#.ibd # Page compressed page # Page compressed encrypted page # Other type of page + =============================================== Additional information: Undo page type: # insert, # update, # other @@ -139,6 +141,7 @@ File::tab#.ibd # Page compressed page # Page compressed encrypted page # Other type of page + =============================================== Additional information: Undo page type: # insert, # update, # other @@ -160,14 +163,14 @@ Filename::tab#.ibd ============================================================================== PAGE_NO | PAGE_TYPE | EXTRA INFO ============================================================================== -#:: # | File Space Header | - -#:: # | Insert Buffer Bitmap | - -#:: # | Inode page | - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Freshly allocated page | - +#::# | File Space Header | - +#::# | Insert Buffer Bitmap | - +#::# | Inode page | - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Freshly allocated page | - # Variables used by page type dump for ibdata1 Variables (--variable-name=value) @@ -184,6 +187,7 @@ allow-mismatches 0 write crc32 page-type-summary FALSE page-type-dump MYSQLTEST_VARDIR/tmp/dump.txt +per-page-details FALSE log (No default value) leaf FALSE merge 0 @@ -194,14 +198,14 @@ Filename::tab#.ibd ============================================================================== PAGE_NO | PAGE_TYPE | EXTRA INFO ============================================================================== -#:: # | File Space Header | - -#:: # | Insert Buffer Bitmap | - -#:: # | Inode page | - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Index page | index id=#, page level=# leaf #, No. of records=#, garbage=#, n_recs=#, - -#:: # | Freshly allocated page | - +#::# | File Space Header | - +#::# | Insert Buffer Bitmap | - +#::# | Inode page | - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Index page | index id=#, page level=#, No. of records=#, garbage=#, - +#::# | Freshly allocated page | - [6]: check the valid lower bound values for option # allow-mismatches,page,start-page,end-page [9]: check the both short and long options "page" and "start-page" when diff --git a/mysql-test/suite/innodb_zip/t/innochecksum_3.test b/mysql-test/suite/innodb_zip/t/innochecksum_3.test index 54c67ff1a9b..dab10dcc997 100644 --- a/mysql-test/suite/innodb_zip/t/innochecksum_3.test +++ b/mysql-test/suite/innodb_zip/t/innochecksum_3.test @@ -15,7 +15,9 @@ let MYSQLD_BASEDIR= `SELECT @@basedir`; let MYSQLD_DATADIR= `SELECT @@datadir`; let SEARCH_FILE= $MYSQLTEST_VARDIR/log/my_restart.err; + call mtr.add_suppression("InnoDB: Unable to read tablespace .* page no .* into the buffer pool after 100 attempts"); +call mtr.add_suppression("InnoDB: innodb_checksum_algorithm is set to.*"); --echo [1]: Further Test are for rewrite checksum (innodb|crc32|none) for all ibd file & start the server. diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 61006c8d89d..d1476cca67f 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -31,16 +31,17 @@ The database buffer buf_pool Created 11/5/1995 Heikki Tuuri *******************************************************/ -#include "ha_prototypes.h" - +#include "univ.i" +#include "mtr0types.h" +#include "mach0data.h" #include "page0size.h" #include "buf0buf.h" -#include "os0api.h" +#include <string.h> + +#ifdef UNIV_NONINL +#include "buf0buf.ic" +#endif -#ifdef UNIV_INNOCHECKSUM -#include "string.h" -#include "mach0data.h" -#endif /* UNIV_INNOCHECKSUM */ #ifndef UNIV_INNOCHECKSUM #include "mem0mem.h" #include "btr0btr.h" @@ -64,7 +65,6 @@ Created 11/5/1995 Heikki Tuuri #include "fsp0sysspace.h" #endif /* !UNIV_INNOCHECKSUM */ #include "page0zip.h" -#include "buf0checksum.h" #include "sync0sync.h" #include "buf0dump.h" #include "ut0new.h" @@ -645,10 +645,10 @@ buf_page_is_checksum_valid_crc32( #ifdef UNIV_INNOCHECKSUM if (log_file && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " crc32 calculated = %u;" - " recorded checksum field1 = %lu recorded" - " checksum field2 =%lu\n", cur_page_num, + " recorded checksum field1 = " ULINTPF " recorded" + " checksum field2 =" ULINTPF "\n", cur_page_num, crc32, checksum_field1, checksum_field2); } #endif /* UNIV_INNOCHECKSUM */ @@ -702,33 +702,34 @@ buf_page_is_checksum_valid_innodb( #ifdef UNIV_INNOCHECKSUM if (log_file && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB) { - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " old style: calculated =" - " %lu; recorded = %lu\n", + " " ULINTPF "; recorded = " ULINTPF "\n", cur_page_num, old_checksum, checksum_field2); - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " new style: calculated =" - " %lu; crc32 = %u; recorded = %lu\n", + " " ULINTPF "; crc32 = %u; recorded = " ULINTPF "\n", cur_page_num, new_checksum, buf_calc_page_crc32(read_buf), checksum_field1); } if (log_file && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " old style: calculated =" - " %lu; recorded checksum = %lu\n", + " " ULINTPF "; recorded checksum = " ULINTPF "\n", cur_page_num, old_checksum, checksum_field2); - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " new style: calculated =" - " %lu; recorded checksum = %lu\n", + " " ULINTPF "; recorded checksum = " ULINTPF "\n", cur_page_num, new_checksum, checksum_field1); } #endif /* UNIV_INNOCHECKSUM */ + if (checksum_field2 != mach_read_from_4(read_buf + FIL_PAGE_LSN) && checksum_field2 != old_checksum) { DBUG_LOG("checksum", @@ -788,9 +789,9 @@ buf_page_is_checksum_valid_none( if (log_file && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_NONE) { fprintf(log_file, - "page::%lu; none checksum: calculated" - " = %lu; recorded checksum_field1 = %lu" - " recorded checksum_field2 = %lu\n", + "page::%llu; none checksum: calculated" + " = " ULINTPF "; recorded checksum_field1 = " ULINTPF + " recorded checksum_field2 = " ULINTPF "\n", cur_page_num, BUF_NO_CHECKSUM_MAGIC, checksum_field1, checksum_field2); } @@ -811,16 +812,18 @@ buf_page_is_corrupted( bool check_lsn, const byte* read_buf, const page_size_t& page_size, +#ifndef UNIV_INNOCHECKSUM const fil_space_t* space) +#else + const void* space) +#endif { - ulint checksum_field1; - ulint checksum_field2; - + size_t checksum_field1 = 0; + size_t checksum_field2 = 0; #ifndef UNIV_INNOCHECKSUM DBUG_EXECUTE_IF("buf_page_import_corrupt_failure", return(true); ); - - ulint page_type = mach_read_from_2( - read_buf + FIL_PAGE_TYPE); +#endif + ulint page_type = mach_read_from_2(read_buf + FIL_PAGE_TYPE); /* We can trust page type if page compression is set on tablespace flags because page compression flag means file must have been @@ -833,15 +836,12 @@ buf_page_is_corrupted( decompressed at this stage). */ if ((page_type == FIL_PAGE_PAGE_COMPRESSED || page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) - && space && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags)) { - return(false); - } -#else - if (mach_read_from_4(read_buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0 - || mach_read_from_2(read_buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { +#ifndef UNIV_INNOCHECKSUM + && space && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags) +#endif + ) { return(false); } -#endif if (!page_size.is_compressed() && memcmp(read_buf + FIL_PAGE_LSN + 4, @@ -850,14 +850,13 @@ buf_page_is_corrupted( /* Stored log sequence numbers at the start and the end of page do not match */ - #ifndef UNIV_INNOCHECKSUM ib::info() << "Log sequence number at the start " << mach_read_from_4(read_buf + FIL_PAGE_LSN + 4) << " and the end " << mach_read_from_4(read_buf + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + 4) << " do not match"; -#endif +#endif /* UNIV_INNOCHECKSUM */ return(true); } @@ -920,10 +919,9 @@ buf_page_is_corrupted( && *reinterpret_cast<const ib_uint64_t*>( read_buf + FIL_PAGE_LSN) == 0) { - /* make sure that the page is really empty */ - - ulint i; + ulint i; + /* make sure that the page is really empty */ for (i = 0; i < page_size.logical(); ++i) { /* The FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID has been @@ -944,7 +942,7 @@ buf_page_is_corrupted( #ifdef UNIV_INNOCHECKSUM if (i >= page_size.logical()) { if (log_file) { - fprintf(log_file, "Page::%lu" + fprintf(log_file, "Page::%llu" " is empty and uncorrupted\n", cur_page_num); } @@ -978,31 +976,34 @@ buf_page_is_corrupted( if (buf_page_is_checksum_valid_none(read_buf, checksum_field1, checksum_field2)) { -#ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_NONE, page_id); +#endif /* !UNIV_INNOCHECKSUM */ } -#else /* !UNIV_INNOCHECKSUM */ + +#ifdef UNIV_INNOCHECKSUM if (log_file) { - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " old style: calculated = %u;" - " recorded = " ULINTPF "\n", + " recorded = " ULINTPF ";\n", cur_page_num, buf_calc_page_old_checksum(read_buf), checksum_field2); - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " new style: calculated = %u;" - " crc32 = %u; recorded = " ULINTPF "\n", + " crc32 = %u; recorded = " ULINTPF ";\n", cur_page_num, buf_calc_page_new_checksum(read_buf), buf_calc_page_crc32(read_buf), checksum_field1); } -#endif /* !UNIV_INNOCHECKSUM */ +#endif /* UNIV_INNOCHECKSUM */ + return(false); } @@ -1022,15 +1023,16 @@ buf_page_is_corrupted( if (buf_page_is_checksum_valid_innodb(read_buf, checksum_field1, checksum_field2)) { -#ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_INNODB, page_id); +#endif } -#endif /* !UNIV_INNOCHECKSUM */ + return(false); } @@ -1038,13 +1040,13 @@ buf_page_is_corrupted( if (!legacy_checksum_checked && buf_page_is_checksum_valid_crc32( read_buf, checksum_field1, checksum_field2, true)) { - legacy_big_endian_checksum = true; - return(false); + legacy_big_endian_checksum = true; + return(false); } #ifdef UNIV_INNOCHECKSUM if (log_file) { - fprintf(log_file, "Fail; page %lu" + fprintf(log_file, "Fail; page::%llu;" " invalid (fails crc32 checksum)\n", cur_page_num); } @@ -1061,30 +1063,32 @@ buf_page_is_corrupted( if (buf_page_is_checksum_valid_none(read_buf, checksum_field1, checksum_field2)) { -#ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_NONE, page_id); +#endif } -#else /* !UNIV_INNOCHECKSUM */ +#ifdef UNIV_INNOCHECKSUM if (log_file) { - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " old style: calculated = %u;" - " recorded = %lu\n", cur_page_num, + " recorded = %zu;\n", cur_page_num, buf_calc_page_old_checksum(read_buf), checksum_field2); - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " new style: calculated = %u;" - " crc32 = %u; recorded = %lu\n", + " crc32 = %u; recorded = %zu;\n", cur_page_num, buf_calc_page_new_checksum(read_buf), buf_calc_page_crc32(read_buf), checksum_field1); } -#endif /* !UNIV_INNOCHECKSUM */ +#endif /* UNIV_INNOCHECKSUM */ + return(false); } @@ -1092,26 +1096,28 @@ buf_page_is_corrupted( checksum_field1, checksum_field2, false) || buf_page_is_checksum_valid_crc32(read_buf, checksum_field1, checksum_field2, true)) { -#ifndef UNIV_INNOCHECKSUM + if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_CRC32, page_id); +#endif } -#endif /* !UNIV_INNOCHECKSUM */ return(false); } #ifdef UNIV_INNOCHECKSUM if (log_file) { - fprintf(log_file, "Fail; page %lu" + fprintf(log_file, "Fail; page::%llu;" " invalid (fails innodb checksum)\n", cur_page_num); } #endif /* UNIV_INNOCHECKSUM */ + return(true); case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: @@ -1147,11 +1153,12 @@ buf_page_is_corrupted( #ifdef UNIV_INNOCHECKSUM if (log_file) { - fprintf(log_file, "Fail; page %lu" + fprintf(log_file, "Fail; page::%llu;" " invalid (fails none checksum)\n", cur_page_num); } #endif /* UNIV_INNOCHECKSUM */ + return(true); case SRV_CHECKSUM_ALGORITHM_NONE: diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 7917cbb528b..6e431a6ee0f 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -24,6 +24,7 @@ Modified Jan Lindström jan.lindstrom@mariadb.com *******************************************************/ #include "fil0fil.h" +#include "mtr0types.h" #include "mach0data.h" #include "page0size.h" #include "page0zip.h" diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 9e088571f10..455fd4c8389 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1689,18 +1689,6 @@ thd_is_replication_slave_thread( } /******************************************************************//** -Gets information on the durability property requested by thread. -Used when writing either a prepare or commit record to the log -buffer. @return the durability property. */ -enum durability_properties -thd_requested_durability( -/*=====================*/ - const THD* thd) /*!< in: thread handle */ -{ - return(thd_get_durability_property(thd)); -} - -/******************************************************************//** Returns true if transaction should be flagged as read-only. @return true if the thd is marked as read-only */ bool @@ -4841,10 +4829,6 @@ innobase_commit( this one, to allow then to group commit with us. */ thd_wakeup_subsequent_commits(thd, 0); - if (!read_only) { - trx->flush_log_later = false; - } - /* Now do a write + flush of logs. */ trx_commit_complete_for_mysql(trx); diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index f9ad2007a5f..82ba8bab6e6 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -570,11 +570,6 @@ bool thd_binlog_filter_ok(const MYSQL_THD thd); */ bool thd_sqlcom_can_generate_row_events(const MYSQL_THD thd); -/** Gets information on the durability property requested by a thread. -@param thd Thread handle -@return a durability property. */ -durability_properties thd_get_durability_property(const MYSQL_THD thd); - /** Is strict sql_mode set. @param thd Thread object @return True if sql_mode has strict mode (all or trans), false otherwise. */ diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 9d5f373f5de..67de1174d25 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -211,6 +211,7 @@ struct buf_pools_list_size_t { ulint unzip_LRU_bytes; /*!< unzip_LRU size in bytes */ ulint flush_list_bytes; /*!< flush_list size in bytes */ }; +#endif /* !UNIV_INNOCHECKSUM */ /** Page identifier. */ class page_id_t { @@ -333,6 +334,7 @@ operator<<( std::ostream& out, const page_id_t& page_id); +#ifndef UNIV_INNOCHECKSUM /********************************************************************//** Acquire mutex on all buffer pool instances */ UNIV_INLINE @@ -823,9 +825,16 @@ buf_page_is_corrupted( bool check_lsn, const byte* read_buf, const page_size_t& page_size, +#ifndef UNIV_INNOCHECKSUM const fil_space_t* space = NULL) +#else + const void* space = NULL) +#endif MY_ATTRIBUTE((warn_unused_result)); + + #ifndef UNIV_INNOCHECKSUM + /**********************************************************************//** Gets the space id, page offset, and byte offset within page of a pointer pointing to a buffer frame containing a file page. */ diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index 06785430c25..13b3ec4e37e 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -30,6 +30,7 @@ Created 04/01/2015 Jan Lindström #include "os0event.h" #include "my_crypt.h" +#endif /*! UNIV_INNOCHECKSUM */ /** * Magic pattern in start of crypt data on page 0 @@ -101,6 +102,8 @@ struct fil_space_rotate_state_t } scrubbing; }; +#ifndef UNIV_INNOCHECKSUM + struct fil_space_crypt_t : st_encryption_scheme { public: diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index bf231565657..097be6a5f96 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -28,8 +28,6 @@ Created 10/25/1995 Heikki Tuuri #define fil0fil_h #include "univ.i" -struct fil_space_t; - #ifndef UNIV_INNOCHECKSUM #include "log0recv.h" @@ -281,22 +279,23 @@ but in the MySQL Embedded Server Library and mysqlbackup it is not the default directory, and we must set the base file path explicitly */ extern const char* fil_path_to_mysql_datadir; -/** Initial size of a single-table tablespace in pages */ -#define FIL_IBD_FILE_INITIAL_SIZE 4 - -/** 'null' (undefined) page offset in the context of file spaces */ -#define FIL_NULL ULINT32_UNDEFINED - /* Space address data type; this is intended to be used when addresses accurate to a byte are stored in file pages. If the page part of the address is FIL_NULL, the address is considered undefined. */ typedef byte fil_faddr_t; /*!< 'type' definition in C: an address stored in a file page is a string of bytes */ +#endif /* !UNIV_INNOCHECKSUM */ + +/** Initial size of a single-table tablespace in pages */ +#define FIL_IBD_FILE_INITIAL_SIZE 4 + +/** 'null' (undefined) page offset in the context of file spaces */ +#define FIL_NULL ULINT32_UNDEFINED + #define FIL_ADDR_PAGE 0 /* first in address is the page offset */ #define FIL_ADDR_BYTE 4 /* then comes 2-byte byte offset within page*/ -#endif /* !UNIV_INNOCHECKSUM */ #define FIL_ADDR_SIZE 6 /* address size is 6 bytes */ #ifndef UNIV_INNOCHECKSUM @@ -431,8 +430,6 @@ index */ #define fil_page_index_page_check(page) \ fil_page_type_is_index(fil_page_get_type(page)) -#ifndef UNIV_INNOCHECKSUM - /** Enum values for encryption table option */ enum fil_encryption_t { /** Encrypted if innodb_encrypt_tables=ON (srv_encrypt_tables) */ @@ -454,6 +451,8 @@ extern ulint fil_n_pending_tablespace_flushes; /** Number of files currently open */ extern ulint fil_n_file_opened; +#ifndef UNIV_INNOCHECKSUM + /** Look up a tablespace. The caller should hold an InnoDB table lock or a MDL that prevents the tablespace from being dropped during the operation, @@ -1315,6 +1314,7 @@ fil_page_reset_type( byte* page, ulint type, mtr_t* mtr); + /** Get the file page type. @param[in] page file page @return page type */ @@ -1325,6 +1325,7 @@ fil_page_get_type( { return(mach_read_from_2(page + FIL_PAGE_TYPE)); } + /** Check (and if needed, reset) the page type. Data files created before MySQL 5.1 may contain garbage in the FIL_PAGE_TYPE field. diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index f9167e0462e..e5057b30501 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -158,8 +158,6 @@ descriptor page, but used only in the first. */ FSP_FREE_LIMIT at a time */ /* @} */ -#ifndef UNIV_INNOCHECKSUM - /* @defgroup File Segment Inode Constants (moved from fsp0fsp.c) @{ */ /* FILE SEGMENT INODE @@ -293,6 +291,7 @@ the extent are free and which contain old tuple version to clean. */ /** Offset of the descriptor array on a descriptor page */ #define XDES_ARR_OFFSET (FSP_HEADER_OFFSET + FSP_HEADER_SIZE) +#ifndef UNIV_INNOCHECKSUM /* @} */ /**********************************************************************//** diff --git a/storage/innobase/include/fsp0fsp.ic b/storage/innobase/include/fsp0fsp.ic index 6317a67a089..2da3320eef7 100644 --- a/storage/innobase/include/fsp0fsp.ic +++ b/storage/innobase/include/fsp0fsp.ic @@ -53,6 +53,7 @@ xdes_calc_descriptor_index( return(ut_2pow_remainder(offset, page_size.physical()) / FSP_EXTENT_SIZE); } +#endif /*!UNIV_INNOCHECKSUM */ /**********************************************************************//** Gets a descriptor bit of a page. @@ -80,6 +81,7 @@ xdes_get_bit( bit_index)); } +#ifndef UNIV_INNOCHECKSUM /** Calculates the page where the descriptor of a page resides. @param[in] page_size page size @param[in] offset page offset diff --git a/storage/innobase/include/mach0data.h b/storage/innobase/include/mach0data.h index 5a5574b2537..684a3155a16 100644 --- a/storage/innobase/include/mach0data.h +++ b/storage/innobase/include/mach0data.h @@ -63,6 +63,7 @@ mach_write_to_2( /*============*/ byte* b, /*!< in: pointer to two bytes where to store */ ulint n); /*!< in: ulint integer to be stored, >= 0, < 64k */ +#endif /* !UNIV_INNOCHECKSUM */ /** The following function is used to fetch data from 2 consecutive bytes. The most significant byte is at the lowest address. @param[in] b pointer to 2 bytes where to store @@ -72,6 +73,8 @@ uint16_t mach_read_from_2( const byte* b) MY_ATTRIBUTE((warn_unused_result)); + +#ifndef UNIV_INNOCHECKSUM /********************************************************//** The following function is used to convert a 16-bit data item to the canonical format, for fast bytewise equality test @@ -362,6 +365,8 @@ mach_write_ulonglong( ulint len, /*!< in: length of dest */ bool usign); /*!< in: signed or unsigned flag */ +#endif /* !UNIV_INNOCHECKSUM */ + /** Read 1 to 4 bytes from a file page buffered in the buffer pool. @param[in] ptr pointer where to read @param[in] type MLOG_1BYTE, MLOG_2BYTES, or MLOG_4BYTES @@ -373,8 +378,6 @@ mach_read_ulint( mlog_id_t type) MY_ATTRIBUTE((warn_unused_result)); -#endif /* !UNIV_INNOCHECKSUM */ - #include "mach0data.ic" #endif diff --git a/storage/innobase/include/mach0data.ic b/storage/innobase/include/mach0data.ic index c89e4960480..9c4760b6d3a 100644 --- a/storage/innobase/include/mach0data.ic +++ b/storage/innobase/include/mach0data.ic @@ -865,6 +865,8 @@ mach_write_ulonglong( } } +#endif /* !UNIV_INNOCHECKSUM */ + /** Read 1 to 4 bytes from a file page buffered in the buffer pool. @param[in] ptr pointer where to read @param[in] type MLOG_1BYTE, MLOG_2BYTES, or MLOG_4BYTES @@ -889,5 +891,3 @@ mach_read_ulint( ut_error; return(0); } - -#endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/include/page0page.h b/storage/innobase/include/page0page.h index a8c5f11c17b..0aca13ac159 100644 --- a/storage/innobase/include/page0page.h +++ b/storage/innobase/include/page0page.h @@ -85,8 +85,6 @@ Otherwise written as 0. @see PAGE_ROOT_AUTO_INC */ This field should not be written to after page creation. */ -#ifndef UNIV_INNOCHECKSUM - #define PAGE_BTR_SEG_LEAF 36 /* file segment header for the leaf pages in a B-tree: defined only on the root page of a B-tree, but not in the root of an ibuf tree */ @@ -141,6 +139,8 @@ Otherwise written as 0. @see PAGE_ROOT_AUTO_INC */ #define PAGE_SAME_PAGE 4 #define PAGE_NO_DIRECTION 5 +#ifndef UNIV_INNOCHECKSUM + /* PAGE DIRECTORY ============== */ diff --git a/storage/innobase/include/page0page.ic b/storage/innobase/include/page0page.ic index 1c680ce7cc4..2ad5f26dcc1 100644 --- a/storage/innobase/include/page0page.ic +++ b/storage/innobase/include/page0page.ic @@ -27,8 +27,8 @@ Created 2/2/1994 Heikki Tuuri #ifndef page0page_ic #define page0page_ic -#include "mach0data.h" #ifndef UNIV_INNOCHECKSUM +#include "mach0data.h" #ifdef UNIV_DEBUG # include "log0recv.h" #endif /* !UNIV_DEBUG */ @@ -40,6 +40,7 @@ Created 2/2/1994 Heikki Tuuri #undef UNIV_INLINE #define UNIV_INLINE #endif +#endif /* !UNIV_INNOCHECKSUM */ /************************************************************//** Gets the start of a page. @@ -53,6 +54,7 @@ page_align( return((page_t*) ut_align_down(ptr, UNIV_PAGE_SIZE)); } +#ifndef UNIV_INNOCHECKSUM /************************************************************//** Gets the offset within a page. @return offset from the start of the page */ @@ -181,7 +183,6 @@ page_header_get_field( } #ifndef UNIV_INNOCHECKSUM - /*************************************************************//** Sets the given header field. */ UNIV_INLINE @@ -285,6 +286,8 @@ page_header_reset_last_insert( } } +#endif /* !UNIV_INNOCHECKSUM */ + /************************************************************//** Determine whether the page is in new-style compact format. @return nonzero if the page is in compact format, zero if it is in @@ -298,6 +301,7 @@ page_is_comp( return(page[PAGE_HEADER + PAGE_N_HEAP] & 0x80); } +#ifndef UNIV_INNOCHECKSUM /************************************************************//** TRUE if the record is on a page in compact format. @return nonzero if in compact format */ @@ -326,6 +330,8 @@ page_rec_get_heap_no( } } +#endif /* !UNIV_INNOCHECKSUM */ + /************************************************************//** Determine whether the page is a B-tree leaf. @return true if the page is a B-tree leaf (PAGE_LEVEL = 0) */ @@ -338,6 +344,7 @@ page_is_leaf( return(!*(const uint16*) (page + (PAGE_HEADER + PAGE_LEVEL))); } +#ifndef UNIV_INNOCHECKSUM /************************************************************//** Determine whether the page is empty. @return true if the page is empty (PAGE_N_RECS = 0) */ @@ -627,6 +634,8 @@ page_get_middle_rec( return(page_rec_get_nth(page, middle)); } +#endif /* !UNIV_INNOCHECKSUM */ + /*************************************************************//** Gets the page number. @return page number */ @@ -640,6 +649,7 @@ page_get_page_no( return(mach_read_from_4(page + FIL_PAGE_OFFSET)); } +#ifndef UNIV_INNOCHECKSUM /*************************************************************//** Gets the tablespace identifier. @return space id */ @@ -653,6 +663,8 @@ page_get_space_id( return(mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID)); } +#endif /* !UNIV_INNOCHECKSUM */ + /*************************************************************//** Gets the number of user records on page (infimum and supremum records are not user records). @@ -666,6 +678,7 @@ page_get_n_recs( return(page_header_get_field(page, PAGE_N_RECS)); } +#ifndef UNIV_INNOCHECKSUM /*************************************************************//** Gets the number of dir slots in directory. @return number of slots */ @@ -1054,6 +1067,8 @@ page_rec_get_base_extra_size( return(REC_N_NEW_EXTRA_BYTES + (ulint) !page_rec_is_comp(rec)); } +#endif /* UNIV_INNOCHECKSUM */ + /************************************************************//** Returns the sum of the sizes of the records in the record list, excluding the infimum and supremum records. @@ -1077,6 +1092,7 @@ page_get_data_size( return(ret); } +#ifndef UNIV_INNOCHECKSUM /************************************************************//** Allocates a block of memory from the free list of an index page. */ UNIV_INLINE diff --git a/storage/innobase/include/rem0rec.h b/storage/innobase/include/rem0rec.h index 96d3bdfab8f..8d3f87450f8 100644 --- a/storage/innobase/include/rem0rec.h +++ b/storage/innobase/include/rem0rec.h @@ -27,12 +27,14 @@ Created 5/30/1994 Heikki Tuuri #ifndef rem0rec_h #define rem0rec_h +#ifndef UNIV_INNOCHECKSUM #include "univ.i" #include "data0data.h" #include "rem0types.h" #include "mtr0types.h" #include "page0types.h" #include "trx0types.h" +#endif /*! UNIV_INNOCHECKSUM */ #include <ostream> #include <sstream> @@ -92,6 +94,7 @@ offsets[] array, first passed to rec_get_offsets() */ #define REC_OFFS_NORMAL_SIZE OFFS_IN_REC_NORMAL_SIZE #define REC_OFFS_SMALL_SIZE 10 +#ifndef UNIV_INNOCHECKSUM /******************************************************//** The following function is used to get the pointer of the next chained record on the same page. @@ -1112,4 +1115,5 @@ int wsrep_rec_get_foreign_key( #include "rem0rec.ic" +#endif /* !UNIV_INNOCHECKSUM */ #endif /* rem0rec_h */ diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index da4db5b9435..a5635fc4ecd 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -108,13 +108,8 @@ support cross-platform development and expose comonly used SQL names. */ #include <unistd.h> #endif -#ifdef UNIV_INNOCHECKSUM -extern bool strict_verify; -extern FILE* log_file; -extern uintmax_t cur_page_num; -#endif /* UNIV_INNOCHECKSUM */ - #include "my_pthread.h" + /* Following defines are to enable performance schema instrumentation in each of five InnoDB modules if HAVE_PSI_INTERFACE is defined. */ @@ -487,6 +482,12 @@ in both 32-bit and 64-bit environments. */ # define UINT64PFx "%016" PRIx64 #endif +#ifdef UNIV_INNOCHECKSUM +extern bool strict_verify; +extern FILE* log_file; +extern unsigned long long cur_page_num; +#endif /* UNIV_INNOCHECKSUM */ + typedef int64_t ib_int64_t; typedef uint64_t ib_uint64_t; typedef uint32_t ib_uint32_t; diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index dc8977e49c8..0e0e0aeb357 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -821,13 +821,6 @@ recv_find_max_checkpoint_0(log_group_t** max_group, ulint* max_field) continue; } - group->state = LOG_GROUP_OK; - - group->lsn = mach_read_from_8( - buf + LOG_CHECKPOINT_LSN); - group->lsn_offset = static_cast<ib_uint64_t>( - mach_read_from_4(buf + OFFSET_HIGH32)) << 32 - | mach_read_from_4(buf + OFFSET_LOW32); checkpoint_no = mach_read_from_8( buf + LOG_CHECKPOINT_NO); @@ -838,12 +831,21 @@ recv_find_max_checkpoint_0(log_group_t** max_group, ulint* max_field) DBUG_PRINT("ib_log", ("checkpoint " UINT64PF " at " LSN_PF " found", - checkpoint_no, group->lsn)); + checkpoint_no, + mach_read_from_8(buf + LOG_CHECKPOINT_LSN))); if (checkpoint_no >= max_no) { *max_group = group; *max_field = field; max_no = checkpoint_no; + + group->state = LOG_GROUP_OK; + + group->lsn = mach_read_from_8( + buf + LOG_CHECKPOINT_LSN); + group->lsn_offset = static_cast<ib_uint64_t>( + mach_read_from_4(buf + OFFSET_HIGH32)) << 32 + | mach_read_from_4(buf + OFFSET_LOW32); } } @@ -998,22 +1000,22 @@ recv_find_max_checkpoint(ulint* max_field) continue; } - group->state = LOG_GROUP_OK; - - group->lsn = mach_read_from_8( - buf + LOG_CHECKPOINT_LSN); - group->lsn_offset = mach_read_from_8( - buf + LOG_CHECKPOINT_OFFSET); checkpoint_no = mach_read_from_8( buf + LOG_CHECKPOINT_NO); DBUG_PRINT("ib_log", - ("checkpoint " UINT64PF " at " LSN_PF " found ", - checkpoint_no, group->lsn)); + ("checkpoint " UINT64PF " at " LSN_PF " found", + checkpoint_no, mach_read_from_8( + buf + LOG_CHECKPOINT_LSN))); if (checkpoint_no >= max_no) { *max_field = field; max_no = checkpoint_no; + group->state = LOG_GROUP_OK; + group->lsn = mach_read_from_8( + buf + LOG_CHECKPOINT_LSN); + group->lsn_offset = mach_read_from_8( + buf + LOG_CHECKPOINT_OFFSET); } } @@ -3148,7 +3150,11 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn) err = recv_find_max_checkpoint(&max_cp_field); - if (err != DB_SUCCESS) { + if (err != DB_SUCCESS + || (log_sys->log.format != 0 + && (log_sys->log.format & ~LOG_HEADER_FORMAT_ENCRYPTED) + != LOG_HEADER_FORMAT_CURRENT)) { + log_mutex_exit(); return(err); } @@ -3178,8 +3184,7 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn) case 0: log_mutex_exit(); return(recv_log_format_0_recover(checkpoint_lsn)); - case LOG_HEADER_FORMAT_CURRENT: - case LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED: + default: if (end_lsn == 0) { break; } @@ -3187,8 +3192,6 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn) contiguous_lsn = end_lsn; break; } - /* fall through */ - default: recv_sys->found_corrupt_log = true; log_mutex_exit(); return(DB_ERROR); diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index ffb8808d6b4..b6e0409459b 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -4980,24 +4980,19 @@ page_zip_verify_checksum( const void* data, /*!< in: compressed page */ ulint size) /*!< in: size of compressed page */ { - const unsigned char* p = static_cast<const unsigned char*>(data) - + FIL_PAGE_SPACE_OR_CHKSUM; - - const uint32_t stored = static_cast<uint32_t>( - mach_read_from_4(p)); - -#ifdef UNIV_INNOCHECKSUM - p = static_cast<const unsigned char*>(data) + FIL_PAGE_TYPE; - bool no_checksum = (mach_read_from_2(p) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); - p = static_cast<const unsigned char*>(data) + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION; - bool encrypted = (mach_read_from_4(p) != 0); - p = static_cast<const unsigned char*>(data) + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4; - const uint32_t checksum = static_cast<uint32_t>(mach_read_from_4(p)); - - if (no_checksum) { - return (TRUE); - } -#endif + ib_uint32_t stored; + ib_uint32_t calc; + + stored = static_cast<ib_uint32_t>(mach_read_from_4( + static_cast<const unsigned char*>(data) + FIL_PAGE_SPACE_OR_CHKSUM)); + + ulint page_no MY_ATTRIBUTE((unused)) = + mach_read_from_4(static_cast<const unsigned char*> + (data) + FIL_PAGE_OFFSET); + ulint space_id MY_ATTRIBUTE((unused)) = + mach_read_from_4(static_cast<const unsigned char*> + (data) + FIL_PAGE_SPACE_ID); + const page_id_t page_id(space_id, page_no); #if FIL_PAGE_LSN % 8 #error "FIL_PAGE_LSN must be 64 bit aligned" @@ -5017,7 +5012,7 @@ page_zip_verify_checksum( } if (i >= size) { if (log_file) { - fprintf(log_file, "Page::%lu is empty and" + fprintf(log_file, "Page::%llu is empty and" " uncorrupted\n", cur_page_num); } @@ -5041,21 +5036,12 @@ page_zip_verify_checksum( return(TRUE); } -#ifndef UNIV_INNOCHECKSUM - ulint page_no = mach_read_from_4(static_cast< - const unsigned char*> - (data) + FIL_PAGE_OFFSET); - ulint space_id = mach_read_from_4(static_cast< - const unsigned char*> - (data) + FIL_PAGE_SPACE_ID); - const page_id_t page_id(space_id, page_no); -#endif /* UNIV_INNOCHECKSUM */ - - const uint32_t calc = page_zip_calc_checksum(data, size, curr_algo); + calc = static_cast<ib_uint32_t>(page_zip_calc_checksum( + data, size, curr_algo)); #ifdef UNIV_INNOCHECKSUM if (log_file) { - fprintf(log_file, "page::%lu;" + fprintf(log_file, "page::%llu;" " %s checksum: calculated = %u;" " recorded = %u\n", cur_page_num, buf_checksum_algorithm_name( @@ -5070,21 +5056,17 @@ page_zip_verify_checksum( data, size, SRV_CHECKSUM_ALGORITHM_CRC32); if (log_file) { - fprintf(log_file, "page::%lu: crc32 checksum:" + fprintf(log_file, "page::%llu: crc32 checksum:" " calculated = %u; recorded = %u\n", cur_page_num, crc32, stored); - fprintf(log_file, "page::%lu: none checksum:" + fprintf(log_file, "page::%llu: none checksum:" " calculated = %lu; recorded = %u\n", cur_page_num, BUF_NO_CHECKSUM_MAGIC, stored); } } #endif /* UNIV_INNOCHECKSUM */ - if (stored == calc -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && stored == checksum) -#endif - ) { + if (stored == calc) { return(TRUE); } @@ -5116,11 +5098,7 @@ page_zip_verify_checksum( if (legacy_big_endian_checksum) { const uint32_t calculated = page_zip_calc_checksum(data, size, curr_algo, true); - if (stored == calculated -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && calculated == checksum) -#endif - ) { + if (stored == calculated) { return(TRUE); } @@ -5130,11 +5108,7 @@ page_zip_verify_checksum( uint32_t calculated = page_zip_calc_checksum(data, size, SRV_CHECKSUM_ALGORITHM_INNODB); - if (stored == calculated -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && stored == checksum) -#endif - ) { + if (stored == calculated) { #ifndef UNIV_INNOCHECKSUM if (curr_algo @@ -5154,11 +5128,7 @@ page_zip_verify_checksum( /* If legacy checksum is not checked, do it now. */ if ((legacy_checksum_checked - && stored == calculated) -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && calculated == checksum) -#endif - ) { + && stored == calculated)) { legacy_big_endian_checksum = true; return(TRUE); } @@ -5168,11 +5138,7 @@ page_zip_verify_checksum( case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: case SRV_CHECKSUM_ALGORITHM_INNODB: { - if (stored == BUF_NO_CHECKSUM_MAGIC -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && checksum == BUF_NO_CHECKSUM_MAGIC) -#endif - ) { + if (stored == BUF_NO_CHECKSUM_MAGIC) { #ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { @@ -5191,14 +5157,8 @@ page_zip_verify_checksum( uint32_t calculated1; if (stored == calculated -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && checksum == calculated) -#endif - || stored == (calculated1 = + || stored == (calculated1 = page_zip_calc_checksum(data, size, SRV_CHECKSUM_ALGORITHM_CRC32, true)) -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && checksum == calculated1) -#endif ) { #ifndef UNIV_INNOCHECKSUM if (curr_algo @@ -5222,12 +5182,7 @@ page_zip_verify_checksum( data, size, SRV_CHECKSUM_ALGORITHM_CRC32, true); if (stored == calculated - || stored == calculated1 -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && checksum == calculated) - || ( encrypted == true && checksum == calculated1) -#endif - ) { + || stored == calculated1) { #ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, @@ -5240,11 +5195,7 @@ page_zip_verify_checksum( calculated = page_zip_calc_checksum( data, size, SRV_CHECKSUM_ALGORITHM_INNODB); - if (stored == calculated -#ifdef UNIV_INNOCHECKSUM - || ( encrypted == true && checksum == calculated) -#endif - ) { + if (stored == calculated) { #ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 6655be72ba1..55bb2605d7e 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -1845,9 +1845,7 @@ trx_commit_in_memory( } else if (trx->flush_log_later) { /* Do nothing yet */ trx->must_flush_log_later = true; - } else if (srv_flush_log_at_trx_commit == 0 - || thd_requested_durability(trx->mysql_thd) - == HA_IGNORE_DURABILITY) { + } else if (srv_flush_log_at_trx_commit == 0) { /* Do nothing */ } else { trx_flush_log_if_needed(lsn, trx); @@ -2261,8 +2259,7 @@ trx_commit_complete_for_mysql( { if (trx->id != 0 || !trx->must_flush_log_later - || thd_requested_durability(trx->mysql_thd) - == HA_IGNORE_DURABILITY) { + || (srv_flush_log_at_trx_commit == 1 && trx->active_commit_ordered)) { return; } @@ -2750,18 +2747,7 @@ trx_prepare( trx_sys_mutex_exit(); /*--------------------------------------*/ - switch (thd_requested_durability(trx->mysql_thd)) { - case HA_IGNORE_DURABILITY: - /* We set the HA_IGNORE_DURABILITY during prepare phase of - binlog group commit to not flush redo log for every transaction - here. So that we can flush prepared records of transactions to - redo log in a group right before writing them to binary log - during flush stage of binlog group commit. */ - break; - case HA_REGULAR_DURABILITY: - if (lsn == 0) { - break; - } + if (lsn) { /* Depending on the my.cnf options, we may now write the log buffer to the log files, making the prepared state of the transaction durable if the OS does not crash. We may also diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_parts.result b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_parts.result index 9b7ee0de388..70a8119f942 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_parts.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_parts.result @@ -45,9 +45,11 @@ insert into t1 values (1,10,10); insert into t1 values (2,10,10); insert into t1 values (11,20,20); insert into t1 values (12,20,20); +set @tmp_rfirr= @@rocksdb_force_index_records_in_range; +set rocksdb_force_index_records_in_range= 12; explain select * from t1 force index(col1) where col1=10; id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref col1 col1 5 const 2000 +1 SIMPLE t1 ref col1 col1 5 const 24 select * from t1 force index(col1) where col1=10; pk col1 col2 1 10 10 @@ -56,6 +58,7 @@ select * from t1 use index () where col1=10; pk col1 col2 2 10 10 1 10 10 +set rocksdb_force_index_records_in_range= @tmp_rfirr; drop table t1; # # Issue #108: Index-only scans do not work for partitioned tables and extended keys diff --git a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_parts.test b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_parts.test index fd1177b839e..2e2e079820f 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_parts.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/rocksdb_parts.test @@ -55,9 +55,16 @@ insert into t1 values (2,10,10); insert into t1 values (11,20,20); insert into t1 values (12,20,20); + +set @tmp_rfirr= @@rocksdb_force_index_records_in_range; +set rocksdb_force_index_records_in_range= 12; + explain select * from t1 force index(col1) where col1=10; select * from t1 force index(col1) where col1=10; select * from t1 use index () where col1=10; + +set rocksdb_force_index_records_in_range= @tmp_rfirr; + drop table t1; --echo # |