diff options
Diffstat (limited to 'storage/innobase/page/page0zip.cc')
| -rw-r--r-- | storage/innobase/page/page0zip.cc | 861 |
1 files changed, 488 insertions, 373 deletions
diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index 2bf1f324784..167fc73a6d1 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -24,41 +24,37 @@ Compressed page interface Created June 2005 by Marko Makela *******************************************************/ -// First include (the generated) my_config.h, to get correct platform defines. -#include "my_config.h" - -#include <map> -using namespace std; - -#define THIS_MODULE +#include "page0size.h" #include "page0zip.h" #ifdef UNIV_NONINL # include "page0zip.ic" #endif -#undef THIS_MODULE -#include "fil0fil.h" -#include "buf0checksum.h" -#include "mach0data.h" + +/** A BLOB field reference full of zero, for use in assertions and tests. +Initially, BLOB field references are set to zero, in +dtuple_convert_big_rec(). */ +const byte field_ref_zero[FIELD_REF_SIZE] = { + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, +}; + #ifndef UNIV_INNOCHECKSUM #include "page0page.h" #include "mtr0log.h" -#include "ut0sort.h" #include "dict0dict.h" #include "btr0cur.h" #include "page0types.h" #include "log0recv.h" -#else -#define page_warn_strict_checksum(A,B,C,D) -#endif /* !UNIV_INNOCHECKSUM */ +#include "row0trunc.h" #include "zlib.h" #ifndef UNIV_HOTBACKUP -#ifndef UNIV_INNOCHECKSUM # include "buf0buf.h" # include "btr0sea.h" # include "dict0boot.h" # include "lock0lock.h" # include "srv0srv.h" -#endif /* !UNIV_INNOCHECKSUM */ # include "buf0lru.h" # include "srv0mon.h" # include "ut0crc32.h" @@ -68,26 +64,22 @@ using namespace std; # define buf_LRU_stat_inc_unzip() ((void) 0) #endif /* !UNIV_HOTBACKUP */ +#include <map> +#include <algorithm> + #ifndef UNIV_HOTBACKUP -#ifndef UNIV_INNOCHECKSUM /** Statistics on compression, indexed by page_zip_des_t::ssize - 1 */ -UNIV_INTERN page_zip_stat_t page_zip_stat[PAGE_ZIP_SSIZE_MAX]; +page_zip_stat_t page_zip_stat[PAGE_ZIP_SSIZE_MAX]; /** Statistics on compression, indexed by index->id */ -UNIV_INTERN page_zip_stat_per_index_t page_zip_stat_per_index; -/** Mutex protecting page_zip_stat_per_index */ -UNIV_INTERN ib_mutex_t page_zip_stat_per_index_mutex; -#ifdef HAVE_PSI_INTERFACE -UNIV_INTERN mysql_pfs_key_t page_zip_stat_per_index_mutex_key; -#endif /* HAVE_PSI_INTERFACE */ -#endif /* !UNIV_INNOCHECKSUM */ +page_zip_stat_per_index_t page_zip_stat_per_index; #endif /* !UNIV_HOTBACKUP */ /* Compression level to be used by zlib. Settable by user. */ -UNIV_INTERN uint page_zip_level = DEFAULT_COMPRESSION_LEVEL; +uint page_zip_level = DEFAULT_COMPRESSION_LEVEL; /* Whether or not to log compressed page images to avoid possible compression algorithm changes in zlib. */ -UNIV_INTERN my_bool page_zip_log_pages = false; +my_bool page_zip_log_pages = true; /* Please refer to ../include/page0zip.ic for a description of the compressed page format. */ @@ -117,24 +109,24 @@ static const byte supremum_extra_data[] = { /** Assert that a block of memory is filled with zero bytes. Compare at most sizeof(field_ref_zero) bytes. -@param b in: memory block -@param s in: size of the memory block, in bytes */ -#define ASSERT_ZERO(b, s) \ - ut_ad(!memcmp(b, field_ref_zero, ut_min(s, sizeof field_ref_zero))) +@param b in: memory block +@param s in: size of the memory block, in bytes */ +#define ASSERT_ZERO(b, s) \ + ut_ad(!memcmp(b, field_ref_zero, \ + ut_min(static_cast<size_t>(s), sizeof field_ref_zero))); /** Assert that a BLOB pointer is filled with zero bytes. -@param b in: BLOB pointer */ +@param b in: BLOB pointer */ #define ASSERT_ZERO_BLOB(b) \ ut_ad(!memcmp(b, field_ref_zero, sizeof field_ref_zero)) /* Enable some extra debugging output. This code can be enabled independently of any UNIV_ debugging conditions. */ -#ifndef UNIV_INNOCHECKSUM #if defined UNIV_DEBUG || defined UNIV_ZIP_DEBUG # include <stdarg.h> MY_ATTRIBUTE((format (printf, 1, 2))) /**********************************************************************//** Report a failure to decompress or compress. -@return number of characters printed */ +@return number of characters printed */ static int page_zip_fail_func( @@ -154,21 +146,18 @@ page_zip_fail_func( return(res); } /** Wrapper for page_zip_fail_func() -@param fmt_args in: printf(3) format string and arguments */ +@param fmt_args in: printf(3) format string and arguments */ # define page_zip_fail(fmt_args) page_zip_fail_func fmt_args #else /* UNIV_DEBUG || UNIV_ZIP_DEBUG */ /** Dummy wrapper for page_zip_fail_func() -@param fmt_args ignored: printf(3) format string and arguments */ +@param fmt_args ignored: printf(3) format string and arguments */ # define page_zip_fail(fmt_args) /* empty */ #endif /* UNIV_DEBUG || UNIV_ZIP_DEBUG */ -#endif /* !UNIV_INNOCHECKSUM */ -#ifndef UNIV_INNOCHECKSUM #ifndef UNIV_HOTBACKUP /**********************************************************************//** Determine the guaranteed free space on an empty page. -@return minimum payload size on the page */ -UNIV_INTERN +@return minimum payload size on the page */ ulint page_zip_empty_size( /*================*/ @@ -179,8 +168,7 @@ page_zip_empty_size( /* subtract the page header and the longest uncompressed data needed for one record */ - (PAGE_DATA - + PAGE_ZIP_DIR_SLOT_SIZE - + DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN + + PAGE_ZIP_CLUST_LEAF_SLOT_SIZE + 1/* encoded heap_no==2 in page_zip_write_rec() */ + 1/* end of modification log */ - REC_N_NEW_EXTRA_BYTES/* omitted bytes */) @@ -193,7 +181,7 @@ page_zip_empty_size( /*************************************************************//** Gets the number of elements in the dense page directory, including deleted records (the free list). -@return number of elements in the dense page directory */ +@return number of elements in the dense page directory */ UNIV_INLINE ulint page_zip_dir_elems( @@ -207,7 +195,7 @@ page_zip_dir_elems( /*************************************************************//** Gets the size of the compressed page trailer (the dense page directory), including deleted records (the free list). -@return length of dense page directory, in bytes */ +@return length of dense page directory, in bytes */ UNIV_INLINE ulint page_zip_dir_size( @@ -220,7 +208,7 @@ page_zip_dir_size( /*************************************************************//** Gets an offset to the compressed page trailer (the dense page directory), including deleted records (the free list). -@return offset of the dense page directory */ +@return offset of the dense page directory */ UNIV_INLINE ulint page_zip_dir_start_offs( @@ -236,23 +224,23 @@ page_zip_dir_start_offs( /*************************************************************//** Gets a pointer to the compressed page trailer (the dense page directory), including deleted records (the free list). -@param[in] page_zip compressed page -@param[in] n_dense number of entries in the directory -@return pointer to the dense page directory */ +@param[in] page_zip compressed page +@param[in] n_dense number of entries in the directory +@return pointer to the dense page directory */ #define page_zip_dir_start_low(page_zip, n_dense) \ ((page_zip)->data + page_zip_dir_start_offs(page_zip, n_dense)) /*************************************************************//** Gets a pointer to the compressed page trailer (the dense page directory), including deleted records (the free list). -@param[in] page_zip compressed page -@return pointer to the dense page directory */ +@param[in] page_zip compressed page +@return pointer to the dense page directory */ #define page_zip_dir_start(page_zip) \ page_zip_dir_start_low(page_zip, page_zip_dir_elems(page_zip)) /*************************************************************//** Gets the size of the compressed page trailer (the dense page directory), only including user records (excluding the free list). -@return length of dense page directory comprising existing records, in bytes */ +@return length of dense page directory comprising existing records, in bytes */ UNIV_INLINE ulint page_zip_dir_user_size( @@ -267,7 +255,7 @@ page_zip_dir_user_size( /*************************************************************//** Find the slot of the given record in the dense page directory. -@return dense directory slot, or NULL if record not found */ +@return dense directory slot, or NULL if record not found */ UNIV_INLINE byte* page_zip_dir_find_low( @@ -290,7 +278,7 @@ page_zip_dir_find_low( /*************************************************************//** Find the slot of the given non-free record in the dense page directory. -@return dense directory slot, or NULL if record not found */ +@return dense directory slot, or NULL if record not found */ UNIV_INLINE byte* page_zip_dir_find( @@ -309,7 +297,7 @@ page_zip_dir_find( /*************************************************************//** Find the slot of the given free record in the dense page directory. -@return dense directory slot, or NULL if record not found */ +@return dense directory slot, or NULL if record not found */ UNIV_INLINE byte* page_zip_dir_find_free( @@ -461,7 +449,7 @@ page_zip_get_n_prev_extern( /**********************************************************************//** Encode the length of a fixed-length column. -@return buf + length of encoded val */ +@return buf + length of encoded val */ static byte* page_zip_fixed_field_encode( @@ -489,17 +477,19 @@ page_zip_fixed_field_encode( /**********************************************************************//** Write the index information for the compressed page. -@return used size of buf */ -static +@return used size of buf */ ulint page_zip_fields_encode( /*===================*/ - ulint n, /*!< in: number of fields to compress */ - dict_index_t* index, /*!< in: index comprising at least n fields */ - ulint trx_id_pos,/*!< in: position of the trx_id column - in the index, or ULINT_UNDEFINED if - this is a non-leaf page */ - byte* buf) /*!< out: buffer of (n + 1) * 2 bytes */ + ulint n, /*!< in: number of fields + to compress */ + const dict_index_t* index, /*!< in: index comprising + at least n fields */ + ulint trx_id_pos, + /*!< in: position of the trx_id column + in the index, or ULINT_UNDEFINED if + this is a non-leaf page */ + byte* buf) /*!< out: buffer of (n + 1) * 2 bytes */ { const byte* buf_start = buf; ulint i; @@ -525,8 +515,7 @@ page_zip_fields_encode( const dict_col_t* column = dict_field_get_col(field); - if (UNIV_UNLIKELY(column->len > 255) - || UNIV_UNLIKELY(column->mtype == DATA_BLOB)) { + if (DATA_BIG_COL(column)) { val |= 0x7e; /* max > 255 bytes */ } @@ -670,10 +659,10 @@ page_zip_dir_encode( ut_a(offs < UNIV_PAGE_SIZE - PAGE_DIR); ut_a(offs >= PAGE_ZIP_START); #if PAGE_ZIP_DIR_SLOT_MASK & (PAGE_ZIP_DIR_SLOT_MASK + 1) -# error "PAGE_ZIP_DIR_SLOT_MASK is not 1 less than a power of 2" +# error PAGE_ZIP_DIR_SLOT_MASK is not 1 less than a power of 2 #endif -#if PAGE_ZIP_DIR_SLOT_MASK < UNIV_PAGE_ZIP_SIZE_MAX - 1 -# error "PAGE_ZIP_DIR_SLOT_MASK < UNIV_ZIP_SIZE_MAX - 1" +#if PAGE_ZIP_DIR_SLOT_MASK < UNIV_ZIP_SIZE_MAX - 1 +# error PAGE_ZIP_DIR_SLOT_MASK < UNIV_ZIP_SIZE_MAX - 1 #endif if (UNIV_UNLIKELY(rec_get_n_owned_new(rec))) { offs |= PAGE_ZIP_DIR_SLOT_OWNED; @@ -761,7 +750,6 @@ page_zip_free( /**********************************************************************//** Configure the zlib allocator to use the given memory heap. */ -UNIV_INTERN void page_zip_set_alloc( /*===============*/ @@ -783,16 +771,16 @@ page_zip_set_alloc( #ifdef PAGE_ZIP_COMPRESS_DBG /** Set this variable in a debugger to enable excessive logging in page_zip_compress(). */ -UNIV_INTERN ibool page_zip_compress_dbg; +ibool page_zip_compress_dbg; /** Set this variable in a debugger to enable binary logging of the data passed to deflate(). When this variable is nonzero, it will act as a log file name generator. */ -UNIV_INTERN unsigned page_zip_compress_log; +unsigned page_zip_compress_log; /**********************************************************************//** Wrapper for deflate(). Log the operation if page_zip_compress_dbg is set. -@return deflate() status: Z_OK, Z_BUF_ERROR, ... */ +@return deflate() status: Z_OK, Z_BUF_ERROR, ... */ static int page_zip_compress_deflate( @@ -806,7 +794,10 @@ page_zip_compress_deflate( ut_print_buf(stderr, strm->next_in, strm->avail_in); } if (UNIV_LIKELY_NULL(logfile)) { - fwrite(strm->next_in, 1, strm->avail_in, logfile); + if (fwrite(strm->next_in, 1, strm->avail_in, logfile) + != strm->avail_in) { + perror("fwrite"); + } } status = deflate(strm, flush); if (UNIV_UNLIKELY(page_zip_compress_dbg)) { @@ -819,9 +810,9 @@ page_zip_compress_deflate( # undef deflate /** Debug wrapper for the zlib compression routine deflate(). Log the operation if page_zip_compress_dbg is set. -@param strm in/out: compressed stream -@param flush in: flushing method -@return deflate() status: Z_OK, Z_BUF_ERROR, ... */ +@param strm in/out: compressed stream +@param flush in: flushing method +@return deflate() status: Z_OK, Z_BUF_ERROR, ... */ # define deflate(strm, flush) page_zip_compress_deflate(logfile, strm, flush) /** Declaration of the logfile parameter */ # define FILE_LOGFILE FILE* logfile, @@ -836,7 +827,7 @@ Log the operation if page_zip_compress_dbg is set. /**********************************************************************//** Compress the records of a node pointer page. -@return Z_OK, or a zlib error code */ +@return Z_OK, or a zlib error code */ static int page_zip_compress_node_ptrs( @@ -902,7 +893,7 @@ page_zip_compress_node_ptrs( /**********************************************************************//** Compress the records of a leaf node of a secondary index. -@return Z_OK, or a zlib error code */ +@return Z_OK, or a zlib error code */ static int page_zip_compress_sec( @@ -948,7 +939,7 @@ page_zip_compress_sec( /**********************************************************************//** Compress a record of a leaf node of a clustered index that contains externally stored columns. -@return Z_OK, or a zlib error code */ +@return Z_OK, or a zlib error code */ static int page_zip_compress_clust_ext( @@ -1075,7 +1066,7 @@ page_zip_compress_clust_ext( /**********************************************************************//** Compress the records of a leaf node of a clustered index. -@return Z_OK, or a zlib error code */ +@return Z_OK, or a zlib error code */ static int page_zip_compress_clust( @@ -1205,54 +1196,68 @@ page_zip_compress_clust( } while (--n_dense); func_exit: - return(err); -} + return(err);} /**********************************************************************//** Compress a page. @return TRUE on success, FALSE on failure; page_zip will be left intact on failure. */ -UNIV_INTERN ibool page_zip_compress( /*==============*/ - page_zip_des_t* page_zip,/*!< in: size; out: data, n_blobs, - m_start, m_end, m_nonempty */ - const page_t* page, /*!< in: uncompressed page */ - dict_index_t* index, /*!< in: index of the B-tree node */ - ulint level, /*!< in: compression level */ - mtr_t* mtr) /*!< in: mini-transaction, or NULL */ + page_zip_des_t* page_zip, /*!< in: size; out: data, + n_blobs, m_start, m_end, + m_nonempty */ + const page_t* page, /*!< in: uncompressed page */ + dict_index_t* index, /*!< in: index of the B-tree + node */ + ulint level, /*!< in: commpression level */ + const redo_page_compress_t* page_comp_info, + /*!< in: used for applying + TRUNCATE log + record during recovery */ + mtr_t* mtr) /*!< in/out: mini-transaction, + or NULL */ { - z_stream c_stream; - int err; - ulint n_fields;/* number of index fields needed */ - byte* fields; /*!< index field information */ - byte* buf; /*!< compressed payload of the page */ - byte* buf_end;/* end of buf */ - ulint n_dense; - ulint slot_size;/* amount of uncompressed bytes per record */ - const rec_t** recs; /*!< dense page directory, sorted by address */ - mem_heap_t* heap; - ulint trx_id_col; - ulint n_blobs = 0; - byte* storage;/* storage of uncompressed columns */ + z_stream c_stream; + int err; + ulint n_fields; /* number of index fields + needed */ + byte* fields; /*!< index field information */ + byte* buf; /*!< compressed payload of the + page */ + byte* buf_end; /* end of buf */ + ulint n_dense; + ulint slot_size; /* amount of uncompressed bytes + per record */ + const rec_t** recs; /*!< dense page directory, + sorted by address */ + mem_heap_t* heap; + ulint trx_id_col = ULINT_UNDEFINED; + ulint n_blobs = 0; + byte* storage; /* storage of uncompressed + columns */ + index_id_t ind_id; #ifndef UNIV_HOTBACKUP - ullint usec = ut_time_us(NULL); + uintmax_t usec = ut_time_us(NULL); #endif /* !UNIV_HOTBACKUP */ #ifdef PAGE_ZIP_COMPRESS_DBG - FILE* logfile = NULL; + FILE* logfile = NULL; #endif /* A local copy of srv_cmp_per_index_enabled to avoid reading that variable multiple times in this function since it can be changed at anytime. */ - my_bool cmp_per_index_enabled = srv_cmp_per_index_enabled; + my_bool cmp_per_index_enabled; + cmp_per_index_enabled = srv_cmp_per_index_enabled; ut_a(page_is_comp(page)); - ut_a(fil_page_get_type(page) == FIL_PAGE_INDEX); + ut_a(fil_page_index_page_check(page)); ut_ad(page_simple_validate_new((page_t*) page)); ut_ad(page_zip_simple_validate(page_zip)); - ut_ad(dict_table_is_comp(index->table)); - ut_ad(!dict_index_is_ibuf(index)); + ut_ad(!index + || (index + && dict_table_is_comp(index->table) + && !dict_index_is_ibuf(index))); UNIV_MEM_ASSERT_RW(page, UNIV_PAGE_SIZE); @@ -1272,21 +1277,30 @@ page_zip_compress( == PAGE_NEW_SUPREMUM); } - if (page_is_leaf(page)) { - n_fields = dict_index_get_n_fields(index); + if (truncate_t::s_fix_up_active) { + ut_ad(page_comp_info != NULL); + n_fields = page_comp_info->n_fields; + ind_id = page_comp_info->index_id; } else { - n_fields = dict_index_get_n_unique_in_tree(index); + if (page_is_leaf(page)) { + n_fields = dict_index_get_n_fields(index); + } else { + n_fields = dict_index_get_n_unique_in_tree_nonleaf(index); + } + ind_id = index->id; } /* The dense directory excludes the infimum and supremum records. */ n_dense = page_dir_get_n_heap(page) - PAGE_HEAP_NO_USER_LOW; #ifdef PAGE_ZIP_COMPRESS_DBG if (UNIV_UNLIKELY(page_zip_compress_dbg)) { - fprintf(stderr, "compress %p %p %lu %lu %lu\n", - (void*) page_zip, (void*) page, - (ibool) page_is_leaf(page), - n_fields, n_dense); + ib::info() << "compress " + << static_cast<void*>(page_zip) << " " + << static_cast<const void*>(page) << " " + << page_is_leaf(page) << " " + << n_fields << " " << n_dense; } + if (UNIV_UNLIKELY(page_zip_compress_log)) { /* Create a log file for every compression attempt. */ char logfilename[9]; @@ -1296,7 +1310,10 @@ page_zip_compress( if (logfile) { /* Write the uncompressed page to the log. */ - fwrite(page, 1, UNIV_PAGE_SIZE, logfile); + if (fwrite(page, 1, UNIV_PAGE_SIZE, logfile) + != UNIV_PAGE_SIZE) { + perror("fwrite"); + } /* Record the compressed size as zero. This will be overwritten at successful exit. */ putc(0, logfile); @@ -1310,7 +1327,7 @@ page_zip_compress( page_zip_stat[page_zip->ssize - 1].compressed++; if (cmp_per_index_enabled) { mutex_enter(&page_zip_stat_per_index_mutex); - page_zip_stat_per_index[index->id].compressed++; + page_zip_stat_per_index[ind_id].compressed++; mutex_exit(&page_zip_stat_per_index_mutex); } #endif /* !UNIV_HOTBACKUP */ @@ -1334,13 +1351,17 @@ page_zip_compress( && strcasecmp(index->table_name, "IBUF_DUMMY") != 0) { #ifdef UNIV_DEBUG - fprintf(stderr, - "InnoDB: Simulating a compression failure" - " for table %s, index %s, page %lu (%s)\n", - index->table_name, - index->name, - page_get_page_no(page), - page_is_leaf(page) ? "leaf" : "non-leaf"); + ib::error() + << "InnoDB: Simulating a compression failure" + << " for table " + << (index->table->name.m_name) + << " index " + << index->name() + << " page " + << page_get_page_no(page) + << "(" + << (page_is_leaf(page) ? "leaf" : "non-leaf") + << ")"; #endif @@ -1374,25 +1395,38 @@ page_zip_compress( ut_a(err == Z_OK); c_stream.next_out = buf; + /* Subtract the space reserved for uncompressed data. */ /* Page header and the end marker of the modification log */ c_stream.avail_out = static_cast<uInt>(buf_end - buf - 1); /* Dense page directory and uncompressed columns, if any */ if (page_is_leaf(page)) { - if (dict_index_is_clust(index)) { - trx_id_col = dict_index_get_sys_col_pos( - index, DATA_TRX_ID); - ut_ad(trx_id_col > 0); - ut_ad(trx_id_col != ULINT_UNDEFINED); + if ((index && dict_index_is_clust(index)) + || (page_comp_info + && (page_comp_info->type & DICT_CLUSTERED))) { + + if (index) { + trx_id_col = dict_index_get_sys_col_pos( + index, DATA_TRX_ID); + ut_ad(trx_id_col > 0); + ut_ad(trx_id_col != ULINT_UNDEFINED); + } else if (page_comp_info + && (page_comp_info->type + & DICT_CLUSTERED)) { + trx_id_col = page_comp_info->trx_id_pos; + } slot_size = PAGE_ZIP_DIR_SLOT_SIZE + DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN; + } else { /* Signal the absence of trx_id in page_zip_fields_encode() */ - ut_ad(dict_index_get_sys_col_pos(index, DATA_TRX_ID) - == ULINT_UNDEFINED); + if (index) { + ut_ad(dict_index_get_sys_col_pos( + index, DATA_TRX_ID) == ULINT_UNDEFINED); + } trx_id_col = 0; slot_size = PAGE_ZIP_DIR_SLOT_SIZE; } @@ -1407,9 +1441,20 @@ page_zip_compress( } c_stream.avail_out -= static_cast<uInt>(n_dense * slot_size); - c_stream.avail_in = static_cast<uInt>( - page_zip_fields_encode(n_fields, index, trx_id_col, fields)); + if (truncate_t::s_fix_up_active) { + ut_ad(page_comp_info != NULL); + c_stream.avail_in = static_cast<uInt>( + page_comp_info->field_len); + for (ulint i = 0; i < page_comp_info->field_len; i++) { + fields[i] = page_comp_info->fields[i]; + } + } else { + c_stream.avail_in = static_cast<uInt>( + page_zip_fields_encode( + n_fields, index, trx_id_col, fields)); + } c_stream.next_in = fields; + if (UNIV_LIKELY(!trx_id_col)) { trx_id_col = ULINT_UNDEFINED; } @@ -1482,16 +1527,16 @@ err_exit: } #endif /* PAGE_ZIP_COMPRESS_DBG */ #ifndef UNIV_HOTBACKUP - if (page_is_leaf(page)) { + if (page_is_leaf(page) && index) { dict_index_zip_failure(index); } - ullint time_diff = ut_time_us(NULL) - usec; + uintmax_t time_diff = ut_time_us(NULL) - usec; page_zip_stat[page_zip->ssize - 1].compressed_usec += time_diff; if (cmp_per_index_enabled) { mutex_enter(&page_zip_stat_per_index_mutex); - page_zip_stat_per_index[index->id].compressed_usec + page_zip_stat_per_index[ind_id].compressed_usec += time_diff; mutex_exit(&page_zip_stat_per_index_mutex); } @@ -1549,22 +1594,24 @@ err_exit: byte sz[4]; mach_write_to_4(sz, c_stream.total_out); fseek(logfile, UNIV_PAGE_SIZE, SEEK_SET); - fwrite(sz, 1, sizeof sz, logfile); + if (fwrite(sz, 1, sizeof sz, logfile) != sizeof sz) { + perror("fwrite"); + } fclose(logfile); } #endif /* PAGE_ZIP_COMPRESS_DBG */ #ifndef UNIV_HOTBACKUP - ullint time_diff = ut_time_us(NULL) - usec; + uintmax_t time_diff = ut_time_us(NULL) - usec; page_zip_stat[page_zip->ssize - 1].compressed_ok++; page_zip_stat[page_zip->ssize - 1].compressed_usec += time_diff; if (cmp_per_index_enabled) { mutex_enter(&page_zip_stat_per_index_mutex); - page_zip_stat_per_index[index->id].compressed_ok++; - page_zip_stat_per_index[index->id].compressed_usec += time_diff; + page_zip_stat_per_index[ind_id].compressed_ok++; + page_zip_stat_per_index[ind_id].compressed_usec += time_diff; mutex_exit(&page_zip_stat_per_index_mutex); } - if (page_is_leaf(page)) { + if (page_is_leaf(page) && !truncate_t::s_fix_up_active) { dict_index_zip_success(index); } #endif /* !UNIV_HOTBACKUP */ @@ -1573,34 +1620,6 @@ err_exit: } /**********************************************************************//** -Compare two page directory entries. -@return positive if rec1 > rec2 */ -UNIV_INLINE -ibool -page_zip_dir_cmp( -/*=============*/ - const rec_t* rec1, /*!< in: rec1 */ - const rec_t* rec2) /*!< in: rec2 */ -{ - return(rec1 > rec2); -} - -/**********************************************************************//** -Sort the dense page directory by address (heap_no). */ -static -void -page_zip_dir_sort( -/*==============*/ - rec_t** arr, /*!< in/out: dense page directory */ - rec_t** aux_arr,/*!< in/out: work area */ - ulint low, /*!< in: lower bound of the sorting area, inclusive */ - ulint high) /*!< in: upper bound of the sorting area, exclusive */ -{ - UT_SORT_FUNCTION_BODY(page_zip_dir_sort, arr, aux_arr, low, high, - page_zip_dir_cmp); -} - -/**********************************************************************//** Deallocate the index information initialized by page_zip_fields_decode(). */ static void @@ -1619,16 +1638,17 @@ page_zip_fields_free( /**********************************************************************//** Read the index information for the compressed page. -@return own: dummy index describing the page, or NULL on error */ +@return own: dummy index describing the page, or NULL on error */ static dict_index_t* page_zip_fields_decode( /*===================*/ const byte* buf, /*!< in: index information */ const byte* end, /*!< in: end of buf */ - ulint* trx_id_col)/*!< in: NULL for non-leaf pages; + ulint* trx_id_col,/*!< in: NULL for non-leaf pages; for leaf pages, pointer to where to store the position of the trx_id column */ + bool is_spatial)/*< in: is spatial index or not */ { const byte* b; ulint n; @@ -1660,7 +1680,7 @@ page_zip_fields_decode( return(NULL); } - table = dict_mem_table_create("ZIP_DUMMY", DICT_HDR_SPACE, n, + table = dict_mem_table_create("ZIP_DUMMY", DICT_HDR_SPACE, n, 0, DICT_TF_COMPACT, 0); index = dict_mem_index_create("ZIP_DUMMY", "ZIP_DUMMY", DICT_HDR_SPACE, 0, n); @@ -1730,13 +1750,17 @@ page_zip_fields_decode( ut_ad(b == end); + if (is_spatial) { + index->type |= DICT_SPATIAL; + } + return(index); } /**********************************************************************//** Populate the sparse page directory from the dense directory. -@return TRUE on success, FALSE on failure */ -static +@return TRUE on success, FALSE on failure */ +static __attribute__((nonnull, warn_unused_result)) ibool page_zip_dir_decode( /*================*/ @@ -1747,9 +1771,8 @@ page_zip_dir_decode( filled in */ rec_t** recs, /*!< out: dense page directory sorted by ascending address (and heap_no) */ - rec_t** recs_aux,/*!< in/out: scratch area */ ulint n_dense)/*!< in: number of user records, and - size of recs[] and recs_aux[] */ + size of recs[] */ { ulint i; ulint n_recs; @@ -1824,15 +1847,13 @@ page_zip_dir_decode( recs[i] = page + offs; } - if (UNIV_LIKELY(n_dense > 1)) { - page_zip_dir_sort(recs, recs_aux, 0, n_dense); - } + std::sort(recs, recs + n_dense); return(TRUE); } /**********************************************************************//** Initialize the REC_N_NEW_EXTRA_BYTES of each record. -@return TRUE on success, FALSE on failure */ +@return TRUE on success, FALSE on failure */ static ibool page_zip_set_extra_bytes( @@ -1930,7 +1951,7 @@ page_zip_set_extra_bytes( /**********************************************************************//** Apply the modification log to a record containing externally stored columns. Do not copy the fields that are stored separately. -@return pointer to modification log, or NULL on failure */ +@return pointer to modification log, or NULL on failure */ static const byte* page_zip_apply_log_ext( @@ -1985,8 +2006,8 @@ page_zip_apply_log_ext( - BTR_EXTERN_FIELD_REF_SIZE; if (UNIV_UNLIKELY(data + len >= end)) { - page_zip_fail(("page_zip_apply_log_ext: " - "ext %p+%lu >= %p\n", + page_zip_fail(("page_zip_apply_log_ext:" + " ext %p+%lu >= %p\n", (const void*) data, (ulong) len, (const void*) end)); @@ -2003,8 +2024,8 @@ page_zip_apply_log_ext( /* Copy the last bytes of the record. */ len = rec_get_end(rec, offsets) - next_out; if (UNIV_UNLIKELY(data + len >= end)) { - page_zip_fail(("page_zip_apply_log_ext: " - "last %p+%lu >= %p\n", + page_zip_fail(("page_zip_apply_log_ext:" + " last %p+%lu >= %p\n", (const void*) data, (ulong) len, (const void*) end)); @@ -2019,7 +2040,7 @@ page_zip_apply_log_ext( /**********************************************************************//** Apply the modification log to an uncompressed page. Do not copy the fields that are stored separately. -@return pointer to end of modification log, or NULL on failure */ +@return pointer to end of modification log, or NULL on failure */ static const byte* page_zip_apply_log( @@ -2137,8 +2158,8 @@ page_zip_apply_log( /* Non-leaf nodes should not contain any externally stored columns. */ if (UNIV_UNLIKELY(hs & REC_STATUS_NODE_PTR)) { - page_zip_fail(("page_zip_apply_log: " - "%lu&REC_STATUS_NODE_PTR\n", + page_zip_fail(("page_zip_apply_log:" + " %lu&REC_STATUS_NODE_PTR\n", (ulong) hs)); return(NULL); } @@ -2154,8 +2175,8 @@ page_zip_apply_log( - REC_NODE_PTR_SIZE; /* Copy the data bytes, except node_ptr. */ if (UNIV_UNLIKELY(data + len >= end)) { - page_zip_fail(("page_zip_apply_log: " - "node_ptr %p+%lu >= %p\n", + page_zip_fail(("page_zip_apply_log:" + " node_ptr %p+%lu >= %p\n", (const void*) data, (ulong) len, (const void*) end)); @@ -2169,8 +2190,8 @@ page_zip_apply_log( /* Copy all data bytes of a record in a secondary index. */ if (UNIV_UNLIKELY(data + len >= end)) { - page_zip_fail(("page_zip_apply_log: " - "sec %p+%lu >= %p\n", + page_zip_fail(("page_zip_apply_log:" + " sec %p+%lu >= %p\n", (const void*) data, (ulong) len, (const void*) end)); @@ -2188,8 +2209,8 @@ page_zip_apply_log( if (UNIV_UNLIKELY(data + l >= end) || UNIV_UNLIKELY(len < (DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN))) { - page_zip_fail(("page_zip_apply_log: " - "trx_id %p+%lu >= %p\n", + page_zip_fail(("page_zip_apply_log:" + " trx_id %p+%lu >= %p\n", (const void*) data, (ulong) l, (const void*) end)); @@ -2204,8 +2225,8 @@ page_zip_apply_log( b = rec + l + (DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN); len = rec_get_end(rec, offsets) - b; if (UNIV_UNLIKELY(data + len >= end)) { - page_zip_fail(("page_zip_apply_log: " - "clust %p+%lu >= %p\n", + page_zip_fail(("page_zip_apply_log:" + " clust %p+%lu >= %p\n", (const void*) data, (ulong) len, (const void*) end)); @@ -2220,7 +2241,7 @@ page_zip_apply_log( /**********************************************************************//** Set the heap_no in a record, and skip the fixed-size record header that is not included in the d_stream. -@return TRUE on success, FALSE if d_stream does not end at rec */ +@return TRUE on success, FALSE if d_stream does not end at rec */ static ibool page_zip_decompress_heap_no( @@ -2245,7 +2266,7 @@ page_zip_decompress_heap_no( /**********************************************************************//** Decompress the records of a node pointer page. -@return TRUE on success, FALSE on failure */ +@return TRUE on success, FALSE on failure */ static ibool page_zip_decompress_node_ptrs( @@ -2434,7 +2455,7 @@ zlib_done: /**********************************************************************//** Decompress the records of a leaf node of a secondary index. -@return TRUE on success, FALSE on failure */ +@return TRUE on success, FALSE on failure */ static ibool page_zip_decompress_sec( @@ -2572,7 +2593,7 @@ zlib_done: /**********************************************************************//** Decompress a record of a leaf node of a clustered index that contains externally stored columns. -@return TRUE on success */ +@return TRUE on success */ static ibool page_zip_decompress_clust_ext( @@ -2682,7 +2703,7 @@ page_zip_decompress_clust_ext( /**********************************************************************//** Compress the records of a leaf node of a clustered index. -@return TRUE on success, FALSE on failure */ +@return TRUE on success, FALSE on failure */ static ibool page_zip_decompress_clust( @@ -2708,9 +2729,7 @@ page_zip_decompress_clust( /* Subtract the space reserved for uncompressed data. */ d_stream->avail_in -= static_cast<uInt>(n_dense) - * (PAGE_ZIP_DIR_SLOT_SIZE - + DATA_TRX_ID_LEN - + DATA_ROLL_PTR_LEN); + * (PAGE_ZIP_CLUST_LEAF_SLOT_SIZE); /* Decompress the records in heap_no order. */ for (slot = 0; slot < n_dense; slot++) { @@ -2952,8 +2971,8 @@ zlib_done: (externs < page_zip->data + page_zip->m_end)) { page_zip_fail(("page_zip_" - "decompress_clust: " - "%p < %p + %lu\n", + "decompress_clust:" + " %p < %p + %lu\n", (const void*) externs, (const void*) page_zip->data, @@ -2982,11 +3001,11 @@ zlib_done: Decompress a page. This function should tolerate errors on the compressed page. Instead of letting assertions fail, it will return FALSE if an inconsistency is detected. -@return TRUE on success, FALSE on failure */ -UNIV_INTERN +@return TRUE on success, FALSE on failure */ +static ibool -page_zip_decompress( -/*================*/ +page_zip_decompress_low( +/*====================*/ page_zip_des_t* page_zip,/*!< in: data, ssize; out: m_start, m_end, m_nonempty, n_blobs */ page_t* page, /*!< out: uncompressed page, may be trashed */ @@ -3002,9 +3021,6 @@ page_zip_decompress( ulint trx_id_col = ULINT_UNDEFINED; mem_heap_t* heap; ulint* offsets; -#ifndef UNIV_HOTBACKUP - ullint usec = ut_time_us(NULL); -#endif /* !UNIV_HOTBACKUP */ ut_ad(page_zip_simple_validate(page_zip)); UNIV_MEM_ASSERT_W(page, UNIV_PAGE_SIZE); @@ -3023,7 +3039,7 @@ page_zip_decompress( heap = mem_heap_create(n_dense * (3 * sizeof *recs) + UNIV_PAGE_SIZE); recs = static_cast<rec_t**>( - mem_heap_alloc(heap, n_dense * (2 * sizeof *recs))); + mem_heap_alloc(heap, n_dense * sizeof *recs)); if (all) { /* Copy the page header. */ @@ -3058,7 +3074,7 @@ page_zip_decompress( /* Copy the page directory. */ if (UNIV_UNLIKELY(!page_zip_dir_decode(page_zip, page, recs, - recs + n_dense, n_dense))) { + n_dense))) { zlib_error: mem_heap_free(heap); return(FALSE); @@ -3111,7 +3127,8 @@ zlib_error: index = page_zip_fields_decode( page + PAGE_ZIP_START, d_stream.next_out, - page_is_leaf(page) ? &trx_id_col : NULL); + page_is_leaf(page) ? &trx_id_col : NULL, + fil_page_get_type(page) == FIL_PAGE_RTREE); if (UNIV_UNLIKELY(!index)) { @@ -3188,8 +3205,36 @@ err_exit: page_zip_fields_free(index); mem_heap_free(heap); + + return(TRUE); +} + +/**********************************************************************//** +Decompress a page. This function should tolerate errors on the compressed +page. Instead of letting assertions fail, it will return FALSE if an +inconsistency is detected. +@return TRUE on success, FALSE on failure */ +ibool +page_zip_decompress( +/*================*/ + page_zip_des_t* page_zip,/*!< in: data, ssize; + out: m_start, m_end, m_nonempty, n_blobs */ + page_t* page, /*!< out: uncompressed page, may be trashed */ + ibool all) /*!< in: TRUE=decompress the whole page; + FALSE=verify but do not copy some + page header fields that should not change + after page creation */ +{ #ifndef UNIV_HOTBACKUP - ullint time_diff = ut_time_us(NULL) - usec; + uintmax_t usec = ut_time_us(NULL); +#endif /* !UNIV_HOTBACKUP */ + + if (!page_zip_decompress_low(page_zip, page, all)) { + return(FALSE); + } + +#ifndef UNIV_HOTBACKUP + uintmax_t time_diff = ut_time_us(NULL) - usec; page_zip_stat[page_zip->ssize - 1].decompressed++; page_zip_stat[page_zip->ssize - 1].decompressed_usec += time_diff; @@ -3244,17 +3289,16 @@ page_zip_hexdump_func( } /** Dump a block of memory on the standard error stream. -@param buf in: data -@param size in: length of the data, in bytes */ +@param buf in: data +@param size in: length of the data, in bytes */ #define page_zip_hexdump(buf, size) page_zip_hexdump_func(#buf, buf, size) /** Flag: make page_zip_validate() compare page headers only */ -UNIV_INTERN ibool page_zip_validate_header_only = FALSE; +ibool page_zip_validate_header_only = FALSE; /**********************************************************************//** Check that the compressed and decompressed pages match. -@return TRUE if valid, FALSE if not */ -UNIV_INTERN +@return TRUE if valid, FALSE if not */ ibool page_zip_validate_low( /*==================*/ @@ -3289,14 +3333,14 @@ page_zip_validate_low( /* page_zip_decompress() expects the uncompressed page to be UNIV_PAGE_SIZE aligned. */ - temp_page_buf = static_cast<byte*>(ut_malloc(2 * UNIV_PAGE_SIZE)); + temp_page_buf = static_cast<byte*>(ut_malloc_nokey(2 * UNIV_PAGE_SIZE)); temp_page = static_cast<byte*>(ut_align(temp_page_buf, UNIV_PAGE_SIZE)); UNIV_MEM_ASSERT_RW(page, UNIV_PAGE_SIZE); UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); temp_page_zip = *page_zip; - valid = page_zip_decompress(&temp_page_zip, temp_page, TRUE); + valid = page_zip_decompress_low(&temp_page_zip, temp_page, TRUE); if (!valid) { fputs("page_zip_validate(): failed to decompress\n", stderr); goto func_exit; @@ -3354,15 +3398,23 @@ page_zip_validate_low( /* Only the minimum record flag differed. Let us ignore it. */ - page_zip_fail(("page_zip_validate: " - "min_rec_flag " - "(%s" - "%lu,%lu,0x%02lx)\n", + page_zip_fail(("page_zip_validate:" + " min_rec_flag" + " (%s%lu,%lu,0x%02lx)\n", sloppy ? "ignored, " : "", page_get_space_id(page), page_get_page_no(page), (ulong) page[offset])); - valid = sloppy; + /* We don't check for spatial index, since + the "minimum record" could be deleted when + doing rtr_update_mbr_field. + GIS_FIXME: need to validate why + rtr_update_mbr_field.() could affect this */ + if (index && dict_index_is_spatial(index)) { + valid = true; + } else { + valid = sloppy; + } goto func_exit; } } @@ -3373,8 +3425,8 @@ page_zip_validate_low( while (rec || trec) { if (page_offset(rec) != page_offset(trec)) { - page_zip_fail(("page_zip_validate: " - "PAGE_FREE list: %u!=%u\n", + page_zip_fail(("page_zip_validate:" + " PAGE_FREE list: %u!=%u\n", (unsigned) page_offset(rec), (unsigned) page_offset(trec))); valid = FALSE; @@ -3395,8 +3447,8 @@ page_zip_validate_low( do { if (page_offset(rec) != page_offset(trec)) { - page_zip_fail(("page_zip_validate: " - "record list: 0x%02x!=0x%02x\n", + page_zip_fail(("page_zip_validate:" + " record list: 0x%02x!=0x%02x\n", (unsigned) page_offset(rec), (unsigned) page_offset(trec))); valid = FALSE; @@ -3413,8 +3465,8 @@ page_zip_validate_low( trec - rec_offs_extra_size(offsets), rec_offs_size(offsets))) { page_zip_fail( - ("page_zip_validate: " - "record content: 0x%02x", + ("page_zip_validate:" + " record content: 0x%02x", (unsigned) page_offset(rec))); valid = FALSE; break; @@ -3443,8 +3495,7 @@ func_exit: /**********************************************************************//** Check that the compressed and decompressed pages match. -@return TRUE if valid, FALSE if not */ -UNIV_INTERN +@return TRUE if valid, FALSE if not */ ibool page_zip_validate( /*==============*/ @@ -3460,7 +3511,7 @@ page_zip_validate( #ifdef UNIV_DEBUG /**********************************************************************//** Assert that the compressed and decompressed page headers match. -@return TRUE */ +@return TRUE */ static ibool page_zip_header_cmp( @@ -3482,7 +3533,7 @@ page_zip_header_cmp( /**********************************************************************//** Write a record on the compressed page that contains externally stored columns. The data must already have been written to the uncompressed page. -@return end of modification log */ +@return end of modification log */ static byte* page_zip_write_rec_ext( @@ -3604,7 +3655,6 @@ page_zip_write_rec_ext( /**********************************************************************//** Write an entire record on the compressed page. The data must already have been written to the uncompressed page. */ -UNIV_INTERN void page_zip_write_rec( /*===============*/ @@ -3789,8 +3839,7 @@ page_zip_write_rec( /***********************************************************//** Parses a log record of writing a BLOB pointer of a record. -@return end of log record or NULL */ -UNIV_INTERN +@return end of log record or NULL */ byte* page_zip_parse_write_blob_ptr( /*==========================*/ @@ -3802,6 +3851,8 @@ page_zip_parse_write_blob_ptr( ulint offset; ulint z_offset; + ut_ad(ptr != NULL); + ut_ad(end_ptr != NULL); ut_ad(!page == !page_zip); if (UNIV_UNLIKELY @@ -3813,9 +3864,9 @@ page_zip_parse_write_blob_ptr( offset = mach_read_from_2(ptr); z_offset = mach_read_from_2(ptr + 2); - if (UNIV_UNLIKELY(offset < PAGE_ZIP_START) - || UNIV_UNLIKELY(offset >= UNIV_PAGE_SIZE) - || UNIV_UNLIKELY(z_offset >= UNIV_PAGE_SIZE)) { + if (offset < PAGE_ZIP_START + || offset >= UNIV_PAGE_SIZE + || z_offset >= UNIV_PAGE_SIZE) { corrupt: recv_sys->found_corrupt_log = TRUE; @@ -3823,8 +3874,8 @@ corrupt: } if (page) { - if (UNIV_UNLIKELY(!page_zip) - || UNIV_UNLIKELY(!page_is_leaf(page))) { + + if (!page_zip || !page_is_leaf(page)) { goto corrupt; } @@ -3849,7 +3900,6 @@ corrupt: /**********************************************************************//** Write a BLOB pointer of a record on the leaf page of a clustered index. The information must already have been updated on the uncompressed page. */ -UNIV_INTERN void page_zip_write_blob_ptr( /*====================*/ @@ -3868,6 +3918,10 @@ page_zip_write_blob_ptr( ulint blob_no; ulint len; + ut_ad(page_zip != NULL); + ut_ad(rec != NULL); + ut_ad(index != NULL); + ut_ad(offsets != NULL); ut_ad(PAGE_ZIP_MATCH(rec, page_zip)); ut_ad(page_simple_validate_new((page_t*) page)); ut_ad(page_zip_simple_validate(page_zip)); @@ -3895,8 +3949,7 @@ page_zip_write_blob_ptr( externs = page_zip->data + page_zip_get_size(page_zip) - (page_dir_get_n_heap(page) - PAGE_HEAP_NO_USER_LOW) - * (PAGE_ZIP_DIR_SLOT_SIZE - + DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN); + * PAGE_ZIP_CLUST_LEAF_SLOT_SIZE; field = rec_get_nth_field(rec, offsets, n, &len); @@ -3932,8 +3985,7 @@ page_zip_write_blob_ptr( /***********************************************************//** Parses a log record of writing the node pointer of a record. -@return end of log record or NULL */ -UNIV_INTERN +@return end of log record or NULL */ byte* page_zip_parse_write_node_ptr( /*==========================*/ @@ -3945,6 +3997,8 @@ page_zip_parse_write_node_ptr( ulint offset; ulint z_offset; + ut_ad(ptr != NULL); + ut_ad(end_ptr!= NULL); ut_ad(!page == !page_zip); if (UNIV_UNLIKELY(end_ptr < ptr + (2 + 2 + REC_NODE_PTR_SIZE))) { @@ -3955,9 +4009,9 @@ page_zip_parse_write_node_ptr( offset = mach_read_from_2(ptr); z_offset = mach_read_from_2(ptr + 2); - if (UNIV_UNLIKELY(offset < PAGE_ZIP_START) - || UNIV_UNLIKELY(offset >= UNIV_PAGE_SIZE) - || UNIV_UNLIKELY(z_offset >= UNIV_PAGE_SIZE)) { + if (offset < PAGE_ZIP_START + || offset >= UNIV_PAGE_SIZE + || z_offset >= UNIV_PAGE_SIZE) { corrupt: recv_sys->found_corrupt_log = TRUE; @@ -3970,8 +4024,7 @@ corrupt: byte* storage; ulint heap_no; - if (UNIV_UNLIKELY(!page_zip) - || UNIV_UNLIKELY(page_is_leaf(page))) { + if (!page_zip || page_is_leaf(page)) { goto corrupt; } @@ -4007,7 +4060,6 @@ corrupt: /**********************************************************************//** Write the node pointer of a record on a non-leaf compressed page. */ -UNIV_INTERN void page_zip_write_node_ptr( /*====================*/ @@ -4074,7 +4126,6 @@ page_zip_write_node_ptr( /**********************************************************************//** Write the trx_id and roll_ptr of a record on a B-tree leaf node page. */ -UNIV_INTERN void page_zip_write_trx_id_and_roll_ptr( /*===============================*/ @@ -4176,7 +4227,7 @@ page_zip_clear_rec( there is an array of node_ptr immediately before the dense page directory, at the very end of the page. */ storage = page_zip_dir_start(page_zip); - ut_ad(dict_index_get_n_unique_in_tree(index) == + ut_ad(dict_index_get_n_unique_in_tree_nonleaf(index) == rec_offs_n_fields(offsets) - 1); field = rec_get_nth_field(rec, offsets, rec_offs_n_fields(offsets) - 1, @@ -4233,7 +4284,6 @@ page_zip_clear_rec( /**********************************************************************//** Write the "deleted" flag of a record on a compressed page. The flag must already have been written on the uncompressed page. */ -UNIV_INTERN void page_zip_rec_set_deleted( /*=====================*/ @@ -4257,7 +4307,6 @@ page_zip_rec_set_deleted( /**********************************************************************//** Write the "owned" flag of a record on a compressed page. The n_owned field must already have been written on the uncompressed page. */ -UNIV_INTERN void page_zip_rec_set_owned( /*===================*/ @@ -4277,7 +4326,6 @@ page_zip_rec_set_owned( /**********************************************************************//** Insert a record to the dense page directory. */ -UNIV_INTERN void page_zip_dir_insert( /*================*/ @@ -4356,7 +4404,6 @@ page_zip_dir_insert( /**********************************************************************//** Shift the dense page directory and the array of BLOB pointers when a record is deleted. */ -UNIV_INTERN void page_zip_dir_delete( /*================*/ @@ -4429,8 +4476,7 @@ page_zip_dir_delete( externs = page_zip->data + page_zip_get_size(page_zip) - (page_dir_get_n_heap(page) - PAGE_HEAP_NO_USER_LOW) - * (PAGE_ZIP_DIR_SLOT_SIZE - + DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN); + * PAGE_ZIP_CLUST_LEAF_SLOT_SIZE; ext_end = externs - page_zip->n_blobs * BTR_EXTERN_FIELD_REF_SIZE; @@ -4454,7 +4500,6 @@ skip_blobs: /**********************************************************************//** Add a slot to the dense page directory. */ -UNIV_INTERN void page_zip_dir_add_slot( /*==================*/ @@ -4488,19 +4533,15 @@ page_zip_dir_add_slot( * (DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN); externs = stored - page_zip->n_blobs * BTR_EXTERN_FIELD_REF_SIZE; - ASSERT_ZERO(externs - - (PAGE_ZIP_DIR_SLOT_SIZE - + DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN), - PAGE_ZIP_DIR_SLOT_SIZE - + DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN); - memmove(externs - (PAGE_ZIP_DIR_SLOT_SIZE - + DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN), + ASSERT_ZERO(externs - PAGE_ZIP_CLUST_LEAF_SLOT_SIZE, + PAGE_ZIP_CLUST_LEAF_SLOT_SIZE); + memmove(externs - PAGE_ZIP_CLUST_LEAF_SLOT_SIZE, externs, stored - externs); } else { stored = dir - page_zip->n_blobs * BTR_EXTERN_FIELD_REF_SIZE; ASSERT_ZERO(stored - PAGE_ZIP_DIR_SLOT_SIZE, - PAGE_ZIP_DIR_SLOT_SIZE); + static_cast<size_t>(PAGE_ZIP_DIR_SLOT_SIZE)); } /* Move the uncompressed area backwards to make space @@ -4510,8 +4551,7 @@ page_zip_dir_add_slot( /***********************************************************//** Parses a log record of writing to the header of a page. -@return end of log record or NULL */ -UNIV_INTERN +@return end of log record or NULL */ byte* page_zip_parse_write_header( /*========================*/ @@ -4523,7 +4563,8 @@ page_zip_parse_write_header( ulint offset; ulint len; - ut_ad(ptr && end_ptr); + ut_ad(ptr != NULL); + ut_ad(end_ptr!= NULL); ut_ad(!page == !page_zip); if (UNIV_UNLIKELY(end_ptr < ptr + (1 + 1))) { @@ -4534,20 +4575,20 @@ page_zip_parse_write_header( offset = (ulint) *ptr++; len = (ulint) *ptr++; - if (UNIV_UNLIKELY(!len) || UNIV_UNLIKELY(offset + len >= PAGE_DATA)) { + if (len == 0 || offset + len >= PAGE_DATA) { corrupt: recv_sys->found_corrupt_log = TRUE; return(NULL); } - if (UNIV_UNLIKELY(end_ptr < ptr + len)) { + if (end_ptr < ptr + len) { return(NULL); } if (page) { - if (UNIV_UNLIKELY(!page_zip)) { + if (!page_zip) { goto corrupt; } @@ -4569,7 +4610,6 @@ corrupt: #ifndef UNIV_HOTBACKUP /**********************************************************************//** Write a log record of writing to the uncompressed header portion of a page. */ -UNIV_INTERN void page_zip_write_header_log( /*======================*/ @@ -4614,7 +4654,6 @@ bits in the same mini-transaction in such a way that the modification will be redo-logged. @return TRUE on success, FALSE on failure; page_zip will be left intact on failure, but page will be overwritten. */ -UNIV_INTERN ibool page_zip_reorganize( /*================*/ @@ -4632,7 +4671,6 @@ page_zip_reorganize( page_t* page = buf_block_get_frame(block); buf_block_t* temp_block; page_t* temp_page; - ulint log_mode; ut_ad(mtr_memo_contains(mtr, block, MTR_MEMO_PAGE_X_FIX)); ut_ad(page_is_comp(page)); @@ -4642,12 +4680,11 @@ page_zip_reorganize( UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); /* Disable logging */ - log_mode = mtr_set_log_mode(mtr, MTR_LOG_NONE); + mtr_log_t log_mode = mtr_set_log_mode(mtr, MTR_LOG_NONE); #ifndef UNIV_HOTBACKUP temp_block = buf_block_alloc(buf_pool); btr_search_drop_page_hash_index(block); - block->check_index_page_at_flush = TRUE; #else /* !UNIV_HOTBACKUP */ ut_ad(block == back_block1); temp_block = back_block2; @@ -4657,12 +4694,10 @@ page_zip_reorganize( /* Copy the old page to temporary space */ buf_frame_copy(temp_page, page); - btr_blob_dbg_remove(page, index, "zip_reorg"); - /* Recreate the page: note that global data on page (possible segment headers, next page-field, etc.) is preserved intact */ - page_create(block, mtr, TRUE); + page_create(block, mtr, TRUE, dict_index_is_spatial(index)); /* Copy the records from the temporary space to the recreated page; do not copy the lock bits yet */ @@ -4671,7 +4706,14 @@ page_zip_reorganize( page_get_infimum_rec(temp_page), index, mtr); - if (!dict_index_is_clust(index) && page_is_leaf(temp_page)) { + /* Temp-Tables are not shared across connection and so we avoid + locking of temp-tables as there would be no 2 trx trying to + operate on same temp-table in parallel. + max_trx_id is use to track which all trxs wrote to the page + in parallel but in case of temp-table this can is not needed. */ + if (!dict_index_is_clust(index) + && !dict_table_is_temporary(index->table) + && page_is_leaf(temp_page)) { /* Copy max trx id to recreated page */ trx_id_t max_trx_id = page_get_max_trx_id(temp_page); page_set_max_trx_id(block, NULL, max_trx_id, NULL); @@ -4681,7 +4723,8 @@ page_zip_reorganize( /* Restore logging. */ mtr_set_log_mode(mtr, log_mode); - if (!page_zip_compress(page_zip, page, index, page_zip_level, mtr)) { + if (!page_zip_compress(page_zip, page, index, + page_zip_level, NULL, mtr)) { #ifndef UNIV_HOTBACKUP buf_block_free(temp_block); @@ -4703,7 +4746,6 @@ Copy the records of a page byte for byte. Do not copy the page header or trailer, except those B-tree header fields that are directly related to the storage of records. Also copy PAGE_MAX_TRX_ID. NOTE: The caller must update the lock table and the adaptive hash index. */ -UNIV_INTERN void page_zip_copy_recs( /*===============*/ @@ -4716,8 +4758,10 @@ page_zip_copy_recs( dict_index_t* index, /*!< in: index of the B-tree */ mtr_t* mtr) /*!< in: mini-transaction */ { - ut_ad(mtr_memo_contains_page(mtr, page, MTR_MEMO_PAGE_X_FIX)); - ut_ad(mtr_memo_contains_page(mtr, src, MTR_MEMO_PAGE_X_FIX)); + ut_ad(mtr_memo_contains_page(mtr, page, MTR_MEMO_PAGE_X_FIX) + || dict_table_is_intrinsic(index->table)); + ut_ad(mtr_memo_contains_page(mtr, src, MTR_MEMO_PAGE_X_FIX) + || dict_table_is_intrinsic(index->table)); ut_ad(!dict_index_is_ibuf(index)); #ifdef UNIV_ZIP_DEBUG /* The B-tree operations that call this function may set @@ -4734,7 +4778,9 @@ page_zip_copy_recs( /* The PAGE_MAX_TRX_ID must be set on leaf pages of secondary indexes. It does not matter on other pages. */ - ut_a(dict_index_is_clust(index) || !page_is_leaf(src) + ut_a(dict_index_is_clust(index) + || dict_table_is_temporary(index->table) + || !page_is_leaf(src) || page_get_max_trx_id(src)); UNIV_MEM_ASSERT_W(page, UNIV_PAGE_SIZE); @@ -4786,16 +4832,13 @@ page_zip_copy_recs( #ifdef UNIV_ZIP_DEBUG ut_a(page_zip_validate(page_zip, page, index)); #endif /* UNIV_ZIP_DEBUG */ - btr_blob_dbg_add(page, index, "page_zip_copy_recs"); - page_zip_compress_write_log(page_zip, page, index, mtr); } #endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Parses a log record of compressing an index page. -@return end of log record or NULL */ -UNIV_INTERN +@return end of log record or NULL */ byte* page_zip_parse_compress( /*====================*/ @@ -4808,7 +4851,7 @@ page_zip_parse_compress( ulint trailer_size; ut_ad(ptr != NULL); - ut_ad(end_ptr != NULL); + ut_ad(end_ptr!= NULL); ut_ad(!page == !page_zip); if (UNIV_UNLIKELY(ptr + (2 + 2) > end_ptr)) { @@ -4827,8 +4870,7 @@ page_zip_parse_compress( } if (page) { - if (UNIV_UNLIKELY(!page_zip) - || UNIV_UNLIKELY(page_zip_get_size(page_zip) < size)) { + if (!page_zip || page_zip_get_size(page_zip) < size) { corrupt: recv_sys->found_corrupt_log = TRUE; @@ -4855,19 +4897,22 @@ corrupt: } #endif /* !UNIV_INNOCHECKSUM */ -/**********************************************************************//** -Calculate the compressed page checksum. -@return page checksum */ -UNIV_INTERN -ulint +/** Calculate the compressed page checksum. +@param[in] data compressed page +@param[in] size size of compressed page +@param[in] algo algorithm to use +@param[in] use_legacy_big_endian only used if algo is +SRV_CHECKSUM_ALGORITHM_CRC32 or SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 - if true +then use big endian byteorder when converting byte strings to integers. +@return page checksum */ +uint32_t page_zip_calc_checksum( -/*===================*/ - const void* data, /*!< in: compressed page */ - ulint size, /*!< in: size of compressed page */ - srv_checksum_algorithm_t algo) /*!< in: algorithm to use */ + const void* data, + ulint size, + srv_checksum_algorithm_t algo, + bool use_legacy_big_endian /* = false */) { - uLong adler; - ib_uint32_t crc32; + uint32_t adler; const Bytef* s = static_cast<const byte*>(data); /* Exclude FIL_PAGE_SPACE_OR_CHKSUM, FIL_PAGE_LSN, @@ -4876,16 +4921,25 @@ page_zip_calc_checksum( switch (algo) { case SRV_CHECKSUM_ALGORITHM_CRC32: case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: - - ut_ad(size > FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - - crc32 = ut_crc32(s + FIL_PAGE_OFFSET, - FIL_PAGE_LSN - FIL_PAGE_OFFSET) - ^ ut_crc32(s + FIL_PAGE_TYPE, 2) - ^ ut_crc32(s + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, - size - FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - - return((ulint) crc32); + { + ut_ad(size > FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + + ut_crc32_func_t crc32_func = use_legacy_big_endian + ? ut_crc32_legacy_big_endian + : ut_crc32; + + const uint32_t crc32 + = crc32_func( + s + FIL_PAGE_OFFSET, + FIL_PAGE_LSN - FIL_PAGE_OFFSET) + ^ crc32_func( + s + FIL_PAGE_TYPE, 2) + ^ crc32_func( + s + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, + size - FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + + return(crc32); + } case SRV_CHECKSUM_ALGORITHM_INNODB: case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: ut_ad(size > FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); @@ -4898,7 +4952,7 @@ page_zip_calc_checksum( static_cast<uInt>(size) - FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - return((ulint) adler); + return(adler); case SRV_CHECKSUM_ALGORITHM_NONE: case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: return(BUF_NO_CHECKSUM_MAGIC); @@ -4912,54 +4966,66 @@ page_zip_calc_checksum( /**********************************************************************//** Verify a compressed page's checksum. -@return TRUE if the stored checksum is valid according to the value of +@return TRUE if the stored checksum is valid according to the value of innodb_checksum_algorithm */ -UNIV_INTERN ibool page_zip_verify_checksum( /*=====================*/ - const void* data, /*!< in: compressed page */ - ulint size) /*!< in: size of compressed page */ + const void* data, /*!< in: compressed page */ + ulint size /*!< in: size of compressed page */ +#ifdef UNIV_INNOCHECKSUM + /* these variables are used only for innochecksum tool. */ + ,uintmax_t page_no, /*!< in: page number of + given read_buf */ + bool strict_check, /*!< in: true if strict-check + option is enable */ + bool is_log_enabled, /*!< in: true if log option is + enabled */ + FILE* log_file /*!< in: file pointer to + log_file */ +#endif /* UNIV_INNOCHECKSUM */ +) { - ib_uint32_t stored; - ib_uint32_t calc; - ib_uint32_t crc32 = 0 /* silence bogus warning */; - ib_uint32_t innodb = 0 /* silence bogus warning */; - - stored = static_cast<ib_uint32_t>(mach_read_from_4( - static_cast<const unsigned char*>(data) + FIL_PAGE_SPACE_OR_CHKSUM)); + const unsigned char* p = static_cast<const unsigned char*>(data) + + FIL_PAGE_SPACE_OR_CHKSUM; - ulint page_no MY_ATTRIBUTE((unused)) = - mach_read_from_4(static_cast<const unsigned char*> - (data) + FIL_PAGE_OFFSET); - ulint space_id MY_ATTRIBUTE((unused)) = - mach_read_from_4(static_cast<const unsigned char*> - (data) + FIL_PAGE_SPACE_ID); + const uint32_t stored = static_cast<uint32_t>( + mach_read_from_4(p)); #if FIL_PAGE_LSN % 8 #error "FIL_PAGE_LSN must be 64 bit aligned" #endif -#ifndef UNIV_INNOCHECKSUM - /* innochecksum doesn't compile with ut_d. Since we don't - need to check for empty pages when running innochecksum, - just don't include this code. */ /* Check if page is empty */ if (stored == 0 && *reinterpret_cast<const ib_uint64_t*>(static_cast<const char*>( data) + FIL_PAGE_LSN) == 0) { /* make sure that the page is really empty */ +#ifdef UNIV_INNOCHECKSUM ulint i; for (i = 0; i < size; i++) { + if (*((const char*) data + i) != 0) + break; + } + if (i >= size) { + if (is_log_enabled) { + fprintf(log_file, "Page::%lu is empty and" + " uncorrupted\n", page_no); + } + + return(TRUE); + } +#else + for (ulint i = 0; i < size; i++) { if (*((const char*) data + i) != 0) { return(FALSE); } } /* Empty page */ return(TRUE); +#endif /* UNIV_INNOCHECKSUM */ } -#endif const srv_checksum_algorithm_t curr_algo = static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm); @@ -4968,9 +5034,44 @@ page_zip_verify_checksum( return(TRUE); } - calc = static_cast<ib_uint32_t>(page_zip_calc_checksum( - data, size, curr_algo)); - +#ifndef UNIV_INNOCHECKSUM + ulint page_no = mach_read_from_4(static_cast< + const unsigned char*> + (data) + FIL_PAGE_OFFSET); + ulint space_id = mach_read_from_4(static_cast< + const unsigned char*> + (data) + FIL_PAGE_SPACE_ID); + const page_id_t page_id(space_id, page_no); +#endif /* UNIV_INNOCHECKSUM */ + + const uint32_t calc = page_zip_calc_checksum(data, size, curr_algo); + +#ifdef UNIV_INNOCHECKSUM + if (is_log_enabled) { + fprintf(log_file, "page::%lu;" + " %s checksum: calculated = %u;" + " recorded = %u\n", page_no, + buf_checksum_algorithm_name( + static_cast<srv_checksum_algorithm_t>( + srv_checksum_algorithm)), + calc, stored); + } + + if (!strict_check) { + + const uint32_t crc32 = page_zip_calc_checksum( + data, size, SRV_CHECKSUM_ALGORITHM_CRC32); + + if (is_log_enabled) { + fprintf(log_file, "page::%lu: crc32 checksum:" + " calculated = %u; recorded = %u\n", + page_no, crc32, stored); + fprintf(log_file, "page::%lu: none checksum:" + " calculated = %lu; recorded = %u\n", + page_no, BUF_NO_CHECKSUM_MAGIC, stored); + } + } +#endif /* UNIV_INNOCHECKSUM */ if (stored == calc) { return(TRUE); } @@ -4980,28 +5081,36 @@ page_zip_verify_checksum( case SRV_CHECKSUM_ALGORITHM_CRC32: if (stored == BUF_NO_CHECKSUM_MAGIC) { +#ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_NONE, - space_id, page_no); + page_id); } +#endif /* UNIV_INNOCHECKSUM */ return(TRUE); } - innodb = static_cast<ib_uint32_t>(page_zip_calc_checksum( - data, size, SRV_CHECKSUM_ALGORITHM_INNODB)); + if (stored == page_zip_calc_checksum(data, size, curr_algo, + true)) { + return(TRUE); + } - if (stored == innodb) { + if (stored == page_zip_calc_checksum( + data, size, SRV_CHECKSUM_ALGORITHM_INNODB)) { + +#ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_INNODB, - space_id, page_no); + page_id); } +#endif /* UNIV_INNOCHECKSUM */ return(TRUE); } @@ -5011,54 +5120,60 @@ page_zip_verify_checksum( case SRV_CHECKSUM_ALGORITHM_INNODB: if (stored == BUF_NO_CHECKSUM_MAGIC) { +#ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_NONE, - space_id, page_no); + page_id); } +#endif /* UNIV_INNOCHECKSUM */ return(TRUE); } - crc32 = static_cast<ib_uint32_t>(page_zip_calc_checksum( - data, size, SRV_CHECKSUM_ALGORITHM_CRC32)); - - if (stored == crc32) { + if (stored == page_zip_calc_checksum( + data, size, SRV_CHECKSUM_ALGORITHM_CRC32) + || stored == page_zip_calc_checksum( + data, size, SRV_CHECKSUM_ALGORITHM_CRC32, true)) { +#ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_CRC32, - space_id, page_no); + page_id); } - +#endif /* UNIV_INNOCHECKSUM */ return(TRUE); } break; case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: - crc32 = static_cast<ib_uint32_t>(page_zip_calc_checksum( - data, size, SRV_CHECKSUM_ALGORITHM_CRC32)); - - if (stored == crc32) { + if (stored == page_zip_calc_checksum( + data, size, SRV_CHECKSUM_ALGORITHM_CRC32) + || stored == page_zip_calc_checksum( + data, size, SRV_CHECKSUM_ALGORITHM_CRC32, true)) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( - curr_algo, SRV_CHECKSUM_ALGORITHM_CRC32, - space_id, page_no); - + curr_algo, + SRV_CHECKSUM_ALGORITHM_CRC32, + page_id); +#endif /* UNIV_INNOCHECKSUM */ return(TRUE); } - innodb = static_cast<ib_uint32_t>(page_zip_calc_checksum( - data, size, SRV_CHECKSUM_ALGORITHM_INNODB)); + if (stored == page_zip_calc_checksum( + data, size, SRV_CHECKSUM_ALGORITHM_INNODB)) { - if (stored == innodb) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_INNODB, - space_id, page_no); + page_id); +#endif /* UNIV_INNOCHECKSUM */ return(TRUE); } |