From cdd1dc829be671b5d866d00841179c9dae10358f Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Thu, 23 Mar 2017 15:46:11 +0530 Subject: MW-28, codership/mysql-wsrep#28 Fix sync_thread_levels debug assert Introduced a new wsrep_trx_print_locking() which may be called under lock_sys->mutex if the trx has locks. Signed-off-by: Sachin Setiya --- storage/innobase/include/trx0trx.h | 17 ++++++ storage/innobase/lock/lock0lock.cc | 8 +-- storage/innobase/trx/trx0trx.cc | 112 +++++++++++++++++++++++++++++++++++++ storage/xtradb/include/trx0trx.h | 17 ++++++ storage/xtradb/lock/lock0lock.cc | 8 +-- storage/xtradb/trx/trx0trx.cc | 112 +++++++++++++++++++++++++++++++++++++ 6 files changed, 262 insertions(+), 12 deletions(-) (limited to 'storage') diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index 5936fa90e84..7848cb06955 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -334,6 +334,23 @@ trx_print_latched( or 0 to use the default max length */ MY_ATTRIBUTE((nonnull)); +#ifdef WITH_WSREP +/**********************************************************************//** +Prints info about a transaction. +Transaction information may be retrieved without having trx_sys->mutex acquired +so it may not be completely accurate. The caller must own lock_sys->mutex +and the trx must have some locks to make sure that it does not escape +without locking lock_sys->mutex. */ +UNIV_INTERN +void +wsrep_trx_print_locking( +/*==============*/ + FILE* f, /*!< in: output stream */ + const trx_t* trx, /*!< in: transaction */ + ulint max_query_len) /*!< in: max query length to print, + or 0 to use the default max length */ + MY_ATTRIBUTE((nonnull)); +#endif /* WITH_WSREP */ /**********************************************************************//** Prints info about a transaction. Acquires and releases lock_sys->mutex and trx_sys->mutex. */ diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 4812dc9ae91..a951eff4203 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -1752,7 +1752,6 @@ wsrep_kill_victim( is in the queue*/ } else if (lock->trx != trx) { if (wsrep_log_conflicts) { - mutex_enter(&trx_sys->mutex); if (bf_this) { fputs("\n*** Priority TRANSACTION:\n", stderr); @@ -1761,7 +1760,7 @@ wsrep_kill_victim( stderr); } - trx_print_latched(stderr, trx, 3000); + wsrep_trx_print_locking(stderr, trx, 3000); if (bf_other) { fputs("\n*** Priority TRANSACTION:\n", @@ -1770,10 +1769,7 @@ wsrep_kill_victim( fputs("\n*** Victim TRANSACTION:\n", stderr); } - - trx_print_latched(stderr, lock->trx, 3000); - - mutex_exit(&trx_sys->mutex); + wsrep_trx_print_locking(stderr, lock->trx, 3000); fputs("*** WAITING FOR THIS LOCK TO BE GRANTED:\n", stderr); diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 318f1e284ec..17a8b296f7a 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -1885,6 +1885,118 @@ trx_print_latched( mem_heap_get_size(trx->lock.lock_heap)); } +#ifdef WITH_WSREP +/**********************************************************************//** +Prints info about a transaction. +Transaction information may be retrieved without having trx_sys->mutex acquired +so it may not be completely accurate. The caller must own lock_sys->mutex +and the trx must have some locks to make sure that it does not escape +without locking lock_sys->mutex. */ +UNIV_INTERN +void +wsrep_trx_print_locking( +/*==========*/ + FILE* f, + /*!< in: output stream */ + const trx_t* trx, + /*!< in: transaction */ + ulint max_query_len) + /*!< in: max query length to print, + or 0 to use the default max length */ +{ + ibool newline; + const char* op_info; + + ut_ad(lock_mutex_own()); + ut_ad(trx->lock.trx_locks.count > 0); + + fprintf(f, "TRANSACTION " TRX_ID_FMT, trx->id); + + /* trx->state may change since trx_sys->mutex is not required */ + switch (trx->state) { + case TRX_STATE_NOT_STARTED: + fputs(", not started", f); + goto state_ok; + case TRX_STATE_ACTIVE: + fprintf(f, ", ACTIVE %lu sec", + (ulong) difftime(time(NULL), trx->start_time)); + goto state_ok; + case TRX_STATE_PREPARED: + fprintf(f, ", ACTIVE (PREPARED) %lu sec", + (ulong) difftime(time(NULL), trx->start_time)); + goto state_ok; + case TRX_STATE_COMMITTED_IN_MEMORY: + fputs(", COMMITTED IN MEMORY", f); + goto state_ok; + } + fprintf(f, ", state %lu", (ulong) trx->state); + ut_ad(0); +state_ok: + + /* prevent a race condition */ + op_info = trx->op_info; + + if (*op_info) { + putc(' ', f); + fputs(op_info, f); + } + + if (trx->is_recovered) { + fputs(" recovered trx", f); + } + + if (trx->declared_to_be_inside_innodb) { + fprintf(f, ", thread declared inside InnoDB %lu", + (ulong) trx->n_tickets_to_enter_innodb); + } + + putc('\n', f); + + if (trx->n_mysql_tables_in_use > 0 || trx->mysql_n_tables_locked > 0) { + fprintf(f, "mysql tables in use %lu, locked %lu\n", + (ulong) trx->n_mysql_tables_in_use, + (ulong) trx->mysql_n_tables_locked); + } + + newline = TRUE; + + /* trx->lock.que_state of an ACTIVE transaction may change + while we are not holding trx->mutex. We perform a dirty read + for performance reasons. */ + + switch (trx->lock.que_state) { + case TRX_QUE_RUNNING: + newline = FALSE; break; + case TRX_QUE_LOCK_WAIT: + fputs("LOCK WAIT ", f); break; + case TRX_QUE_ROLLING_BACK: + fputs("ROLLING BACK ", f); break; + case TRX_QUE_COMMITTING: + fputs("COMMITTING ", f); break; + default: + fprintf(f, "que state %lu ", (ulong) trx->lock.que_state); + } + + if (trx->has_search_latch) { + newline = TRUE; + fputs(", holds adaptive hash latch", f); + } + + if (trx->undo_no != 0) { + newline = TRUE; + fprintf(f, ", undo log entries " TRX_ID_FMT, trx->undo_no); + } + + if (newline) { + putc('\n', f); + } + + if (trx->mysql_thd != NULL) { + innobase_mysql_print_thd( + f, trx->mysql_thd, static_cast(max_query_len)); + } +} +#endif /* WITH_WSREP */ /**********************************************************************//** Prints info about a transaction. Acquires and releases lock_sys->mutex and trx_sys->mutex. */ diff --git a/storage/xtradb/include/trx0trx.h b/storage/xtradb/include/trx0trx.h index 9e2064c3dc2..b5fa3930bbb 100644 --- a/storage/xtradb/include/trx0trx.h +++ b/storage/xtradb/include/trx0trx.h @@ -348,6 +348,23 @@ trx_print_latched( or 0 to use the default max length */ MY_ATTRIBUTE((nonnull)); +#ifdef WITH_WSREP +/**********************************************************************//** +Prints info about a transaction. +Transaction information may be retrieved without having trx_sys->mutex acquired +so it may not be completely accurate. The caller must own lock_sys->mutex +and the trx must have some locks to make sure that it does not escape +without locking lock_sys->mutex. */ +UNIV_INTERN +void +wsrep_trx_print_locking( +/*==============*/ + FILE* f, /*!< in: output stream */ + const trx_t* trx, /*!< in: transaction */ + ulint max_query_len) /*!< in: max query length to print, + or 0 to use the default max length */ + MY_ATTRIBUTE((nonnull)); +#endif /* WITH_WSREP */ /**********************************************************************//** Prints info about a transaction. Acquires and releases lock_sys->mutex and trx_sys->mutex. */ diff --git a/storage/xtradb/lock/lock0lock.cc b/storage/xtradb/lock/lock0lock.cc index 0d555ed2dd7..717fbf02536 100644 --- a/storage/xtradb/lock/lock0lock.cc +++ b/storage/xtradb/lock/lock0lock.cc @@ -1762,7 +1762,6 @@ wsrep_kill_victim( is in the queue*/ } else if (lock->trx != trx) { if (wsrep_log_conflicts) { - mutex_enter(&trx_sys->mutex); if (bf_this) { fputs("\n*** Priority TRANSACTION:\n", stderr); @@ -1771,7 +1770,7 @@ wsrep_kill_victim( stderr); } - trx_print_latched(stderr, trx, 3000); + wsrep_trx_print_locking(stderr, trx, 3000); if (bf_other) { fputs("\n*** Priority TRANSACTION:\n", @@ -1780,10 +1779,7 @@ wsrep_kill_victim( fputs("\n*** Victim TRANSACTION:\n", stderr); } - - trx_print_latched(stderr, lock->trx, 3000); - - mutex_exit(&trx_sys->mutex); + wsrep_trx_print_locking(stderr, lock->trx, 3000); fputs("*** WAITING FOR THIS LOCK TO BE GRANTED:\n", stderr); diff --git a/storage/xtradb/trx/trx0trx.cc b/storage/xtradb/trx/trx0trx.cc index 439897a5b96..92d7525ea84 100644 --- a/storage/xtradb/trx/trx0trx.cc +++ b/storage/xtradb/trx/trx0trx.cc @@ -2161,6 +2161,118 @@ trx_print_latched( mem_heap_get_size(trx->lock.lock_heap)); } +#ifdef WITH_WSREP +/**********************************************************************//** +Prints info about a transaction. +Transaction information may be retrieved without having trx_sys->mutex acquired +so it may not be completely accurate. The caller must own lock_sys->mutex +and the trx must have some locks to make sure that it does not escape +without locking lock_sys->mutex. */ +UNIV_INTERN +void +wsrep_trx_print_locking( +/*==========*/ + FILE* f, + /*!< in: output stream */ + const trx_t* trx, + /*!< in: transaction */ + ulint max_query_len) + /*!< in: max query length to print, + or 0 to use the default max length */ +{ + ibool newline; + const char* op_info; + + ut_ad(lock_mutex_own()); + ut_ad(trx->lock.trx_locks.count > 0); + + fprintf(f, "TRANSACTION " TRX_ID_FMT, trx->id); + + /* trx->state may change since trx_sys->mutex is not required */ + switch (trx->state) { + case TRX_STATE_NOT_STARTED: + fputs(", not started", f); + goto state_ok; + case TRX_STATE_ACTIVE: + fprintf(f, ", ACTIVE %lu sec", + (ulong) difftime(time(NULL), trx->start_time)); + goto state_ok; + case TRX_STATE_PREPARED: + fprintf(f, ", ACTIVE (PREPARED) %lu sec", + (ulong) difftime(time(NULL), trx->start_time)); + goto state_ok; + case TRX_STATE_COMMITTED_IN_MEMORY: + fputs(", COMMITTED IN MEMORY", f); + goto state_ok; + } + fprintf(f, ", state %lu", (ulong) trx->state); + ut_ad(0); +state_ok: + + /* prevent a race condition */ + op_info = trx->op_info; + + if (*op_info) { + putc(' ', f); + fputs(op_info, f); + } + + if (trx->is_recovered) { + fputs(" recovered trx", f); + } + + if (trx->declared_to_be_inside_innodb) { + fprintf(f, ", thread declared inside InnoDB %lu", + (ulong) trx->n_tickets_to_enter_innodb); + } + + putc('\n', f); + + if (trx->n_mysql_tables_in_use > 0 || trx->mysql_n_tables_locked > 0) { + fprintf(f, "mysql tables in use %lu, locked %lu\n", + (ulong) trx->n_mysql_tables_in_use, + (ulong) trx->mysql_n_tables_locked); + } + + newline = TRUE; + + /* trx->lock.que_state of an ACTIVE transaction may change + while we are not holding trx->mutex. We perform a dirty read + for performance reasons. */ + + switch (trx->lock.que_state) { + case TRX_QUE_RUNNING: + newline = FALSE; break; + case TRX_QUE_LOCK_WAIT: + fputs("LOCK WAIT ", f); break; + case TRX_QUE_ROLLING_BACK: + fputs("ROLLING BACK ", f); break; + case TRX_QUE_COMMITTING: + fputs("COMMITTING ", f); break; + default: + fprintf(f, "que state %lu ", (ulong) trx->lock.que_state); + } + + if (trx->has_search_latch) { + newline = TRUE; + fputs(", holds adaptive hash latch", f); + } + + if (trx->undo_no != 0) { + newline = TRUE; + fprintf(f, ", undo log entries " TRX_ID_FMT, trx->undo_no); + } + + if (newline) { + putc('\n', f); + } + + if (trx->mysql_thd != NULL) { + innobase_mysql_print_thd( + f, trx->mysql_thd, static_cast(max_query_len)); + } +} +#endif /* WITH_WSREP */ /**********************************************************************//** Prints info about a transaction. Acquires and releases lock_sys->mutex and trx_sys->mutex. */ -- cgit v1.2.1 From 16b2b1eae5e72dc0533e4709815c13803e102b1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 8 Apr 2017 16:46:26 +0300 Subject: Use page_is_leaf() where applicable --- storage/innobase/btr/btr0btr.cc | 2 +- storage/innobase/btr/btr0defragment.cc | 2 +- storage/innobase/dict/dict0stats.cc | 4 ++-- storage/xtradb/btr/btr0btr.cc | 2 +- storage/xtradb/btr/btr0defragment.cc | 2 +- storage/xtradb/dict/dict0stats.cc | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) (limited to 'storage') diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index 470825fa246..019ea974ea7 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -3241,7 +3241,7 @@ func_start: btr_page_create(new_block, new_page_zip, cursor->index, btr_page_get_level(page, mtr), mtr); /* Only record the leaf level page splits. */ - if (btr_page_get_level(page, mtr) == 0) { + if (page_is_leaf(page)) { cursor->index->stat_defrag_n_page_split ++; cursor->index->stat_defrag_modified_counter ++; btr_defragment_save_defrag_stats_if_needed(cursor->index); diff --git a/storage/innobase/btr/btr0defragment.cc b/storage/innobase/btr/btr0defragment.cc index 26bab936de5..ea7fb116af2 100644 --- a/storage/innobase/btr/btr0defragment.cc +++ b/storage/innobase/btr/btr0defragment.cc @@ -233,7 +233,7 @@ btr_defragment_add_index( return NULL; } - if (btr_page_get_level(page, &mtr) == 0) { + if (page_is_leaf(page)) { // Index root is a leaf page, no need to defragment. mtr_commit(&mtr); return NULL; diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index 6a28f3cdf8f..33b6c2e23e0 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -1576,7 +1576,7 @@ dict_stats_analyze_index_below_cur( page = buf_block_get_frame(block); - if (btr_page_get_level(page, mtr) == 0) { + if (page_is_leaf(page)) { /* leaf level */ break; } @@ -1620,7 +1620,7 @@ dict_stats_analyze_index_below_cur( } /* make sure we got a leaf page as a result from the above loop */ - ut_ad(btr_page_get_level(page, &mtr) == 0); + ut_ad(page_is_leaf(page)); /* scan the leaf page and find the number of distinct keys, when looking only at the first n_prefix columns; also estimate diff --git a/storage/xtradb/btr/btr0btr.cc b/storage/xtradb/btr/btr0btr.cc index 417eeb2c367..48411b6ff6a 100644 --- a/storage/xtradb/btr/btr0btr.cc +++ b/storage/xtradb/btr/btr0btr.cc @@ -3277,7 +3277,7 @@ func_start: btr_page_create(new_block, new_page_zip, cursor->index, btr_page_get_level(page, mtr), mtr); /* Only record the leaf level page splits. */ - if (btr_page_get_level(page, mtr) == 0) { + if (page_is_leaf(page)) { cursor->index->stat_defrag_n_page_split ++; cursor->index->stat_defrag_modified_counter ++; btr_defragment_save_defrag_stats_if_needed(cursor->index); diff --git a/storage/xtradb/btr/btr0defragment.cc b/storage/xtradb/btr/btr0defragment.cc index 6e7a3fd2411..ca00eed40a9 100644 --- a/storage/xtradb/btr/btr0defragment.cc +++ b/storage/xtradb/btr/btr0defragment.cc @@ -233,7 +233,7 @@ btr_defragment_add_index( return NULL; } - if (btr_page_get_level(page, &mtr) == 0) { + if (page_is_leaf(page)) { // Index root is a leaf page, no need to defragment. mtr_commit(&mtr); return NULL; diff --git a/storage/xtradb/dict/dict0stats.cc b/storage/xtradb/dict/dict0stats.cc index 6a28f3cdf8f..33b6c2e23e0 100644 --- a/storage/xtradb/dict/dict0stats.cc +++ b/storage/xtradb/dict/dict0stats.cc @@ -1576,7 +1576,7 @@ dict_stats_analyze_index_below_cur( page = buf_block_get_frame(block); - if (btr_page_get_level(page, mtr) == 0) { + if (page_is_leaf(page)) { /* leaf level */ break; } @@ -1620,7 +1620,7 @@ dict_stats_analyze_index_below_cur( } /* make sure we got a leaf page as a result from the above loop */ - ut_ad(btr_page_get_level(page, &mtr) == 0); + ut_ad(page_is_leaf(page)); /* scan the leaf page and find the number of distinct keys, when looking only at the first n_prefix columns; also estimate -- cgit v1.2.1 From 3bb32e8682f849413ce994277e9029ba5732baae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 15 Apr 2017 03:36:23 +0300 Subject: MDEV-10509: Remove excessive server error logging on InnoDB ALTER TABLE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disable the output that was added in MDEV-6812 if log_warnings=2 or less. Also, remove some redundant messages. TODO: Implement MDEV-12512 to supercede MDEV-6812 and properly report the progress of ALTER TABLE…ALGORITHM=INPLACE. --- storage/innobase/row/row0merge.cc | 90 +++++++++++++++++++++++++------------- storage/xtradb/row/row0merge.cc | 91 ++++++++++++++++++++++++++------------- 2 files changed, 122 insertions(+), 59 deletions(-) (limited to 'storage') diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 5531263b248..c7c0092e7a4 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -25,6 +26,7 @@ Completed by Sunny Bains and Marko Makela *******************************************************/ #include #include +#include #include "row0merge.h" #include "row0ext.h" @@ -2628,7 +2630,11 @@ row_merge_sort( } #endif /* UNIV_SOLARIS */ - sql_print_information("InnoDB: Online DDL : merge-sorting has estimated %lu runs", num_runs); + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL : merge-sorting" + " has estimated " ULINTPF " runs", + num_runs); + } /* Merge the runs until we have one big run */ do { @@ -4057,9 +4063,11 @@ row_merge_build_indexes( duplicate keys. */ innobase_rec_reset(table); - sql_print_information("InnoDB: Online DDL : Start"); - sql_print_information("InnoDB: Online DDL : Start reading clustered " - "index of the table and create temporary files"); + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL : Start reading" + " clustered index of the table" + " and create temporary files"); + } pct_cost = COST_READ_CLUSTERED_INDEX * 100 / (total_static_cost + total_dynamic_cost); @@ -4086,8 +4094,11 @@ row_merge_build_indexes( pct_progress += pct_cost; - sql_print_information("InnoDB: Online DDL : End of reading " - "clustered index of the table and create temporary files"); + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL : End of reading " + "clustered index of the table" + " and create temporary files"); + } for (i = 0; i < n_indexes; i++) { total_index_blocks += merge_files[i].offset; @@ -4183,8 +4194,7 @@ wait_again: DEBUG_FTS_SORT_PRINT("FTS_SORT: Complete Insert\n"); #endif } else if (merge_files[i].fd != -1) { - char buf[3 * NAME_LEN]; - char *bufend; + char buf[NAME_LEN + 1]; row_merge_dup_t dup = { sort_idx, table, col_map, 0}; @@ -4193,18 +4203,25 @@ wait_again: total_index_blocks)) / (total_static_cost + total_dynamic_cost) * PCT_COST_MERGESORT_INDEX * 100; - - bufend = innobase_convert_name( + char* bufend = innobase_convert_name( buf, sizeof buf, - indexes[i]->name, strlen(indexes[i]->name), + indexes[i]->name, + strlen(indexes[i]->name), trx->mysql_thd, FALSE); - buf[bufend - buf]='\0'; - sql_print_information("InnoDB: Online DDL : Start merge-sorting" - " index %s (%lu / %lu), estimated cost : %2.4f", - buf, (i+1), n_indexes, pct_cost); + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL :" + " Start merge-sorting" + " index %s" + " (" ULINTPF + " / " ULINTPF ")," + " estimated cost :" + " %2.4f", + buf, i + 1, n_indexes, + pct_cost); + } error = row_merge_sort( trx, &dup, &merge_files[i], @@ -4214,9 +4231,14 @@ wait_again: pct_progress += pct_cost; - sql_print_information("InnoDB: Online DDL : End of " - " merge-sorting index %s (%lu / %lu)", - buf, (i+1), n_indexes); + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL :" + " End of " + " merge-sorting index %s" + " (" ULINTPF + " / " ULINTPF ")", + buf, i + 1, n_indexes); + } DBUG_EXECUTE_IF( "ib_merge_wait_after_sort", @@ -4229,10 +4251,15 @@ wait_again: (total_static_cost + total_dynamic_cost) * PCT_COST_INSERT_INDEX * 100; - sql_print_information("InnoDB: Online DDL : Start " - "building index %s (%lu / %lu), estimated " - "cost : %2.4f", buf, (i+1), - n_indexes, pct_cost); + if (global_system_variables.log_warnings > 2) { + sql_print_information( + "InnoDB: Online DDL : Start " + "building index %s" + " (" ULINTPF + " / " ULINTPF "), estimated " + "cost : %2.4f", buf, i + 1, + n_indexes, pct_cost); + } error = row_merge_insert_index_tuples( trx->id, sort_idx, old_table, @@ -4241,9 +4268,13 @@ wait_again: crypt_data, crypt_block, new_table->space); pct_progress += pct_cost; - sql_print_information("InnoDB: Online DDL : " - "End of building index %s (%lu / %lu)", - buf, (i+1), n_indexes); + if (global_system_variables.log_warnings > 2) { + sql_print_information( + "InnoDB: Online DDL : " + "End of building index %s" + " (" ULINTPF " / " ULINTPF ")", + buf, i + 1, n_indexes); + } } } @@ -4260,15 +4291,16 @@ wait_again: ut_ad(sort_idx->online_status == ONLINE_INDEX_COMPLETE); } else { - sql_print_information("InnoDB: Online DDL : Start applying row log"); + if (global_system_variables.log_warnings > 2) { + sql_print_information( + "InnoDB: Online DDL : Applying" + " log to index"); + } DEBUG_SYNC_C("row_log_apply_before"); error = row_log_apply(trx, sort_idx, table); DEBUG_SYNC_C("row_log_apply_after"); - sql_print_information("InnoDB: Online DDL : End of applying row log"); } - sql_print_information("InnoDB: Online DDL : Completed"); - if (error != DB_SUCCESS) { trx->error_key_num = key_numbers[i]; goto func_exit; diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index 8752cf489c9..7651248f463 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -26,6 +26,7 @@ Completed by Sunny Bains and Marko Makela *******************************************************/ #include #include +#include #include "row0merge.h" #include "row0ext.h" @@ -2635,7 +2636,11 @@ row_merge_sort( thd_progress_init(trx->mysql_thd, 1); } - sql_print_information("InnoDB: Online DDL : merge-sorting has estimated %lu runs", num_runs); + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL : merge-sorting" + " has estimated " ULINTPF " runs", + num_runs); + } /* Merge the runs until we have one big run */ do { @@ -4060,9 +4065,11 @@ row_merge_build_indexes( duplicate keys. */ innobase_rec_reset(table); - sql_print_information("InnoDB: Online DDL : Start"); - sql_print_information("InnoDB: Online DDL : Start reading clustered " - "index of the table and create temporary files"); + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL : Start reading" + " clustered index of the table" + " and create temporary files"); + } pct_cost = COST_READ_CLUSTERED_INDEX * 100 / (total_static_cost + total_dynamic_cost); @@ -4089,8 +4096,11 @@ row_merge_build_indexes( pct_progress += pct_cost; - sql_print_information("InnoDB: Online DDL : End of reading " - "clustered index of the table and create temporary files"); + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL : End of reading " + "clustered index of the table" + " and create temporary files"); + } for (i = 0; i < n_indexes; i++) { total_index_blocks += merge_files[i].offset; @@ -4186,8 +4196,7 @@ wait_again: DEBUG_FTS_SORT_PRINT("FTS_SORT: Complete Insert\n"); #endif } else if (merge_files[i].fd != -1) { - char buf[3 * NAME_LEN]; - char *bufend; + char buf[NAME_LEN + 1]; row_merge_dup_t dup = { sort_idx, table, col_map, 0}; @@ -4196,18 +4205,25 @@ wait_again: total_index_blocks)) / (total_static_cost + total_dynamic_cost) * PCT_COST_MERGESORT_INDEX * 100; - - bufend = innobase_convert_name( + char* bufend = innobase_convert_name( buf, sizeof buf, - indexes[i]->name, strlen(indexes[i]->name), + indexes[i]->name, + strlen(indexes[i]->name), trx->mysql_thd, FALSE); - buf[bufend - buf]='\0'; - sql_print_information("InnoDB: Online DDL : Start merge-sorting" - " index %s (%lu / %lu), estimated cost : %2.4f", - buf, (i+1), n_indexes, pct_cost); + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL :" + " Start merge-sorting" + " index %s" + " (" ULINTPF + " / " ULINTPF ")," + " estimated cost :" + " %2.4f", + buf, i + 1, n_indexes, + pct_cost); + } error = row_merge_sort( trx, &dup, &merge_files[i], @@ -4217,9 +4233,14 @@ wait_again: pct_progress += pct_cost; - sql_print_information("InnoDB: Online DDL : End of " - " merge-sorting index %s (%lu / %lu)", - buf, (i+1), n_indexes); + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL :" + " End of " + " merge-sorting index %s" + " (" ULINTPF + " / " ULINTPF ")", + buf, i + 1, n_indexes); + } DBUG_EXECUTE_IF( "ib_merge_wait_after_sort", @@ -4232,10 +4253,15 @@ wait_again: (total_static_cost + total_dynamic_cost) * PCT_COST_INSERT_INDEX * 100; - sql_print_information("InnoDB: Online DDL : Start " - "building index %s (%lu / %lu), estimated " - "cost : %2.4f", buf, (i+1), - n_indexes, pct_cost); + if (global_system_variables.log_warnings > 2) { + sql_print_information( + "InnoDB: Online DDL : Start " + "building index %s" + " (" ULINTPF + " / " ULINTPF "), estimated " + "cost : %2.4f", buf, i + 1, + n_indexes, pct_cost); + } error = row_merge_insert_index_tuples( trx->id, sort_idx, old_table, @@ -4244,9 +4270,13 @@ wait_again: crypt_data, crypt_block, new_table->space); pct_progress += pct_cost; - sql_print_information("InnoDB: Online DDL : " - "End of building index %s (%lu / %lu)", - buf, (i+1), n_indexes); + if (global_system_variables.log_warnings > 2) { + sql_print_information( + "InnoDB: Online DDL : " + "End of building index %s" + " (" ULINTPF " / " ULINTPF ")", + buf, i + 1, n_indexes); + } } } @@ -4263,15 +4293,16 @@ wait_again: ut_ad(sort_idx->online_status == ONLINE_INDEX_COMPLETE); } else { - sql_print_information("InnoDB: Online DDL : Start applying row log"); + if (global_system_variables.log_warnings > 2) { + sql_print_information( + "InnoDB: Online DDL : Applying" + " log to index"); + } DEBUG_SYNC_C("row_log_apply_before"); error = row_log_apply(trx, sort_idx, table); DEBUG_SYNC_C("row_log_apply_after"); - sql_print_information("InnoDB: Online DDL : End of applying row log"); } - sql_print_information("InnoDB: Online DDL : Completed"); - if (error != DB_SUCCESS) { trx->error_key_num = key_numbers[i]; goto func_exit; -- cgit v1.2.1 From d34a67b067606f61c9ec6700769770418d5bb47d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 19 Apr 2017 22:30:18 +0300 Subject: MDEV-12534 Use atomic operations whenever available Allow 64-bit atomic operations on 32-bit systems, only relying on HAVE_ATOMIC_BUILTINS_64, disregarding the width of the register file. Define UNIV_WORD_SIZE correctly on all systems, including Windows. In MariaDB 10.0 and 10.1, it was incorrectly defined as 4 on 64-bit Windows. Define HAVE_ATOMIC_BUILTINS_64 on Windows (64-bit atomics are available on both 32-bit and 64-bit Windows platforms; the operations were unnecessarily disabled even on 64-bit Windows). MONITOR_OS_PENDING_READS, MONITOR_OS_PENDING_WRITES: Enable by default. os_file_n_pending_preads, os_file_n_pending_pwrites, os_n_pending_reads, os_n_pending_writes: Remove. Use the monitor counters instead. os_file_count_mutex: Remove. On a system that does not support 64-bit atomics, monitor_mutex will be used instead. --- storage/innobase/buf/buf0rea.cc | 14 +-- storage/innobase/include/os0file.h | 12 +-- storage/innobase/include/os0sync.h | 5 +- storage/innobase/include/srv0mon.h | 62 +++++++++--- storage/innobase/include/univ.i | 12 +-- storage/innobase/os/os0file.cc | 198 +++++++++---------------------------- storage/innobase/srv/srv0mon.cc | 5 +- storage/innobase/srv/srv0srv.cc | 4 +- storage/innobase/sync/sync0arr.cc | 7 +- storage/xtradb/buf/buf0rea.cc | 14 +-- storage/xtradb/include/os0file.h | 12 +-- storage/xtradb/include/os0sync.h | 5 +- storage/xtradb/include/srv0mon.h | 62 +++++++++--- storage/xtradb/include/univ.i | 12 +-- storage/xtradb/os/os0file.cc | 183 +++++++++------------------------- storage/xtradb/srv/srv0mon.cc | 5 +- storage/xtradb/srv/srv0srv.cc | 4 +- storage/xtradb/sync/sync0arr.cc | 7 +- 18 files changed, 216 insertions(+), 407 deletions(-) (limited to 'storage') diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 7c8369c0c09..27f45308813 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -881,15 +881,11 @@ buf_read_recv_pages( count++; if (count > 1000) { - fprintf(stderr, - "InnoDB: Error: InnoDB has waited for" - " 10 seconds for pending\n" - "InnoDB: reads to the buffer pool to" - " be finished.\n" - "InnoDB: Number of pending reads %lu," - " pending pread calls %lu\n", - (ulong) buf_pool->n_pend_reads, - (ulong) os_file_n_pending_preads); + ib_logf(IB_LOG_LEVEL_ERROR, + "waited for 10 seconds for " ULINTPF + " pending reads to the buffer pool to" + " be finished", + buf_pool->n_pend_reads); os_aio_print_debug = TRUE; } diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h index 6cb4f54d629..bdb4682eafb 100644 --- a/storage/innobase/include/os0file.h +++ b/storage/innobase/include/os0file.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -51,16 +51,6 @@ extern ibool os_has_said_disk_full; /** Flag: enable debug printout for asynchronous i/o */ extern ibool os_aio_print_debug; -/** Number of pending os_file_pread() operations */ -extern ulint os_file_n_pending_preads; -/** Number of pending os_file_pwrite() operations */ -extern ulint os_file_n_pending_pwrites; - -/** Number of pending read operations */ -extern ulint os_n_pending_reads; -/** Number of pending write operations */ -extern ulint os_n_pending_writes; - #ifdef __WIN__ /** We define always WIN_ASYNC_IO, and check at run-time whether diff --git a/storage/innobase/include/os0sync.h b/storage/innobase/include/os0sync.h index 0754210c47a..dd6de43dc0b 100644 --- a/storage/innobase/include/os0sync.h +++ b/storage/innobase/include/os0sync.h @@ -667,10 +667,7 @@ os_atomic_clear(volatile lock_word_t* ptr) # define HAVE_ATOMIC_BUILTINS # define HAVE_ATOMIC_BUILTINS_BYTE - -# ifndef _WIN32 -# define HAVE_ATOMIC_BUILTINS_64 -# endif +# define HAVE_ATOMIC_BUILTINS_64 /**********************************************************//** Atomic compare and exchange of signed integers (both 32 and 64 bit). diff --git a/storage/innobase/include/srv0mon.h b/storage/innobase/include/srv0mon.h index 2d90f47eefe..09af5d4159b 100644 --- a/storage/innobase/include/srv0mon.h +++ b/storage/innobase/include/srv0mon.h @@ -2,6 +2,7 @@ Copyright (c) 2010, 2013, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the @@ -541,22 +542,30 @@ on the counters */ /** Increment a monitor counter under mutex protection. Use MONITOR_INC if appropriate mutex protection already exists. +@param mutex mutex to acquire and release @param monitor monitor to be incremented by 1 -@param mutex mutex to acquire and relese */ -# define MONITOR_MUTEX_INC(mutex, monitor) \ +@param enabled whether the monitor is enabled */ +#define MONITOR_MUTEX_INC_LOW(mutex, monitor, enabled) \ ut_ad(!mutex_own(mutex)); \ - if (MONITOR_IS_ON(monitor)) { \ + if (enabled) { \ mutex_enter(mutex); \ if (++MONITOR_VALUE(monitor) > MONITOR_MAX_VALUE(monitor)) { \ MONITOR_MAX_VALUE(monitor) = MONITOR_VALUE(monitor); \ } \ mutex_exit(mutex); \ } +/** Increment a monitor counter under mutex protection. +Use MONITOR_INC if appropriate mutex protection already exists. +@param mutex mutex to acquire and release +@param monitor monitor to be incremented by 1 */ +#define MONITOR_MUTEX_INC(mutex, monitor) \ + MONITOR_MUTEX_INC_LOW(mutex, monitor, MONITOR_IS_ON(monitor)) /** Decrement a monitor counter under mutex protection. Use MONITOR_DEC if appropriate mutex protection already exists. +@param mutex mutex to acquire and release @param monitor monitor to be decremented by 1 -@param mutex mutex to acquire and relese */ -# define MONITOR_MUTEX_DEC(mutex, monitor) \ +@param enabled whether the monitor is enabled */ +#define MONITOR_MUTEX_DEC_LOW(mutex, monitor, enabled) \ ut_ad(!mutex_own(mutex)); \ if (MONITOR_IS_ON(monitor)) { \ mutex_enter(mutex); \ @@ -565,13 +574,20 @@ Use MONITOR_DEC if appropriate mutex protection already exists. } \ mutex_exit(mutex); \ } +/** Decrement a monitor counter under mutex protection. +Use MONITOR_DEC if appropriate mutex protection already exists. +@param mutex mutex to acquire and release +@param monitor monitor to be decremented by 1 */ +#define MONITOR_MUTEX_DEC(mutex, monitor) \ + MONITOR_MUTEX_DEC_LOW(mutex, monitor, MONITOR_IS_ON(monitor)) #if defined HAVE_ATOMIC_BUILTINS_64 /** Atomically increment a monitor counter. Use MONITOR_INC if appropriate mutex protection exists. -@param monitor monitor to be incremented by 1 */ -# define MONITOR_ATOMIC_INC(monitor) \ - if (MONITOR_IS_ON(monitor)) { \ +@param monitor monitor to be incremented by 1 +@param enabled whether the monitor is enabled */ +# define MONITOR_ATOMIC_INC_LOW(monitor, enabled) \ + if (enabled) { \ ib_uint64_t value; \ value = os_atomic_increment_uint64( \ (ib_uint64_t*) &MONITOR_VALUE(monitor), 1); \ @@ -584,9 +600,10 @@ Use MONITOR_INC if appropriate mutex protection exists. /** Atomically decrement a monitor counter. Use MONITOR_DEC if appropriate mutex protection exists. -@param monitor monitor to be decremented by 1 */ -# define MONITOR_ATOMIC_DEC(monitor) \ - if (MONITOR_IS_ON(monitor)) { \ +@param monitor monitor to be decremented by 1 +@param enabled whether the monitor is enabled */ +# define MONITOR_ATOMIC_DEC_LOW(monitor, enabled) \ + if (enabled) { \ ib_uint64_t value; \ value = os_atomic_decrement_uint64( \ (ib_uint64_t*) &MONITOR_VALUE(monitor), 1); \ @@ -617,14 +634,29 @@ srv_mon_free(void); /** Atomically increment a monitor counter. Use MONITOR_INC if appropriate mutex protection exists. -@param monitor monitor to be incremented by 1 */ -# define MONITOR_ATOMIC_INC(monitor) MONITOR_MUTEX_INC(&monitor_mutex, monitor) +@param monitor monitor to be incremented by 1 +@param enabled whether the monitor is enabled */ +# define MONITOR_ATOMIC_INC_LOW(monitor, enabled) \ + MONITOR_MUTEX_INC_LOW(&monitor_mutex, monitor, enabled) /** Atomically decrement a monitor counter. Use MONITOR_DEC if appropriate mutex protection exists. -@param monitor monitor to be decremented by 1 */ -# define MONITOR_ATOMIC_DEC(monitor) MONITOR_MUTEX_DEC(&monitor_mutex, monitor) +@param monitor monitor to be decremented by 1 +@param enabled whether the monitor is enabled */ +# define MONITOR_ATOMIC_DEC_LOW(monitor, enabled) \ + MONITOR_MUTEX_DEC_LOW(&monitor_mutex, monitor, enabled) #endif /* HAVE_ATOMIC_BUILTINS_64 */ +/** Atomically increment a monitor counter if it is enabled. +Use MONITOR_INC if appropriate mutex protection exists. +@param monitor monitor to be incremented by 1 */ +#define MONITOR_ATOMIC_INC(monitor) \ + MONITOR_ATOMIC_INC_LOW(monitor, MONITOR_IS_ON(monitor)) +/** Atomically decrement a monitor counter if it is enabled. +Use MONITOR_DEC if appropriate mutex protection exists. +@param monitor monitor to be decremented by 1 */ +#define MONITOR_ATOMIC_DEC(monitor) \ + MONITOR_ATOMIC_DEC_LOW(monitor, MONITOR_IS_ON(monitor)) + #define MONITOR_DEC(monitor) \ if (MONITOR_IS_ON(monitor)) { \ MONITOR_VALUE(monitor)--; \ diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index 824a06829f9..fc0bf0aeeca 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -285,22 +285,12 @@ definitions: */ #endif /* !UNIV_MUST_NOT_INLINE */ -#ifdef _WIN32 -#define UNIV_WORD_SIZE 4 -#elif defined(_WIN64) -#define UNIV_WORD_SIZE 8 -#else -/** MySQL config.h generated by GNU autoconf will define SIZEOF_LONG in Posix */ -#define UNIV_WORD_SIZE SIZEOF_LONG -#endif +#define UNIV_WORD_SIZE SIZEOF_SIZE_T /** The following alignment is used in memory allocations in memory heap management to ensure correct alignment for doubles etc. */ #define UNIV_MEM_ALIGNMENT 8 -/** The following alignment is used in aligning lints etc. */ -#define UNIV_WORD_ALIGNMENT UNIV_WORD_SIZE - /* DATABASE VERSION CONTROL ======================== diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index eb8b3a53d32..ff67c90eecf 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. -Copyright (c) 2012, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -284,21 +284,6 @@ UNIV_INTERN time_t os_last_printout; UNIV_INTERN ibool os_has_said_disk_full = FALSE; -#if !defined(UNIV_HOTBACKUP) \ - && (!defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8) -/** The mutex protecting the following counts of pending I/O operations */ -static os_ib_mutex_t os_file_count_mutex; -#endif /* !UNIV_HOTBACKUP && (!HAVE_ATOMIC_BUILTINS || UNIV_WORD_SIZE < 8) */ - -/** Number of pending os_file_pread() operations */ -UNIV_INTERN ulint os_file_n_pending_preads = 0; -/** Number of pending os_file_pwrite() operations */ -UNIV_INTERN ulint os_file_n_pending_pwrites = 0; -/** Number of pending write operations */ -UNIV_INTERN ulint os_n_pending_writes = 0; -/** Number of pending read operations */ -UNIV_INTERN ulint os_n_pending_reads = 0; - #ifdef UNIV_DEBUG # ifndef UNIV_HOTBACKUP /**********************************************************************//** @@ -752,10 +737,6 @@ void os_io_init_simple(void) /*===================*/ { -#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 - os_file_count_mutex = os_mutex_create(); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD_SIZE < 8 */ - for (ulint i = 0; i < OS_FILE_N_SEEK_MUTEXES; i++) { os_file_seek_mutexes[i] = os_mutex_create(); } @@ -2364,9 +2345,6 @@ os_file_pread( os_offset_t offset) /*!< in: file offset from where to read */ { off_t offs; -#if defined(HAVE_PREAD) && !defined(HAVE_BROKEN_PREAD) - ssize_t n_bytes; -#endif /* HAVE_PREAD && !HAVE_BROKEN_PREAD */ ut_ad(n); @@ -2383,33 +2361,12 @@ os_file_pread( os_n_file_reads++; -#if defined(HAVE_PREAD) && !defined(HAVE_BROKEN_PREAD) -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_increment_ulint(&os_n_pending_reads, 1); - (void) os_atomic_increment_ulint(&os_file_n_pending_preads, 1); - MONITOR_ATOMIC_INC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_preads++; - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* HAVE_ATOMIC_BUILTINS && UNIV_WORD == 8 */ - - n_bytes = pread(file, buf, n, offs); - -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_decrement_ulint(&os_n_pending_reads, 1); - (void) os_atomic_decrement_ulint(&os_file_n_pending_preads, 1); - MONITOR_ATOMIC_DEC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_preads--; - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD == 8 */ + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_READS); +#ifdef HAVE_PREAD + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_READS, monitor); + ssize_t n_bytes = pread(file, buf, n, offs); + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); return(n_bytes); #else { @@ -2419,15 +2376,7 @@ os_file_pread( ulint i; #endif /* !UNIV_HOTBACKUP */ -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_increment_ulint(&os_n_pending_reads, 1); - MONITOR_ATOMIC_INC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* HAVE_ATOMIC_BUILTINS && UNIV_WORD == 8 */ + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_READS, monitor); #ifndef UNIV_HOTBACKUP /* Protect the seek / read operation with a mutex */ i = ((ulint) file) % OS_FILE_N_SEEK_MUTEXES; @@ -2447,16 +2396,7 @@ os_file_pread( os_mutex_exit(os_file_seek_mutexes[i]); #endif /* !UNIV_HOTBACKUP */ -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_decrement_ulint(&os_n_pending_reads, 1); - MONITOR_ATOIC_DEC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* HAVE_ATOMIC_BUILTINS && UNIV_WORD_SIZE == 8 */ - + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); return(ret); } #endif @@ -2493,32 +2433,11 @@ os_file_pwrite( os_n_file_writes++; -#if defined(HAVE_PWRITE) && !defined(HAVE_BROKEN_PREAD) -#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_pwrites++; - os_n_pending_writes++; - MONITOR_INC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); -#else - (void) os_atomic_increment_ulint(&os_n_pending_writes, 1); - (void) os_atomic_increment_ulint(&os_file_n_pending_pwrites, 1); - MONITOR_ATOMIC_INC(MONITOR_OS_PENDING_WRITES); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD < 8 */ - + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_WRITES); +#ifdef HAVE_PWRITE + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_WRITES, monitor); ret = pwrite(file, buf, (ssize_t) n, offs); - -#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_pwrites--; - os_n_pending_writes--; - MONITOR_DEC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); -#else - (void) os_atomic_decrement_ulint(&os_n_pending_writes, 1); - (void) os_atomic_decrement_ulint(&os_file_n_pending_pwrites, 1); - MONITOR_ATOMIC_DEC(MONITOR_OS_PENDING_WRITES); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD < 8 */ + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_WRITES, monitor); return(ret); #else @@ -2528,10 +2447,7 @@ os_file_pwrite( ulint i; # endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes++; - MONITOR_INC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_WRITES, monitor); # ifndef UNIV_HOTBACKUP /* Protect the seek / write operation with a mutex */ @@ -2555,14 +2471,10 @@ func_exit: os_mutex_exit(os_file_seek_mutexes[i]); # endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes--; - MONITOR_DEC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); - + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_WRITES, monitor); return(ret); } -#endif /* !UNIV_HOTBACKUP */ +#endif /* HAVE_PWRITE */ } #endif @@ -2597,6 +2509,7 @@ os_file_read_func( os_n_file_reads++; os_bytes_read_since_printout += n; + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_READS); try_again: ut_ad(buf); @@ -2605,10 +2518,7 @@ try_again: low = (DWORD) offset & 0xFFFFFFFF; high = (DWORD) (offset >> 32); - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_READS, monitor); #ifndef UNIV_HOTBACKUP /* Protect the seek / read operation with a mutex */ @@ -2626,11 +2536,7 @@ try_again: os_mutex_exit(os_file_seek_mutexes[i]); #endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); - + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); goto error_handling; } @@ -2640,10 +2546,7 @@ try_again: os_mutex_exit(os_file_seek_mutexes[i]); #endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); if (ret && len == n) { return(TRUE); @@ -2728,6 +2631,7 @@ os_file_read_no_error_handling_func( os_n_file_reads++; os_bytes_read_since_printout += n; + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_READS); try_again: ut_ad(buf); @@ -2736,10 +2640,7 @@ try_again: low = (DWORD) offset & 0xFFFFFFFF; high = (DWORD) (offset >> 32); - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_READS, monitor); #ifndef UNIV_HOTBACKUP /* Protect the seek / read operation with a mutex */ @@ -2757,11 +2658,7 @@ try_again: os_mutex_exit(os_file_seek_mutexes[i]); #endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); - + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); goto error_handling; } @@ -2771,10 +2668,7 @@ try_again: os_mutex_exit(os_file_seek_mutexes[i]); #endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); if (ret && len == n) { return(TRUE); @@ -2854,7 +2748,6 @@ os_file_write_func( ulint n) /*!< in: number of bytes to write */ { ut_ad(!srv_read_only_mode); - #ifdef __WIN__ BOOL ret; DWORD len; @@ -2876,14 +2769,12 @@ os_file_write_func( ut_ad(buf); ut_ad(n > 0); + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_WRITES); retry: low = (DWORD) offset & 0xFFFFFFFF; high = (DWORD) (offset >> 32); - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes++; - MONITOR_INC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_WRITES, monitor); #ifndef UNIV_HOTBACKUP /* Protect the seek / write operation with a mutex */ @@ -2901,10 +2792,7 @@ retry: os_mutex_exit(os_file_seek_mutexes[i]); #endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes--; - MONITOR_DEC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_WRITES, monitor); ut_print_timestamp(stderr); @@ -2928,10 +2816,7 @@ retry: os_mutex_exit(os_file_seek_mutexes[i]); #endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes--; - MONITOR_DEC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_WRITES, monitor); if (ret && len == n) { @@ -5751,19 +5636,24 @@ os_aio_print( time_elapsed = 0.001 + difftime(current_time, os_last_printout); fprintf(file, - "Pending flushes (fsync) log: %lu; buffer pool: %lu\n" - "%lu OS file reads, %lu OS file writes, %lu OS fsyncs\n", - (ulong) fil_n_pending_log_flushes, - (ulong) fil_n_pending_tablespace_flushes, - (ulong) os_n_file_reads, - (ulong) os_n_file_writes, - (ulong) os_n_fsyncs); - - if (os_file_n_pending_preads != 0 || os_file_n_pending_pwrites != 0) { + "Pending flushes (fsync) log: " ULINTPF + "; buffer pool: " ULINTPF "\n" + ULINTPF " OS file reads, " + ULINTPF " OS file writes, " + ULINTPF " OS fsyncs\n", + fil_n_pending_log_flushes, + fil_n_pending_tablespace_flushes, + os_n_file_reads, + os_n_file_writes, + os_n_fsyncs); + + const ulint n_reads = MONITOR_VALUE(MONITOR_OS_PENDING_READS); + const ulint n_writes = MONITOR_VALUE(MONITOR_OS_PENDING_WRITES); + + if (n_reads != 0 || n_writes != 0) { fprintf(file, - "%lu pending preads, %lu pending pwrites\n", - (ulong) os_file_n_pending_preads, - (ulong) os_file_n_pending_pwrites); + ULINTPF " pending reads, " ULINTPF " pending writes\n", + n_reads, n_writes); } if (os_n_file_reads == os_n_file_reads_old) { diff --git a/storage/innobase/srv/srv0mon.cc b/storage/innobase/srv/srv0mon.cc index 4a709160ea6..ea21a4c1454 100644 --- a/storage/innobase/srv/srv0mon.cc +++ b/storage/innobase/srv/srv0mon.cc @@ -2,6 +2,7 @@ Copyright (c) 2010, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -643,11 +644,11 @@ static monitor_info_t innodb_counter_info[] = MONITOR_DEFAULT_START, MONITOR_OVLD_OS_FSYNC}, {"os_pending_reads", "os", "Number of reads pending", - MONITOR_NONE, + MONITOR_DEFAULT_ON, MONITOR_DEFAULT_START, MONITOR_OS_PENDING_READS}, {"os_pending_writes", "os", "Number of writes pending", - MONITOR_NONE, + MONITOR_DEFAULT_ON, MONITOR_DEFAULT_START, MONITOR_OS_PENDING_WRITES}, {"os_log_bytes_written", "os", diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 723ae838f07..58e69c3c9e7 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -1409,10 +1409,10 @@ srv_export_innodb_status(void) mutex_enter(&srv_innodb_monitor_mutex); export_vars.innodb_data_pending_reads = - os_n_pending_reads; + MONITOR_VALUE(MONITOR_OS_PENDING_READS); export_vars.innodb_data_pending_writes = - os_n_pending_writes; + MONITOR_VALUE(MONITOR_OS_PENDING_WRITES); export_vars.innodb_data_pending_fsyncs = fil_n_pending_log_flushes diff --git a/storage/innobase/sync/sync0arr.cc b/storage/innobase/sync/sync0arr.cc index c7163695a3f..6ffe3cab8c8 100644 --- a/storage/innobase/sync/sync0arr.cc +++ b/storage/innobase/sync/sync0arr.cc @@ -1069,9 +1069,10 @@ sync_array_print_long_waits( now the values of pending calls of these. */ fprintf(stderr, - "InnoDB: Pending preads %lu, pwrites %lu\n", - (ulong) os_file_n_pending_preads, - (ulong) os_file_n_pending_pwrites); + "InnoDB: Pending reads " UINT64PF + ", writes " UINT64PF "\n", + MONITOR_VALUE(MONITOR_OS_PENDING_READS), + MONITOR_VALUE(MONITOR_OS_PENDING_WRITES)); srv_print_innodb_monitor = TRUE; os_event_set(srv_monitor_event); diff --git a/storage/xtradb/buf/buf0rea.cc b/storage/xtradb/buf/buf0rea.cc index c28df72df92..9053f38924e 100644 --- a/storage/xtradb/buf/buf0rea.cc +++ b/storage/xtradb/buf/buf0rea.cc @@ -986,15 +986,11 @@ not_to_recover: count++; if (count > 1000) { - fprintf(stderr, - "InnoDB: Error: InnoDB has waited for" - " 10 seconds for pending\n" - "InnoDB: reads to the buffer pool to" - " be finished.\n" - "InnoDB: Number of pending reads %lu," - " pending pread calls %lu\n", - (ulong) buf_pool->n_pend_reads, - (ulong) os_file_n_pending_preads); + ib_logf(IB_LOG_LEVEL_ERROR, + "waited for 10 seconds for " ULINTPF + " pending reads to the buffer pool to" + " be finished", + buf_pool->n_pend_reads); os_aio_print_debug = TRUE; } diff --git a/storage/xtradb/include/os0file.h b/storage/xtradb/include/os0file.h index 60a9c3475e7..b8f30977fbe 100644 --- a/storage/xtradb/include/os0file.h +++ b/storage/xtradb/include/os0file.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -52,16 +52,6 @@ extern ibool os_has_said_disk_full; /** Flag: enable debug printout for asynchronous i/o */ extern ibool os_aio_print_debug; -/** Number of pending os_file_pread() operations */ -extern ulint os_file_n_pending_preads; -/** Number of pending os_file_pwrite() operations */ -extern ulint os_file_n_pending_pwrites; - -/** Number of pending read operations */ -extern ulint os_n_pending_reads; -/** Number of pending write operations */ -extern ulint os_n_pending_writes; - #ifdef __WIN__ /** We define always WIN_ASYNC_IO, and check at run-time whether diff --git a/storage/xtradb/include/os0sync.h b/storage/xtradb/include/os0sync.h index f6207555f1a..48c56a73369 100644 --- a/storage/xtradb/include/os0sync.h +++ b/storage/xtradb/include/os0sync.h @@ -718,10 +718,7 @@ os_atomic_clear(volatile lock_word_t* ptr) # define HAVE_ATOMIC_BUILTINS # define HAVE_ATOMIC_BUILTINS_BYTE - -# ifndef _WIN32 -# define HAVE_ATOMIC_BUILTINS_64 -# endif +# define HAVE_ATOMIC_BUILTINS_64 /**********************************************************//** Atomic compare and exchange of signed integers (both 32 and 64 bit). diff --git a/storage/xtradb/include/srv0mon.h b/storage/xtradb/include/srv0mon.h index 2d90f47eefe..09af5d4159b 100644 --- a/storage/xtradb/include/srv0mon.h +++ b/storage/xtradb/include/srv0mon.h @@ -2,6 +2,7 @@ Copyright (c) 2010, 2013, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the @@ -541,22 +542,30 @@ on the counters */ /** Increment a monitor counter under mutex protection. Use MONITOR_INC if appropriate mutex protection already exists. +@param mutex mutex to acquire and release @param monitor monitor to be incremented by 1 -@param mutex mutex to acquire and relese */ -# define MONITOR_MUTEX_INC(mutex, monitor) \ +@param enabled whether the monitor is enabled */ +#define MONITOR_MUTEX_INC_LOW(mutex, monitor, enabled) \ ut_ad(!mutex_own(mutex)); \ - if (MONITOR_IS_ON(monitor)) { \ + if (enabled) { \ mutex_enter(mutex); \ if (++MONITOR_VALUE(monitor) > MONITOR_MAX_VALUE(monitor)) { \ MONITOR_MAX_VALUE(monitor) = MONITOR_VALUE(monitor); \ } \ mutex_exit(mutex); \ } +/** Increment a monitor counter under mutex protection. +Use MONITOR_INC if appropriate mutex protection already exists. +@param mutex mutex to acquire and release +@param monitor monitor to be incremented by 1 */ +#define MONITOR_MUTEX_INC(mutex, monitor) \ + MONITOR_MUTEX_INC_LOW(mutex, monitor, MONITOR_IS_ON(monitor)) /** Decrement a monitor counter under mutex protection. Use MONITOR_DEC if appropriate mutex protection already exists. +@param mutex mutex to acquire and release @param monitor monitor to be decremented by 1 -@param mutex mutex to acquire and relese */ -# define MONITOR_MUTEX_DEC(mutex, monitor) \ +@param enabled whether the monitor is enabled */ +#define MONITOR_MUTEX_DEC_LOW(mutex, monitor, enabled) \ ut_ad(!mutex_own(mutex)); \ if (MONITOR_IS_ON(monitor)) { \ mutex_enter(mutex); \ @@ -565,13 +574,20 @@ Use MONITOR_DEC if appropriate mutex protection already exists. } \ mutex_exit(mutex); \ } +/** Decrement a monitor counter under mutex protection. +Use MONITOR_DEC if appropriate mutex protection already exists. +@param mutex mutex to acquire and release +@param monitor monitor to be decremented by 1 */ +#define MONITOR_MUTEX_DEC(mutex, monitor) \ + MONITOR_MUTEX_DEC_LOW(mutex, monitor, MONITOR_IS_ON(monitor)) #if defined HAVE_ATOMIC_BUILTINS_64 /** Atomically increment a monitor counter. Use MONITOR_INC if appropriate mutex protection exists. -@param monitor monitor to be incremented by 1 */ -# define MONITOR_ATOMIC_INC(monitor) \ - if (MONITOR_IS_ON(monitor)) { \ +@param monitor monitor to be incremented by 1 +@param enabled whether the monitor is enabled */ +# define MONITOR_ATOMIC_INC_LOW(monitor, enabled) \ + if (enabled) { \ ib_uint64_t value; \ value = os_atomic_increment_uint64( \ (ib_uint64_t*) &MONITOR_VALUE(monitor), 1); \ @@ -584,9 +600,10 @@ Use MONITOR_INC if appropriate mutex protection exists. /** Atomically decrement a monitor counter. Use MONITOR_DEC if appropriate mutex protection exists. -@param monitor monitor to be decremented by 1 */ -# define MONITOR_ATOMIC_DEC(monitor) \ - if (MONITOR_IS_ON(monitor)) { \ +@param monitor monitor to be decremented by 1 +@param enabled whether the monitor is enabled */ +# define MONITOR_ATOMIC_DEC_LOW(monitor, enabled) \ + if (enabled) { \ ib_uint64_t value; \ value = os_atomic_decrement_uint64( \ (ib_uint64_t*) &MONITOR_VALUE(monitor), 1); \ @@ -617,14 +634,29 @@ srv_mon_free(void); /** Atomically increment a monitor counter. Use MONITOR_INC if appropriate mutex protection exists. -@param monitor monitor to be incremented by 1 */ -# define MONITOR_ATOMIC_INC(monitor) MONITOR_MUTEX_INC(&monitor_mutex, monitor) +@param monitor monitor to be incremented by 1 +@param enabled whether the monitor is enabled */ +# define MONITOR_ATOMIC_INC_LOW(monitor, enabled) \ + MONITOR_MUTEX_INC_LOW(&monitor_mutex, monitor, enabled) /** Atomically decrement a monitor counter. Use MONITOR_DEC if appropriate mutex protection exists. -@param monitor monitor to be decremented by 1 */ -# define MONITOR_ATOMIC_DEC(monitor) MONITOR_MUTEX_DEC(&monitor_mutex, monitor) +@param monitor monitor to be decremented by 1 +@param enabled whether the monitor is enabled */ +# define MONITOR_ATOMIC_DEC_LOW(monitor, enabled) \ + MONITOR_MUTEX_DEC_LOW(&monitor_mutex, monitor, enabled) #endif /* HAVE_ATOMIC_BUILTINS_64 */ +/** Atomically increment a monitor counter if it is enabled. +Use MONITOR_INC if appropriate mutex protection exists. +@param monitor monitor to be incremented by 1 */ +#define MONITOR_ATOMIC_INC(monitor) \ + MONITOR_ATOMIC_INC_LOW(monitor, MONITOR_IS_ON(monitor)) +/** Atomically decrement a monitor counter if it is enabled. +Use MONITOR_DEC if appropriate mutex protection exists. +@param monitor monitor to be decremented by 1 */ +#define MONITOR_ATOMIC_DEC(monitor) \ + MONITOR_ATOMIC_DEC_LOW(monitor, MONITOR_IS_ON(monitor)) + #define MONITOR_DEC(monitor) \ if (MONITOR_IS_ON(monitor)) { \ MONITOR_VALUE(monitor)--; \ diff --git a/storage/xtradb/include/univ.i b/storage/xtradb/include/univ.i index 6acb0eec319..7497e7f5d05 100644 --- a/storage/xtradb/include/univ.i +++ b/storage/xtradb/include/univ.i @@ -304,22 +304,12 @@ definitions: */ #endif /* !UNIV_MUST_NOT_INLINE */ -#ifdef _WIN32 -#define UNIV_WORD_SIZE 4 -#elif defined(_WIN64) -#define UNIV_WORD_SIZE 8 -#else -/** MySQL config.h generated by GNU autoconf will define SIZEOF_LONG in Posix */ -#define UNIV_WORD_SIZE SIZEOF_LONG -#endif +#define UNIV_WORD_SIZE SIZEOF_SIZE_T /** The following alignment is used in memory allocations in memory heap management to ensure correct alignment for doubles etc. */ #define UNIV_MEM_ALIGNMENT 8 -/** The following alignment is used in aligning lints etc. */ -#define UNIV_WORD_ALIGNMENT UNIV_WORD_SIZE - /* DATABASE VERSION CONTROL ======================== diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index a30b688ad74..48ae49f28e4 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -291,21 +291,6 @@ UNIV_INTERN time_t os_last_printout; UNIV_INTERN ibool os_has_said_disk_full = FALSE; -#if !defined(UNIV_HOTBACKUP) \ - && (!defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8) -/** The mutex protecting the following counts of pending I/O operations */ -static os_ib_mutex_t os_file_count_mutex; -#endif /* !UNIV_HOTBACKUP && (!HAVE_ATOMIC_BUILTINS || UNIV_WORD_SIZE < 8) */ - -/** Number of pending os_file_pread() operations */ -UNIV_INTERN ulint os_file_n_pending_preads = 0; -/** Number of pending os_file_pwrite() operations */ -UNIV_INTERN ulint os_file_n_pending_pwrites = 0; -/** Number of pending write operations */ -UNIV_INTERN ulint os_n_pending_writes = 0; -/** Number of pending read operations */ -UNIV_INTERN ulint os_n_pending_reads = 0; - #ifdef UNIV_DEBUG # ifndef UNIV_HOTBACKUP /**********************************************************************//** @@ -887,10 +872,6 @@ void os_io_init_simple(void) /*===================*/ { -#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 - os_file_count_mutex = os_mutex_create(); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD_SIZE < 8 */ - for (ulint i = 0; i < OS_FILE_N_SEEK_MUTEXES; i++) { os_file_seek_mutexes[i] = os_mutex_create(); } @@ -2580,10 +2561,6 @@ os_file_pread( trx_t* trx) { off_t offs; -#if defined(HAVE_PREAD) && !defined(HAVE_BROKEN_PREAD) - ssize_t n_bytes; - ssize_t n_read; -#endif /* HAVE_PREAD && !HAVE_BROKEN_PREAD */ ulint sec; ulint ms; ib_uint64_t start_time; @@ -2613,22 +2590,16 @@ os_file_pread( } else { start_time = 0; } -#if defined(HAVE_PREAD) && !defined(HAVE_BROKEN_PREAD) -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_increment_ulint(&os_n_pending_reads, 1); - (void) os_atomic_increment_ulint(&os_file_n_pending_preads, 1); - MONITOR_ATOMIC_INC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_preads++; - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* HAVE_ATOMIC_BUILTINS && UNIV_WORD == 8 */ + + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_READS); +#ifdef HAVE_PREAD + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_READS, monitor); + + ssize_t n_bytes; /* Handle partial reads and signal interruptions correctly */ for (n_bytes = 0; n_bytes < (ssize_t) n; ) { - n_read = pread(file, buf, (ssize_t)n - n_bytes, offs); + ssize_t n_read = pread(file, buf, (ssize_t)n - n_bytes, offs); if (n_read > 0) { n_bytes += n_read; offs += n_read; @@ -2640,17 +2611,7 @@ os_file_pread( } } -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_decrement_ulint(&os_n_pending_reads, 1); - (void) os_atomic_decrement_ulint(&os_file_n_pending_preads, 1); - MONITOR_ATOMIC_DEC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_preads--; - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD == 8 */ + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); if (UNIV_UNLIKELY(start_time != 0)) { @@ -2669,15 +2630,7 @@ os_file_pread( ulint i; #endif /* !UNIV_HOTBACKUP */ -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_increment_ulint(&os_n_pending_reads, 1); - MONITOR_ATOMIC_INC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* HAVE_ATOMIC_BUILTINS && UNIV_WORD == 8 */ + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_READS, monitor); #ifndef UNIV_HOTBACKUP /* Protect the seek / read operation with a mutex */ i = ((ulint) file) % OS_FILE_N_SEEK_MUTEXES; @@ -2707,15 +2660,7 @@ os_file_pread( os_mutex_exit(os_file_seek_mutexes[i]); #endif /* !UNIV_HOTBACKUP */ -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_decrement_ulint(&os_n_pending_reads, 1); - MONITOR_ATOIC_DEC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* HAVE_ATOMIC_BUILTINS && UNIV_WORD_SIZE == 8 */ + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); if (UNIV_UNLIKELY(start_time != 0) { @@ -2761,18 +2706,9 @@ os_file_pwrite( os_n_file_writes++; -#if defined(HAVE_PWRITE) && !defined(HAVE_BROKEN_PREAD) -#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_pwrites++; - os_n_pending_writes++; - MONITOR_INC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); -#else - (void) os_atomic_increment_ulint(&os_n_pending_writes, 1); - (void) os_atomic_increment_ulint(&os_file_n_pending_pwrites, 1); - MONITOR_ATOMIC_INC(MONITOR_OS_PENDING_WRITES); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD < 8 */ + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_WRITES); +#ifdef HAVE_PWRITE + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_WRITES, monitor); /* Handle partial writes and signal interruptions correctly */ for (ret = 0; ret < (ssize_t) n; ) { @@ -2791,17 +2727,7 @@ os_file_pwrite( } } -#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_pwrites--; - os_n_pending_writes--; - MONITOR_DEC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); -#else - (void) os_atomic_decrement_ulint(&os_n_pending_writes, 1); - (void) os_atomic_decrement_ulint(&os_file_n_pending_pwrites, 1); - MONITOR_ATOMIC_DEC(MONITOR_OS_PENDING_WRITES); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD < 8 */ + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_WRITES, monitor); return(ret); #else @@ -2811,10 +2737,7 @@ os_file_pwrite( ulint i; # endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes++; - MONITOR_INC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_WRITES, monitor); # ifndef UNIV_HOTBACKUP /* Protect the seek / write operation with a mutex */ @@ -2848,14 +2771,10 @@ func_exit: os_mutex_exit(os_file_seek_mutexes[i]); # endif /* !UNIV_HOTBACKUP */ - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes--; - MONITOR_DEC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); - + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_WRITES, monitor); return(ret); } -#endif /* !UNIV_HOTBACKUP */ +#endif /* HAVE_PWRITE */ } #endif @@ -2887,15 +2806,13 @@ os_file_read_func( os_n_file_reads++; os_bytes_read_since_printout += n; + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_READS); try_again: ut_ad(buf); ut_ad(n > 0); - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_READS, monitor); memset (&overlapped, 0, sizeof (overlapped)); overlapped.Offset = (DWORD)(offset & 0xFFFFFFFF); @@ -2908,10 +2825,7 @@ try_again: else if(GetLastError() == ERROR_IO_PENDING) { ret = GetOverlappedResult(file, &overlapped, (DWORD *)&len, TRUE); } - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); if (ret && len == n) { return(TRUE); @@ -2995,15 +2909,13 @@ os_file_read_no_error_handling_func( os_n_file_reads++; os_bytes_read_since_printout += n; + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_READS); try_again: ut_ad(buf); ut_ad(n > 0); - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_READS, monitor); memset (&overlapped, 0, sizeof (overlapped)); overlapped.Offset = (DWORD)(offset & 0xFFFFFFFF); @@ -3016,10 +2928,7 @@ try_again: else if(GetLastError() == ERROR_IO_PENDING) { ret = GetOverlappedResult(file, &overlapped, (DWORD *)&len, TRUE); } - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_READS, monitor); if (ret && len == n) { return(TRUE); @@ -3113,12 +3022,10 @@ os_file_write_func( ut_ad(buf); ut_ad(n > 0); + const bool monitor = MONITOR_IS_ON(MONITOR_OS_PENDING_WRITES); retry: - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes++; - MONITOR_INC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_INC_LOW(MONITOR_OS_PENDING_WRITES, monitor); memset (&overlapped, 0, sizeof (overlapped)); overlapped.Offset = (DWORD)(offset & 0xFFFFFFFF); @@ -3133,10 +3040,7 @@ retry: ret = GetOverlappedResult(file, &overlapped, (DWORD *)&len, TRUE); } - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes--; - MONITOR_DEC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); + MONITOR_ATOMIC_DEC_LOW(MONITOR_OS_PENDING_WRITES, monitor); if (ret && len == n) { @@ -4230,10 +4134,6 @@ os_aio_free(void) } } -#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 - os_mutex_free(os_file_count_mutex); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD_SIZE < 8 */ - for (ulint i = 0; i < OS_FILE_N_SEEK_MUTEXES; i++) { os_mutex_free(os_file_seek_mutexes[i]); } @@ -5909,19 +5809,24 @@ os_aio_print( time_elapsed = 0.001 + difftime(current_time, os_last_printout); fprintf(file, - "Pending flushes (fsync) log: %lu; buffer pool: %lu\n" - "%lu OS file reads, %lu OS file writes, %lu OS fsyncs\n", - (ulong) fil_n_pending_log_flushes, - (ulong) fil_n_pending_tablespace_flushes, - (ulong) os_n_file_reads, - (ulong) os_n_file_writes, - (ulong) os_n_fsyncs); - - if (os_file_n_pending_preads != 0 || os_file_n_pending_pwrites != 0) { + "Pending flushes (fsync) log: " ULINTPF + "; buffer pool: " ULINTPF "\n" + ULINTPF " OS file reads, " + ULINTPF " OS file writes, " + ULINTPF " OS fsyncs\n", + fil_n_pending_log_flushes, + fil_n_pending_tablespace_flushes, + os_n_file_reads, + os_n_file_writes, + os_n_fsyncs); + + const ulint n_reads = MONITOR_VALUE(MONITOR_OS_PENDING_READS); + const ulint n_writes = MONITOR_VALUE(MONITOR_OS_PENDING_WRITES); + + if (n_reads != 0 || n_writes != 0) { fprintf(file, - "%lu pending preads, %lu pending pwrites\n", - (ulong) os_file_n_pending_preads, - (ulong) os_file_n_pending_pwrites); + ULINTPF " pending reads, " ULINTPF " pending writes\n", + n_reads, n_writes); } if (os_n_file_reads == os_n_file_reads_old) { diff --git a/storage/xtradb/srv/srv0mon.cc b/storage/xtradb/srv/srv0mon.cc index 4a709160ea6..ea21a4c1454 100644 --- a/storage/xtradb/srv/srv0mon.cc +++ b/storage/xtradb/srv/srv0mon.cc @@ -2,6 +2,7 @@ Copyright (c) 2010, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -643,11 +644,11 @@ static monitor_info_t innodb_counter_info[] = MONITOR_DEFAULT_START, MONITOR_OVLD_OS_FSYNC}, {"os_pending_reads", "os", "Number of reads pending", - MONITOR_NONE, + MONITOR_DEFAULT_ON, MONITOR_DEFAULT_START, MONITOR_OS_PENDING_READS}, {"os_pending_writes", "os", "Number of writes pending", - MONITOR_NONE, + MONITOR_DEFAULT_ON, MONITOR_DEFAULT_START, MONITOR_OS_PENDING_WRITES}, {"os_log_bytes_written", "os", diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index ffa1e36ebcd..c47119e63cf 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -1735,10 +1735,10 @@ srv_export_innodb_status(void) mutex_enter(&srv_innodb_monitor_mutex); export_vars.innodb_data_pending_reads = - os_n_pending_reads; + MONITOR_VALUE(MONITOR_OS_PENDING_READS); export_vars.innodb_data_pending_writes = - os_n_pending_writes; + MONITOR_VALUE(MONITOR_OS_PENDING_WRITES); export_vars.innodb_data_pending_fsyncs = fil_n_pending_log_flushes diff --git a/storage/xtradb/sync/sync0arr.cc b/storage/xtradb/sync/sync0arr.cc index e16eddae80c..7325cb49145 100644 --- a/storage/xtradb/sync/sync0arr.cc +++ b/storage/xtradb/sync/sync0arr.cc @@ -1161,9 +1161,10 @@ sync_array_print_long_waits( now the values of pending calls of these. */ fprintf(stderr, - "InnoDB: Pending preads %lu, pwrites %lu\n", - (ulong) os_file_n_pending_preads, - (ulong) os_file_n_pending_pwrites); + "InnoDB: Pending reads " UINT64PF + ", writes " UINT64PF "\n", + MONITOR_VALUE(MONITOR_OS_PENDING_READS), + MONITOR_VALUE(MONITOR_OS_PENDING_WRITES)); srv_print_innodb_monitor = TRUE; os_event_set(srv_monitor_event); -- cgit v1.2.1 From 87b6df31c4ae97c09dc67e370978112fe39764bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 21 Apr 2017 04:36:50 +0300 Subject: MDEV-12488 Remove type mismatch in InnoDB printf-like calls This is a reduced version of an originally much larger patch. We will keep the definition of the ulint, lint data types unchanged, and we will not be replacing fprintf() calls with ib_logf(). On Windows, use the standard format strings instead of nonstandard extensions. This patch fixes some errors in format strings. Most notably, an IMPORT TABLESPACE error message in InnoDB was displaying the number of columns instead of the mismatching flags. --- storage/innobase/include/univ.i | 16 ++--- storage/innobase/log/log0recv.cc | 6 +- storage/innobase/row/row0import.cc | 128 ++++++++++++++++++------------------ storage/innobase/row/row0merge.cc | 65 +++++++++--------- storage/xtradb/include/univ.i | 16 ++--- storage/xtradb/log/log0recv.cc | 6 +- storage/xtradb/row/row0import.cc | 131 ++++++++++++++++++------------------- storage/xtradb/row/row0merge.cc | 66 +++++++++---------- 8 files changed, 208 insertions(+), 226 deletions(-) (limited to 'storage') diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index fc0bf0aeeca..eff834503ad 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -419,13 +419,12 @@ the word size of the machine, that is on a 32-bit platform 32 bits, and on a macro ULINTPF. */ -#ifdef __WIN__ +#ifdef _WIN32 /* Use the integer types and formatting strings defined in Visual Studio. */ -# define UINT32PF "%I32u" -# define INT64PF "%I64d" -# define UINT64PF "%I64u" -# define UINT64PFx "%016I64x" -# define DBUG_LSN_PF "%llu" +# define UINT32PF "%u" +# define INT64PF "%lld" +# define UINT64PF "%llu" +# define UINT64PFx "%016llx" typedef __int64 ib_int64_t; typedef unsigned __int64 ib_uint64_t; typedef unsigned __int32 ib_uint32_t; @@ -435,13 +434,12 @@ typedef unsigned __int32 ib_uint32_t; # define INT64PF "%" PRId64 # define UINT64PF "%" PRIu64 # define UINT64PFx "%016" PRIx64 -# define DBUG_LSN_PF UINT64PF typedef int64_t ib_int64_t; typedef uint64_t ib_uint64_t; typedef uint32_t ib_uint32_t; -# endif /* __WIN__ */ +#endif -# define IB_ID_FMT UINT64PF +#define IB_ID_FMT UINT64PF #ifdef _WIN64 typedef unsigned __int64 ulint; diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index a90041aaa90..ec9c2795d0a 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1702,7 +1702,7 @@ recv_recover_page_func( } DBUG_PRINT("ib_log", - ("apply " DBUG_LSN_PF ": %u len %u " + ("apply " LSN_PF ": %u len %u " "page %u:%u", recv->start_lsn, (unsigned) recv->type, (unsigned) recv->len, @@ -2350,7 +2350,7 @@ loop: recv_sys->recovered_lsn = new_recovered_lsn; DBUG_PRINT("ib_log", - ("scan " DBUG_LSN_PF ": log rec %u len %u " + ("scan " LSN_PF ": log rec %u len %u " "page %u:%u", old_lsn, (unsigned) type, (unsigned) len, (unsigned) space, (unsigned) page_no)); @@ -2441,7 +2441,7 @@ loop: #endif /* UNIV_LOG_DEBUG */ DBUG_PRINT("ib_log", - ("scan " DBUG_LSN_PF ": multi-log rec %u " + ("scan " LSN_PF ": multi-log rec %u " "len %u page %u:%u", recv_sys->recovered_lsn, (unsigned) type, (unsigned) len, diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index 12609c3a65d..da8e1aab2c9 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -573,8 +574,8 @@ AbstractCallback::init( } else if (!is_compressed_table() && m_page_size != UNIV_PAGE_SIZE) { ib_logf(IB_LOG_LEVEL_ERROR, - "Page size %lu of ibd file is not the same " - "as the server page size %lu", + "Page size " ULINTPF " of ibd file is not the same " + "as the server page size " ULINTPF, m_page_size, UNIV_PAGE_SIZE); return(DB_CORRUPTION); @@ -583,8 +584,8 @@ AbstractCallback::init( ib_logf(IB_LOG_LEVEL_ERROR, "File size " UINT64PF " is not a multiple " - "of the page size %lu", - (ib_uint64_t) file_size, (ulong) m_page_size); + "of the page size " ULINTPF, + (ib_uint64_t) file_size, m_page_size); return(DB_CORRUPTION); } @@ -725,8 +726,8 @@ FetchIndexRootPages::operator() ( if (block->page.offset * m_page_size != offset) { ib_logf(IB_LOG_LEVEL_ERROR, "Page offset doesn't match file offset: " - "page offset: %lu, file offset: %lu", - (ulint) block->page.offset, + "page offset: %u, file offset: " ULINTPF, + block->page.offset, (ulint) (offset / m_page_size)); err = DB_CORRUPTION; @@ -1159,10 +1160,9 @@ row_import::match_index_columns( ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, - "Index field count %lu doesn't match" - " tablespace metadata file value %lu", - (ulong) index->n_fields, - (ulong) cfg_index->m_n_fields); + "Index field count %u doesn't match" + " tablespace metadata file value " ULINTPF, + index->n_fields, cfg_index->m_n_fields); return(DB_ERROR); } @@ -1179,34 +1179,31 @@ row_import::match_index_columns( ER_TABLE_SCHEMA_MISMATCH, "Index field name %s doesn't match" " tablespace metadata field name %s" - " for field position %lu", - field->name, cfg_field->name, (ulong) i); + " for field position " ULINTPF, + field->name, cfg_field->name, i); err = DB_ERROR; } if (cfg_field->prefix_len != field->prefix_len) { ib_errf(thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Index %s field %s prefix len %lu" - " doesn't match metadata file value" - " %lu", - index->name, field->name, - (ulong) field->prefix_len, - (ulong) cfg_field->prefix_len); + ER_TABLE_SCHEMA_MISMATCH, + "Index %s field %s prefix len %u" + " doesn't match metadata file value %u", + index->name, field->name, + field->prefix_len, cfg_field->prefix_len); err = DB_ERROR; } if (cfg_field->fixed_len != field->fixed_len) { ib_errf(thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Index %s field %s fixed len %lu" - " doesn't match metadata file value" - " %lu", - index->name, field->name, - (ulong) field->fixed_len, - (ulong) cfg_field->fixed_len); + ER_TABLE_SCHEMA_MISMATCH, + "Index %s field %s fixed len %u" + " doesn't match metadata file value %u", + index->name, field->name, + field->fixed_len, + cfg_field->fixed_len); err = DB_ERROR; } @@ -1248,12 +1245,11 @@ row_import::match_table_columns( } else if (cfg_col_index != col->ind) { ib_errf(thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Column %s ordinal value mismatch, it's at " - "%lu in the table and %lu in the tablespace " - "meta-data file", - col_name, - (ulong) col->ind, (ulong) cfg_col_index); + ER_TABLE_SCHEMA_MISMATCH, + "Column %s ordinal value mismatch, it's at %u" + " in the table and " ULINTPF + " in the tablespace meta-data file", + col_name, col->ind, cfg_col_index); err = DB_ERROR; } else { @@ -1336,19 +1332,19 @@ row_import::match_schema( { /* Do some simple checks. */ - if (m_flags != m_table->flags) { + if ((m_table->flags ^ m_flags) & ~DICT_TF_MASK_DATA_DIR) { ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, - "Table flags don't match, server table has 0x%lx " - "and the meta-data file has 0x%lx", - (ulong) m_table->n_cols, (ulong) m_flags); + "Table flags don't match, server table has 0x%x" + " and the meta-data file has 0x%lx", + m_table->flags, ulong(m_flags)); return(DB_ERROR); } else if (m_table->n_cols != m_n_cols) { ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, - "Number of columns don't match, table has %lu " - "columns but the tablespace meta-data file has " - "%lu columns", - (ulong) m_table->n_cols, (ulong) m_n_cols); + "Number of columns don't match, table has %u" + " columns but the tablespace meta-data file has " + ULINTPF " columns", + m_table->n_cols, m_n_cols); return(DB_ERROR); } else if (UT_LIST_GET_LEN(m_table->indexes) != m_n_indexes) { @@ -1358,11 +1354,10 @@ row_import::match_schema( table matching the IMPORT definition. */ ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, - "Number of indexes don't match, table has %lu " - "indexes but the tablespace meta-data file has " - "%lu indexes", - (ulong) UT_LIST_GET_LEN(m_table->indexes), - (ulong) m_n_indexes); + "Number of indexes don't match, table has " ULINTPF + " indexes but the tablespace meta-data file has " + ULINTPF " indexes", + UT_LIST_GET_LEN(m_table->indexes), m_n_indexes); return(DB_ERROR); } @@ -1438,8 +1433,8 @@ row_import::set_root_by_heuristic() UNIV_NOTHROW table_name, sizeof(table_name), m_table->name, FALSE); ib_logf(IB_LOG_LEVEL_WARN, - "Table %s should have %lu indexes but the tablespace " - "has %lu indexes", + "Table %s should have " ULINTPF + " indexes but the tablespace has " ULINTPF " indexes", table_name, UT_LIST_GET_LEN(m_table->indexes), m_n_indexes); @@ -1677,9 +1672,10 @@ PageConverter::adjust_cluster_index_blob_column( ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, ER_INNODB_INDEX_CORRUPT, - "Externally stored column(%lu) has a reference " - "length of %lu in the cluster index %s", - (ulong) i, (ulong) len, index_name); + "Externally stored column(" ULINTPF + ") has a reference length of " ULINTPF + " in the cluster index %s", + i, len, index_name); return(DB_CORRUPTION); } @@ -2040,7 +2036,8 @@ PageConverter::update_page( return(err); } - ib_logf(IB_LOG_LEVEL_WARN, "Unknown page type (%lu)", page_type); + ib_logf(IB_LOG_LEVEL_WARN, "Unknown page type (" ULINTPF ")", + page_type); return(DB_CORRUPTION); } @@ -2074,7 +2071,8 @@ PageConverter::validate( if (checksum != 0) { /* Checksum check passed in buf_page_is_corrupted(). */ ib_logf(IB_LOG_LEVEL_WARN, - "%s: Page %lu checksum %lu should be zero.", + "%s: Page %lu checksum " ULINTPF + " should be zero.", m_filepath, (ulong) (offset / m_page_size), checksum); } @@ -2388,11 +2386,10 @@ row_import_adjust_root_pages_of_secondary_indexes( ib_errf(trx->mysql_thd, IB_LOG_LEVEL_WARN, ER_INNODB_INDEX_CORRUPT, - "Index '%s' contains %lu entries, " - "should be %lu, you should recreate " + "Index '%s' contains " ULINTPF " entries, " + "should be " ULINTPF ", you should recreate " "this index.", index_name, - (ulong) purge.get_n_rows(), - (ulong) n_rows_in_table); + purge.get_n_rows(), n_rows_in_table); index->type |= DICT_CORRUPT; @@ -2743,7 +2740,7 @@ row_import_read_index_data( if (len > OS_FILE_MAX_PATH) { ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_INNODB_INDEX_CORRUPT, - "Index name length (%lu) is too long, " + "Index name length (" ULINTPF ") is too long, " "the meta-data is corrupt", len); return(DB_CORRUPTION); @@ -2824,8 +2821,8 @@ row_import_read_indexes( } else if (cfg->m_n_indexes > 1024) { // FIXME: What is the upper limit? */ ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_IO_READ_ERROR, - "Number of indexes in meta-data file is too high: %lu", - (ulong) cfg->m_n_indexes); + "Number of indexes in meta-data file is too high: " + ULINTPF, cfg->m_n_indexes); cfg->m_n_indexes = 0; return(DB_CORRUPTION); @@ -2923,8 +2920,8 @@ row_import_read_columns( if (len == 0 || len > 128) { ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_IO_READ_ERROR, - "Column name length %lu, is invalid", - (ulong) len); + "Column name length " ULINTPF ", is invalid", + len); return(DB_CORRUPTION); } @@ -3095,8 +3092,9 @@ row_import_read_v1( ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, "Tablespace to be imported has a different " "page size than this server. Server page size " - "is %lu, whereas tablespace page size is %lu", - UNIV_PAGE_SIZE, (ulong) cfg->m_page_size); + "is " ULINTPF ", whereas tablespace page size is " + ULINTPF, + UNIV_PAGE_SIZE, cfg->m_page_size); return(DB_ERROR); } @@ -3161,8 +3159,8 @@ row_import_read_meta_data( return(row_import_read_v1(file, thd, &cfg)); default: ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_IO_READ_ERROR, - "Unsupported meta-data version number (%lu), " - "file ignored", (ulong) cfg.m_version); + "Unsupported meta-data version number (" ULINTPF "), " + "file ignored", cfg.m_version); } return(DB_ERROR); diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index f214633e9ce..33345c2e6f2 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -95,7 +96,8 @@ row_merge_tuple_print( } ut_print_buf(f, dfield_get_data(field), len); if (len != field_len) { - fprintf(f, " (total %lu bytes)", field_len); + fprintf(f, " (total " ULINTPF " bytes)", + field_len); } } } @@ -781,9 +783,9 @@ row_merge_buf_write( ut_ad(b < &block[srv_sort_buf_size]); #ifdef UNIV_DEBUG if (row_merge_print_write) { - fprintf(stderr, "row_merge_buf_write %p,%d,%lu %lu", - (void*) b, of->fd, (ulong) of->offset, - (ulong) i); + fprintf(stderr, "row_merge_buf_write %p,%d," + ULINTPF " " ULINTPF, + (void*) b, of->fd, of->offset, i); row_merge_tuple_print(stderr, entry, n_fields); } #endif /* UNIV_DEBUG */ @@ -800,8 +802,8 @@ row_merge_buf_write( #endif /* UNIV_DEBUG_VALGRIND */ #ifdef UNIV_DEBUG if (row_merge_print_write) { - fprintf(stderr, "row_merge_buf_write %p,%d,%lu EOF\n", - (void*) b, of->fd, (ulong) of->offset); + fprintf(stderr, "row_merge_buf_write %p,%d," ULINTPF " EOF\n", + (void*) b, of->fd, of->offset); } #endif /* UNIV_DEBUG */ } @@ -857,15 +859,8 @@ row_merge_read( #ifdef UNIV_DEBUG if (row_merge_print_block_read) { - fprintf(stderr, "row_merge_read fd=%d ofs=%lu\n", - fd, (ulong) offset); - } -#endif /* UNIV_DEBUG */ - -#ifdef UNIV_DEBUG - if (row_merge_print_block_read) { - fprintf(stderr, "row_merge_read fd=%d ofs=%lu\n", - fd, (ulong) offset); + fprintf(stderr, "row_merge_read fd=%d ofs=" ULINTPF "\n", + fd, offset); } #endif /* UNIV_DEBUG */ @@ -908,8 +903,8 @@ row_merge_write( #ifdef UNIV_DEBUG if (row_merge_print_block_write) { - fprintf(stderr, "row_merge_write fd=%d ofs=%lu\n", - fd, (ulong) offset); + fprintf(stderr, "row_merge_write fd=%d ofs=" ULINTPF "\n", + fd, offset); } #endif /* UNIV_DEBUG */ @@ -957,9 +952,10 @@ row_merge_read_rec( *mrec = NULL; #ifdef UNIV_DEBUG if (row_merge_print_read) { - fprintf(stderr, "row_merge_read %p,%p,%d,%lu EOF\n", + fprintf(stderr, "row_merge_read %p,%p,%d," ULINTPF + " EOF\n", (const void*) b, (const void*) block, - fd, (ulong) *foffs); + fd, *foffs); } #endif /* UNIV_DEBUG */ return(NULL); @@ -1074,9 +1070,9 @@ err_exit: func_exit: #ifdef UNIV_DEBUG if (row_merge_print_read) { - fprintf(stderr, "row_merge_read %p,%p,%d,%lu ", + fprintf(stderr, "row_merge_read %p,%p,%d," ULINTPF " ", (const void*) b, (const void*) block, - fd, (ulong) *foffs); + fd, *foffs); rec_print_comp(stderr, *mrec, offsets); putc('\n', stderr); } @@ -1110,8 +1106,8 @@ row_merge_write_rec_low( ut_ad(e == rec_offs_extra_size(offsets) + 1); if (row_merge_print_write) { - fprintf(stderr, "row_merge_write %p,%d,%lu ", - (void*) b, fd, (ulong) foffs); + fprintf(stderr, "row_merge_write %p,%d," ULINTPF " ", + (void*) b, fd, foffs); rec_print_comp(stderr, mrec, offsets); putc('\n', stderr); } @@ -1213,8 +1209,8 @@ row_merge_write_eof( ut_ad(foffs); #ifdef UNIV_DEBUG if (row_merge_print_write) { - fprintf(stderr, "row_merge_write %p,%p,%d,%lu EOF\n", - (void*) b, (void*) block, fd, (ulong) *foffs); + fprintf(stderr, "row_merge_write %p,%p,%d," ULINTPF " EOF\n", + (void*) b, (void*) block, fd, *foffs); } #endif /* UNIV_DEBUG */ @@ -2053,11 +2049,12 @@ row_merge_blocks( #ifdef UNIV_DEBUG if (row_merge_print_block) { fprintf(stderr, - "row_merge_blocks fd=%d ofs=%lu + fd=%d ofs=%lu" - " = fd=%d ofs=%lu\n", - file->fd, (ulong) *foffs0, - file->fd, (ulong) *foffs1, - of->fd, (ulong) of->offset); + "row_merge_blocks fd=%d ofs=" ULINTPF + " + fd=%d ofs=" ULINTPF + " = fd=%d ofs=" ULINTPF "\n", + file->fd, *foffs0, + file->fd, *foffs1, + of->fd, of->offset); } #endif /* UNIV_DEBUG */ @@ -2156,10 +2153,10 @@ row_merge_blocks_copy( #ifdef UNIV_DEBUG if (row_merge_print_block) { fprintf(stderr, - "row_merge_blocks_copy fd=%d ofs=%lu" - " = fd=%d ofs=%lu\n", - file->fd, (ulong) foffs0, - of->fd, (ulong) of->offset); + "row_merge_blocks_copy fd=%d ofs=" ULINTPF + " = fd=%d ofs=" ULINTPF "\n", + file->fd, *foffs0, + of->fd, of->offset); } #endif /* UNIV_DEBUG */ diff --git a/storage/xtradb/include/univ.i b/storage/xtradb/include/univ.i index 7497e7f5d05..fff5c0a67a1 100644 --- a/storage/xtradb/include/univ.i +++ b/storage/xtradb/include/univ.i @@ -438,13 +438,12 @@ the word size of the machine, that is on a 32-bit platform 32 bits, and on a macro ULINTPF. */ -#ifdef __WIN__ +#ifdef _WIN32 /* Use the integer types and formatting strings defined in Visual Studio. */ -# define UINT32PF "%I32u" -# define INT64PF "%I64d" -# define UINT64PF "%I64u" -# define UINT64PFx "%016I64x" -# define DBUG_LSN_PF "%llu" +# define UINT32PF "%u" +# define INT64PF "%lld" +# define UINT64PF "%llu" +# define UINT64PFx "%016llx" typedef __int64 ib_int64_t; typedef unsigned __int64 ib_uint64_t; typedef unsigned __int32 ib_uint32_t; @@ -454,13 +453,12 @@ typedef unsigned __int32 ib_uint32_t; # define INT64PF "%" PRId64 # define UINT64PF "%" PRIu64 # define UINT64PFx "%016" PRIx64 -# define DBUG_LSN_PF UINT64PF typedef int64_t ib_int64_t; typedef uint64_t ib_uint64_t; typedef uint32_t ib_uint32_t; -# endif /* __WIN__ */ +#endif -# define IB_ID_FMT UINT64PF +#define IB_ID_FMT UINT64PF #ifdef _WIN64 typedef unsigned __int64 ulint; diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index afb8b7f00b8..4edfb1e4347 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -1792,7 +1792,7 @@ recv_recover_page_func( } DBUG_PRINT("ib_log", - ("apply " DBUG_LSN_PF ": %u len %u " + ("apply " LSN_PF ": %u len %u " "page %u:%u", recv->start_lsn, (unsigned) recv->type, (unsigned) recv->len, @@ -2440,7 +2440,7 @@ loop: recv_sys->recovered_lsn = new_recovered_lsn; DBUG_PRINT("ib_log", - ("scan " DBUG_LSN_PF ": log rec %u len %u " + ("scan " LSN_PF ": log rec %u len %u " "page %u:%u", old_lsn, (unsigned) type, (unsigned) len, (unsigned) space, (unsigned) page_no)); @@ -2531,7 +2531,7 @@ loop: #endif /* UNIV_LOG_DEBUG */ DBUG_PRINT("ib_log", - ("scan " DBUG_LSN_PF ": multi-log rec %u " + ("scan " LSN_PF ": multi-log rec %u " "len %u page %u:%u", recv_sys->recovered_lsn, (unsigned) type, (unsigned) len, diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index e1fef3d3716..23b9ba60269 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -573,8 +574,8 @@ AbstractCallback::init( } else if (!is_compressed_table() && m_page_size != UNIV_PAGE_SIZE) { ib_logf(IB_LOG_LEVEL_ERROR, - "Page size %lu of ibd file is not the same " - "as the server page size %lu", + "Page size " ULINTPF " of ibd file is not the same " + "as the server page size " ULINTPF, m_page_size, UNIV_PAGE_SIZE); return(DB_CORRUPTION); @@ -583,8 +584,8 @@ AbstractCallback::init( ib_logf(IB_LOG_LEVEL_ERROR, "File size " UINT64PF " is not a multiple " - "of the page size %lu", - (ib_uint64_t) file_size, (ulong) m_page_size); + "of the page size " ULINTPF, + (ib_uint64_t) file_size, m_page_size); return(DB_CORRUPTION); } @@ -725,8 +726,8 @@ FetchIndexRootPages::operator() ( if (block->page.offset * m_page_size != offset) { ib_logf(IB_LOG_LEVEL_ERROR, "Page offset doesn't match file offset: " - "page offset: %lu, file offset: %lu", - (ulint) block->page.offset, + "page offset: %u, file offset: " ULINTPF, + block->page.offset, (ulint) (offset / m_page_size)); err = DB_CORRUPTION; @@ -1159,10 +1160,9 @@ row_import::match_index_columns( ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, - "Index field count %lu doesn't match" - " tablespace metadata file value %lu", - (ulong) index->n_fields, - (ulong) cfg_index->m_n_fields); + "Index field count %u doesn't match" + " tablespace metadata file value " ULINTPF, + index->n_fields, cfg_index->m_n_fields); return(DB_ERROR); } @@ -1179,34 +1179,31 @@ row_import::match_index_columns( ER_TABLE_SCHEMA_MISMATCH, "Index field name %s doesn't match" " tablespace metadata field name %s" - " for field position %lu", - field->name, cfg_field->name, (ulong) i); + " for field position " ULINTPF, + field->name, cfg_field->name, i); err = DB_ERROR; } if (cfg_field->prefix_len != field->prefix_len) { ib_errf(thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Index %s field %s prefix len %lu" - " doesn't match metadata file value" - " %lu", - index->name, field->name, - (ulong) field->prefix_len, - (ulong) cfg_field->prefix_len); + ER_TABLE_SCHEMA_MISMATCH, + "Index %s field %s prefix len %u" + " doesn't match metadata file value %u", + index->name, field->name, + field->prefix_len, cfg_field->prefix_len); err = DB_ERROR; } if (cfg_field->fixed_len != field->fixed_len) { ib_errf(thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Index %s field %s fixed len %lu" - " doesn't match metadata file value" - " %lu", - index->name, field->name, - (ulong) field->fixed_len, - (ulong) cfg_field->fixed_len); + ER_TABLE_SCHEMA_MISMATCH, + "Index %s field %s fixed len %u" + " doesn't match metadata file value %u", + index->name, field->name, + field->fixed_len, + cfg_field->fixed_len); err = DB_ERROR; } @@ -1248,12 +1245,11 @@ row_import::match_table_columns( } else if (cfg_col_index != col->ind) { ib_errf(thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Column %s ordinal value mismatch, it's at " - "%lu in the table and %lu in the tablespace " - "meta-data file", - col_name, - (ulong) col->ind, (ulong) cfg_col_index); + ER_TABLE_SCHEMA_MISMATCH, + "Column %s ordinal value mismatch, it's at %u" + " in the table and " ULINTPF + " in the tablespace meta-data file", + col_name, col->ind, cfg_col_index); err = DB_ERROR; } else { @@ -1335,23 +1331,20 @@ row_import::match_schema( THD* thd) UNIV_NOTHROW { /* Do some simple checks. */ - const unsigned relevant_flags = m_flags & ~DICT_TF_MASK_DATA_DIR; - const unsigned relevant_table_flags - = m_table->flags & ~DICT_TF_MASK_DATA_DIR; - if (relevant_flags != relevant_table_flags) { + if ((m_table->flags ^ m_flags) & ~DICT_TF_MASK_DATA_DIR) { ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, - "Table flags don't match, server table has 0x%x " - "and the meta-data file has 0x%x", - relevant_table_flags, relevant_flags); + "Table flags don't match, server table has 0x%x" + " and the meta-data file has 0x%lx", + m_table->flags, ulong(m_flags)); return(DB_ERROR); } else if (m_table->n_cols != m_n_cols) { ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, - "Number of columns don't match, table has %lu " - "columns but the tablespace meta-data file has " - "%lu columns", - (ulong) m_table->n_cols, (ulong) m_n_cols); + "Number of columns don't match, table has %u " + "columns but the tablespace meta-data file has " + ULINTPF " columns", + m_table->n_cols, m_n_cols); return(DB_ERROR); } else if (UT_LIST_GET_LEN(m_table->indexes) != m_n_indexes) { @@ -1361,11 +1354,10 @@ row_import::match_schema( table matching the IMPORT definition. */ ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, - "Number of indexes don't match, table has %lu " - "indexes but the tablespace meta-data file has " - "%lu indexes", - (ulong) UT_LIST_GET_LEN(m_table->indexes), - (ulong) m_n_indexes); + "Number of indexes don't match, table has " ULINTPF + " indexes but the tablespace meta-data file has " + ULINTPF " indexes", + UT_LIST_GET_LEN(m_table->indexes), m_n_indexes); return(DB_ERROR); } @@ -1441,8 +1433,8 @@ row_import::set_root_by_heuristic() UNIV_NOTHROW table_name, sizeof(table_name), m_table->name, FALSE); ib_logf(IB_LOG_LEVEL_WARN, - "Table %s should have %lu indexes but the tablespace " - "has %lu indexes", + "Table %s should have " ULINTPF + " indexes but the tablespace has " ULINTPF " indexes", table_name, UT_LIST_GET_LEN(m_table->indexes), m_n_indexes); @@ -1680,9 +1672,10 @@ PageConverter::adjust_cluster_index_blob_column( ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, ER_INNODB_INDEX_CORRUPT, - "Externally stored column(%lu) has a reference " - "length of %lu in the cluster index %s", - (ulong) i, (ulong) len, index_name); + "Externally stored column(" ULINTPF + ") has a reference length of " ULINTPF + " in the cluster index %s", + i, len, index_name); return(DB_CORRUPTION); } @@ -2043,7 +2036,8 @@ PageConverter::update_page( return(err); } - ib_logf(IB_LOG_LEVEL_WARN, "Unknown page type (%lu)", page_type); + ib_logf(IB_LOG_LEVEL_WARN, "Unknown page type (" ULINTPF ")", + page_type); return(DB_CORRUPTION); } @@ -2077,7 +2071,8 @@ PageConverter::validate( if (checksum != 0) { /* Checksum check passed in buf_page_is_corrupted(). */ ib_logf(IB_LOG_LEVEL_WARN, - "%s: Page %lu checksum %lu should be zero.", + "%s: Page %lu checksum " ULINTPF + " should be zero.", m_filepath, (ulong) (offset / m_page_size), checksum); } @@ -2391,11 +2386,10 @@ row_import_adjust_root_pages_of_secondary_indexes( ib_errf(trx->mysql_thd, IB_LOG_LEVEL_WARN, ER_INNODB_INDEX_CORRUPT, - "Index '%s' contains %lu entries, " - "should be %lu, you should recreate " + "Index '%s' contains " ULINTPF " entries, " + "should be " ULINTPF ", you should recreate " "this index.", index_name, - (ulong) purge.get_n_rows(), - (ulong) n_rows_in_table); + purge.get_n_rows(), n_rows_in_table); index->type |= DICT_CORRUPT; @@ -2746,7 +2740,7 @@ row_import_read_index_data( if (len > OS_FILE_MAX_PATH) { ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_INNODB_INDEX_CORRUPT, - "Index name length (%lu) is too long, " + "Index name length (" ULINTPF ") is too long, " "the meta-data is corrupt", len); return(DB_CORRUPTION); @@ -2827,8 +2821,8 @@ row_import_read_indexes( } else if (cfg->m_n_indexes > 1024) { // FIXME: What is the upper limit? */ ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_IO_READ_ERROR, - "Number of indexes in meta-data file is too high: %lu", - (ulong) cfg->m_n_indexes); + "Number of indexes in meta-data file is too high: " + ULINTPF, cfg->m_n_indexes); cfg->m_n_indexes = 0; return(DB_CORRUPTION); @@ -2926,8 +2920,8 @@ row_import_read_columns( if (len == 0 || len > 128) { ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_IO_READ_ERROR, - "Column name length %lu, is invalid", - (ulong) len); + "Column name length " ULINTPF ", is invalid", + len); return(DB_CORRUPTION); } @@ -3098,8 +3092,9 @@ row_import_read_v1( ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, "Tablespace to be imported has a different " "page size than this server. Server page size " - "is %lu, whereas tablespace page size is %lu", - UNIV_PAGE_SIZE, (ulong) cfg->m_page_size); + "is " ULINTPF ", whereas tablespace page size is " + ULINTPF, + UNIV_PAGE_SIZE, cfg->m_page_size); return(DB_ERROR); } @@ -3164,8 +3159,8 @@ row_import_read_meta_data( return(row_import_read_v1(file, thd, &cfg)); default: ib_errf(thd, IB_LOG_LEVEL_ERROR, ER_IO_READ_ERROR, - "Unsupported meta-data version number (%lu), " - "file ignored", (ulong) cfg.m_version); + "Unsupported meta-data version number (" ULINTPF "), " + "file ignored", cfg.m_version); } return(DB_ERROR); diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index bcc37fb530c..5082aae15cf 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -96,7 +96,8 @@ row_merge_tuple_print( } ut_print_buf(f, dfield_get_data(field), len); if (len != field_len) { - fprintf(f, " (total %lu bytes)", field_len); + fprintf(f, " (total " ULINTPF " bytes)", + field_len); } } } @@ -789,9 +790,9 @@ row_merge_buf_write( ut_ad(b < &block[srv_sort_buf_size]); #ifdef UNIV_DEBUG if (row_merge_print_write) { - fprintf(stderr, "row_merge_buf_write %p,%d,%lu %lu", - (void*) b, of->fd, (ulong) of->offset, - (ulong) i); + fprintf(stderr, "row_merge_buf_write %p,%d," + ULINTPF " " ULINTPF, + (void*) b, of->fd, of->offset, i); row_merge_tuple_print(stderr, entry, n_fields); } #endif /* UNIV_DEBUG */ @@ -808,8 +809,8 @@ row_merge_buf_write( #endif /* UNIV_DEBUG_VALGRIND */ #ifdef UNIV_DEBUG if (row_merge_print_write) { - fprintf(stderr, "row_merge_buf_write %p,%d,%lu EOF\n", - (void*) b, of->fd, (ulong) of->offset); + fprintf(stderr, "row_merge_buf_write %p,%d," ULINTPF " EOF\n", + (void*) b, of->fd, of->offset); } #endif /* UNIV_DEBUG */ } @@ -865,15 +866,8 @@ row_merge_read( #ifdef UNIV_DEBUG if (row_merge_print_block_read) { - fprintf(stderr, "row_merge_read fd=%d ofs=%lu\n", - fd, (ulong) offset); - } -#endif /* UNIV_DEBUG */ - -#ifdef UNIV_DEBUG - if (row_merge_print_block_read) { - fprintf(stderr, "row_merge_read fd=%d ofs=%lu\n", - fd, (ulong) offset); + fprintf(stderr, "row_merge_read fd=%d ofs=" ULINTPF "\n", + fd, offset); } #endif /* UNIV_DEBUG */ @@ -916,8 +910,8 @@ row_merge_write( #ifdef UNIV_DEBUG if (row_merge_print_block_write) { - fprintf(stderr, "row_merge_write fd=%d ofs=%lu\n", - fd, (ulong) offset); + fprintf(stderr, "row_merge_write fd=%d ofs=" ULINTPF "\n", + fd, offset); } #endif /* UNIV_DEBUG */ @@ -965,9 +959,10 @@ row_merge_read_rec( *mrec = NULL; #ifdef UNIV_DEBUG if (row_merge_print_read) { - fprintf(stderr, "row_merge_read %p,%p,%d,%lu EOF\n", + fprintf(stderr, "row_merge_read %p,%p,%d," ULINTPF + " EOF\n", (const void*) b, (const void*) block, - fd, (ulong) *foffs); + fd, *foffs); } #endif /* UNIV_DEBUG */ return(NULL); @@ -1082,9 +1077,9 @@ err_exit: func_exit: #ifdef UNIV_DEBUG if (row_merge_print_read) { - fprintf(stderr, "row_merge_read %p,%p,%d,%lu ", + fprintf(stderr, "row_merge_read %p,%p,%d," ULINTPF " ", (const void*) b, (const void*) block, - fd, (ulong) *foffs); + fd, *foffs); rec_print_comp(stderr, *mrec, offsets); putc('\n', stderr); } @@ -1118,8 +1113,8 @@ row_merge_write_rec_low( ut_ad(e == rec_offs_extra_size(offsets) + 1); if (row_merge_print_write) { - fprintf(stderr, "row_merge_write %p,%d,%lu ", - (void*) b, fd, (ulong) foffs); + fprintf(stderr, "row_merge_write %p,%d," ULINTPF " ", + (void*) b, fd, foffs); rec_print_comp(stderr, mrec, offsets); putc('\n', stderr); } @@ -1221,8 +1216,8 @@ row_merge_write_eof( ut_ad(foffs); #ifdef UNIV_DEBUG if (row_merge_print_write) { - fprintf(stderr, "row_merge_write %p,%p,%d,%lu EOF\n", - (void*) b, (void*) block, fd, (ulong) *foffs); + fprintf(stderr, "row_merge_write %p,%p,%d," ULINTPF " EOF\n", + (void*) b, (void*) block, fd, *foffs); } #endif /* UNIV_DEBUG */ @@ -2067,11 +2062,12 @@ row_merge_blocks( #ifdef UNIV_DEBUG if (row_merge_print_block) { fprintf(stderr, - "row_merge_blocks fd=%d ofs=%lu + fd=%d ofs=%lu" - " = fd=%d ofs=%lu\n", - file->fd, (ulong) *foffs0, - file->fd, (ulong) *foffs1, - of->fd, (ulong) of->offset); + "row_merge_blocks fd=%d ofs=" ULINTPF + " + fd=%d ofs=" ULINTPF + " = fd=%d ofs=" ULINTPF "\n", + file->fd, *foffs0, + file->fd, *foffs1, + of->fd, of->offset); } #endif /* UNIV_DEBUG */ @@ -2170,10 +2166,10 @@ row_merge_blocks_copy( #ifdef UNIV_DEBUG if (row_merge_print_block) { fprintf(stderr, - "row_merge_blocks_copy fd=%d ofs=%lu" - " = fd=%d ofs=%lu\n", - file->fd, (ulong) foffs0, - of->fd, (ulong) of->offset); + "row_merge_blocks_copy fd=%d ofs=" ULINTPF + " = fd=%d ofs=" ULINTPF "\n", + file->fd, *foffs0, + of->fd, of->offset); } #endif /* UNIV_DEBUG */ -- cgit v1.2.1 From 3a6af51a8adc1e7a7b94534dbf26bdcad836f3cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 21 Apr 2017 05:05:51 +0300 Subject: Do not crash XtraDB when fil_space_acquire() fails This reverts part of commit 50eb40a2a8aa3af6cc271f6028f4d6d74301d030 which backported the code from MariaDB 10.2. The XtraDB version of the code included a ut_error statement (aborting the process) when a tablespace is not found. Luckily this change was not part of a release; MariaDB 10.1.22 had been released some days earlier. --- storage/xtradb/fil/fil0fil.cc | 1 - 1 file changed, 1 deletion(-) (limited to 'storage') diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index e7244d719c8..43788012920 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -7401,7 +7401,6 @@ fil_space_acquire_low(ulint id, bool silent, bool for_io = false) if (!silent) { ib_logf(IB_LOG_LEVEL_WARN, "Trying to access missing" " tablespace " ULINTPF ".", id); - ut_error; } } else if (!for_io && space->is_stopping()) { space = NULL; -- cgit v1.2.1 From e48ae21b0ea99a3d3cc16eebd414cc484fa4f38a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 21 Apr 2017 16:22:46 +0300 Subject: Follow-up to MDEV-12534: Fix warnings on 32-bit systems --- storage/innobase/os/os0file.cc | 4 ++-- storage/innobase/srv/srv0srv.cc | 4 ++-- storage/xtradb/os/os0file.cc | 4 ++-- storage/xtradb/srv/srv0srv.cc | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) (limited to 'storage') diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index ff67c90eecf..d8cb44217bb 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -5647,8 +5647,8 @@ os_aio_print( os_n_file_writes, os_n_fsyncs); - const ulint n_reads = MONITOR_VALUE(MONITOR_OS_PENDING_READS); - const ulint n_writes = MONITOR_VALUE(MONITOR_OS_PENDING_WRITES); + const ulint n_reads = ulint(MONITOR_VALUE(MONITOR_OS_PENDING_READS)); + const ulint n_writes = ulint(MONITOR_VALUE(MONITOR_OS_PENDING_WRITES)); if (n_reads != 0 || n_writes != 0) { fprintf(file, diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 58e69c3c9e7..067e32a83aa 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -1409,10 +1409,10 @@ srv_export_innodb_status(void) mutex_enter(&srv_innodb_monitor_mutex); export_vars.innodb_data_pending_reads = - MONITOR_VALUE(MONITOR_OS_PENDING_READS); + ulint(MONITOR_VALUE(MONITOR_OS_PENDING_READS)); export_vars.innodb_data_pending_writes = - MONITOR_VALUE(MONITOR_OS_PENDING_WRITES); + ulint(MONITOR_VALUE(MONITOR_OS_PENDING_WRITES)); export_vars.innodb_data_pending_fsyncs = fil_n_pending_log_flushes diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 48ae49f28e4..9632bb77df9 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -5820,8 +5820,8 @@ os_aio_print( os_n_file_writes, os_n_fsyncs); - const ulint n_reads = MONITOR_VALUE(MONITOR_OS_PENDING_READS); - const ulint n_writes = MONITOR_VALUE(MONITOR_OS_PENDING_WRITES); + const ulint n_reads = ulint(MONITOR_VALUE(MONITOR_OS_PENDING_READS)); + const ulint n_writes = ulint(MONITOR_VALUE(MONITOR_OS_PENDING_WRITES)); if (n_reads != 0 || n_writes != 0) { fprintf(file, diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index c47119e63cf..5b3f86d8641 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -1735,10 +1735,10 @@ srv_export_innodb_status(void) mutex_enter(&srv_innodb_monitor_mutex); export_vars.innodb_data_pending_reads = - MONITOR_VALUE(MONITOR_OS_PENDING_READS); + ulint(MONITOR_VALUE(MONITOR_OS_PENDING_READS)); export_vars.innodb_data_pending_writes = - MONITOR_VALUE(MONITOR_OS_PENDING_WRITES); + ulint(MONITOR_VALUE(MONITOR_OS_PENDING_WRITES)); export_vars.innodb_data_pending_fsyncs = fil_n_pending_log_flushes -- cgit v1.2.1 From e056d1f1ca91ebe40467ed46be00be0add9cf247 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 21 Apr 2017 17:39:12 +0300 Subject: Fix some InnoDB type mismatch On 64-bit Windows, sizeof(ulint)!=sizeof(ulong). --- storage/innobase/buf/buf0buf.cc | 25 ++++++++++++------------- storage/innobase/row/row0ftsort.cc | 7 +++---- storage/xtradb/buf/buf0buf.cc | 25 ++++++++++++------------- storage/xtradb/row/row0ftsort.cc | 7 +++---- 4 files changed, 30 insertions(+), 34 deletions(-) (limited to 'storage') diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 73d061c4b20..0b305507271 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -5311,23 +5311,22 @@ buf_print_io_instance( pool_info->pages_written_rate); if (pool_info->n_page_get_delta) { - double hit_rate = ((1000 * pool_info->page_read_delta) - / pool_info->n_page_get_delta); + double hit_rate = double(pool_info->page_read_delta) + / pool_info->n_page_get_delta; - if (hit_rate > 1000) { - hit_rate = 1000; + if (hit_rate > 1) { + hit_rate = 1; } - hit_rate = 1000 - hit_rate; - fprintf(file, - "Buffer pool hit rate %lu / 1000," - " young-making rate %lu / 1000 not %lu / 1000\n", - (ulint) hit_rate, - (ulint) (1000 * pool_info->young_making_delta - / pool_info->n_page_get_delta), - (ulint) (1000 * pool_info->not_young_making_delta - / pool_info->n_page_get_delta)); + "Buffer pool hit rate " ULINTPF " / 1000," + " young-making rate " ULINTPF " / 1000 not " + ULINTPF " / 1000\n", + ulint(1000 * (1 - hit_rate)), + ulint(1000 * double(pool_info->young_making_delta) + / pool_info->n_page_get_delta), + ulint(1000 * double(pool_info->not_young_making_delta) + / pool_info->n_page_get_delta)); } else { fputs("No buffer pool page gets since the last printout\n", file); diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index 4865c8e850a..5b8ed009a2a 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -1248,10 +1248,9 @@ row_fts_build_sel_tree_level( int child_left; int child_right; ulint i; - ulint num_item; + ulint num_item = ulint(1) << level; - start = static_cast((1 << level) - 1); - num_item = static_cast(1 << level); + start = num_item - 1; for (i = 0; i < num_item; i++) { child_left = sel_tree[(start + i) * 2 + 1]; @@ -1320,7 +1319,7 @@ row_fts_build_sel_tree( treelevel++; } - start = (1 << treelevel) - 1; + start = (ulint(1) << treelevel) - 1; for (i = 0; i < (int) fts_sort_pll_degree; i++) { sel_tree[i + start] = i; diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index a5d8b661c35..07b84c60c76 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -5593,23 +5593,22 @@ buf_print_io_instance( pool_info->pages_written_rate); if (pool_info->n_page_get_delta) { - double hit_rate = ((1000 * pool_info->page_read_delta) - / pool_info->n_page_get_delta); + double hit_rate = double(pool_info->page_read_delta) + / pool_info->n_page_get_delta; - if (hit_rate > 1000) { - hit_rate = 1000; + if (hit_rate > 1) { + hit_rate = 1; } - hit_rate = 1000 - hit_rate; - fprintf(file, - "Buffer pool hit rate %lu / 1000," - " young-making rate %lu / 1000 not %lu / 1000\n", - (ulint) hit_rate, - (ulint) (1000 * pool_info->young_making_delta - / pool_info->n_page_get_delta), - (ulint) (1000 * pool_info->not_young_making_delta - / pool_info->n_page_get_delta)); + "Buffer pool hit rate " ULINTPF " / 1000," + " young-making rate " ULINTPF " / 1000 not " + ULINTPF " / 1000\n", + ulint(1000 * (1 - hit_rate)), + ulint(1000 * double(pool_info->young_making_delta) + / pool_info->n_page_get_delta), + ulint(1000 * double(pool_info->not_young_making_delta) + / pool_info->n_page_get_delta)); } else { fputs("No buffer pool page gets since the last printout\n", file); diff --git a/storage/xtradb/row/row0ftsort.cc b/storage/xtradb/row/row0ftsort.cc index 134127c6adf..13877b8fe08 100644 --- a/storage/xtradb/row/row0ftsort.cc +++ b/storage/xtradb/row/row0ftsort.cc @@ -1251,10 +1251,9 @@ row_fts_build_sel_tree_level( int child_left; int child_right; ulint i; - ulint num_item; + ulint num_item = ulint(1) << level; - start = static_cast((1 << level) - 1); - num_item = static_cast(1 << level); + start = num_item - 1; for (i = 0; i < num_item; i++) { child_left = sel_tree[(start + i) * 2 + 1]; @@ -1323,7 +1322,7 @@ row_fts_build_sel_tree( treelevel++; } - start = (1 << treelevel) - 1; + start = (ulint(1) << treelevel) - 1; for (i = 0; i < (int) fts_sort_pll_degree; i++) { sel_tree[i + start] = i; -- cgit v1.2.1 From aafaf05a47221e035039d75a8f62dbda2379d044 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 21 Apr 2017 08:45:48 +0300 Subject: Fix some InnoDB type mismatch introduced in 10.1 On 64-bit Windows, sizeof(ulint)!=sizeof(ulong). --- storage/innobase/buf/buf0buf.cc | 17 +++++++---------- storage/innobase/buf/buf0dblwr.cc | 4 ++-- storage/innobase/fil/fil0crypt.cc | 21 ++++++++++++--------- storage/innobase/fil/fil0fil.cc | 3 ++- storage/innobase/fil/fil0pagecompress.cc | 5 +++-- storage/xtradb/buf/buf0buf.cc | 17 +++++++---------- storage/xtradb/buf/buf0dblwr.cc | 4 ++-- storage/xtradb/fil/fil0crypt.cc | 21 ++++++++++++--------- storage/xtradb/fil/fil0fil.cc | 3 ++- storage/xtradb/fil/fil0pagecompress.cc | 5 +++-- 10 files changed, 52 insertions(+), 48 deletions(-) (limited to 'storage') diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 89924dd3675..2133457fcbb 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -6269,9 +6269,9 @@ buf_page_decrypt_after_read( /* decompress using comp_buf to dst_frame */ fil_decompress_page(slot->comp_buf, - dst_frame, - size, - &bpage->write_size); + dst_frame, + ulong(size), + &bpage->write_size); /* Mark this slot as free */ slot->reserved = false; @@ -6323,13 +6323,10 @@ buf_page_decrypt_after_read( #endif /* decompress using comp_buf to dst_frame */ fil_decompress_page(slot->comp_buf, - dst_frame, - size, - &bpage->write_size); - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif + dst_frame, + ulong(size), + &bpage->write_size); + ut_d(fil_page_type_validate(dst_frame)); } /* Mark this slot as free */ diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 2e0cbc1a8b2..f99679f6fec 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -538,7 +538,7 @@ buf_dblwr_process() /* Decompress the page before validating the checksum. */ fil_decompress_page( - NULL, read_buf, UNIV_PAGE_SIZE, + NULL, read_buf, srv_page_size, NULL, true); } @@ -565,7 +565,7 @@ buf_dblwr_process() /* Decompress the page before validating the checksum. */ fil_decompress_page( - NULL, page, UNIV_PAGE_SIZE, NULL, true); + NULL, page, srv_page_size, NULL, true); } if (!fil_space_verify_crypt_checksum(page, zip_size, NULL, page_no) diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 9c5f782da32..b8892b4ec31 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -694,7 +694,8 @@ fil_space_encrypt( comp_mem = (byte *)malloc(UNIV_PAGE_SIZE); uncomp_mem = (byte *)malloc(UNIV_PAGE_SIZE); memcpy(comp_mem, src_frame, UNIV_PAGE_SIZE); - fil_decompress_page(uncomp_mem, comp_mem, UNIV_PAGE_SIZE, NULL); + fil_decompress_page(uncomp_mem, comp_mem, + srv_page_size, NULL); src = uncomp_mem; } @@ -704,7 +705,8 @@ fil_space_encrypt( /* Need to decompress the page if it was also compressed */ if (page_compressed_encrypted) { memcpy(comp_mem, tmp_mem, UNIV_PAGE_SIZE); - fil_decompress_page(tmp_mem, comp_mem, UNIV_PAGE_SIZE, NULL); + fil_decompress_page(tmp_mem, comp_mem, + srv_page_size, NULL); } bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size, space); @@ -1492,20 +1494,21 @@ fil_crypt_realloc_iops( if (10 * state->cnt_waited > state->batch) { /* if we waited more than 10% re-estimate max_iops */ - uint avg_wait_time_us = + ulint avg_wait_time_us = state->sum_waited_us / state->cnt_waited; + if (avg_wait_time_us == 0) { + avg_wait_time_us = 1; // prevent division by zero + } + DBUG_PRINT("ib_crypt", - ("thr_no: %u - update estimated_max_iops from %u to %u.", + ("thr_no: %u - update estimated_max_iops from %u to " + ULINTPF ".", state->thread_no, state->estimated_max_iops, 1000000 / avg_wait_time_us)); - if (avg_wait_time_us == 0) { - avg_wait_time_us = 1; // prevent division by zero - } - - state->estimated_max_iops = 1000000 / avg_wait_time_us; + state->estimated_max_iops = uint(1000000 / avg_wait_time_us); state->cnt_waited = 0; state->sum_waited_us = 0; } else { diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 6fec41839ee..ceda86e16b6 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -6708,7 +6708,8 @@ fil_iterate( /* If the original page is page_compressed, we need to decompress page before we can update it. */ if (page_compressed) { - fil_decompress_page(NULL, dst, size, NULL); + fil_decompress_page(NULL, dst, ulong(size), + NULL); updated = true; } diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index bfc054701a4..42f894f08d7 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -300,7 +300,8 @@ fil_compress_page( #endif /* HAVE_SNAPPY */ case PAGE_ZLIB_ALGORITHM: - err = compress2(out_buf+header_len, (ulong*)&write_size, buf, len, comp_level); + err = compress2(out_buf+header_len, (ulong*)&write_size, buf, + uLong(len), comp_level); if (err != Z_OK) { /* If error we leave the actual page as it was */ @@ -364,7 +365,7 @@ fil_compress_page( uncomp_page = static_cast(ut_malloc(UNIV_PAGE_SIZE)); memcpy(comp_page, out_buf, UNIV_PAGE_SIZE); - fil_decompress_page(uncomp_page, comp_page, len, NULL); + fil_decompress_page(uncomp_page, comp_page, ulong(len), NULL); if(buf_page_is_corrupted(false, uncomp_page, 0, space)) { buf_page_print(uncomp_page, 0, BUF_PAGE_PRINT_NO_CRASH); diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 3fe07800fd3..7f77f96c2fe 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -6429,9 +6429,9 @@ buf_page_decrypt_after_read( /* decompress using comp_buf to dst_frame */ fil_decompress_page(slot->comp_buf, - dst_frame, - size, - &bpage->write_size); + dst_frame, + ulong(size), + &bpage->write_size); /* Mark this slot as free */ slot->reserved = false; @@ -6483,13 +6483,10 @@ buf_page_decrypt_after_read( #endif /* decompress using comp_buf to dst_frame */ fil_decompress_page(slot->comp_buf, - dst_frame, - size, - &bpage->write_size); - -#ifdef UNIV_DEBUG - fil_page_type_validate(dst_frame); -#endif + dst_frame, + ulong(size), + &bpage->write_size); + ut_d(fil_page_type_validate(dst_frame)); } /* Mark this slot as free */ diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index b11c32064bf..ddef13ef8ea 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -538,7 +538,7 @@ buf_dblwr_process() /* Decompress the page before validating the checksum. */ fil_decompress_page( - NULL, read_buf, UNIV_PAGE_SIZE, + NULL, read_buf, srv_page_size, NULL, true); } @@ -565,7 +565,7 @@ buf_dblwr_process() /* Decompress the page before validating the checksum. */ fil_decompress_page( - NULL, page, UNIV_PAGE_SIZE, NULL, true); + NULL, page, srv_page_size, NULL, true); } if (!fil_space_verify_crypt_checksum(page, zip_size, NULL, page_no) diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 9c5f782da32..b8892b4ec31 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -694,7 +694,8 @@ fil_space_encrypt( comp_mem = (byte *)malloc(UNIV_PAGE_SIZE); uncomp_mem = (byte *)malloc(UNIV_PAGE_SIZE); memcpy(comp_mem, src_frame, UNIV_PAGE_SIZE); - fil_decompress_page(uncomp_mem, comp_mem, UNIV_PAGE_SIZE, NULL); + fil_decompress_page(uncomp_mem, comp_mem, + srv_page_size, NULL); src = uncomp_mem; } @@ -704,7 +705,8 @@ fil_space_encrypt( /* Need to decompress the page if it was also compressed */ if (page_compressed_encrypted) { memcpy(comp_mem, tmp_mem, UNIV_PAGE_SIZE); - fil_decompress_page(tmp_mem, comp_mem, UNIV_PAGE_SIZE, NULL); + fil_decompress_page(tmp_mem, comp_mem, + srv_page_size, NULL); } bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size, space); @@ -1492,20 +1494,21 @@ fil_crypt_realloc_iops( if (10 * state->cnt_waited > state->batch) { /* if we waited more than 10% re-estimate max_iops */ - uint avg_wait_time_us = + ulint avg_wait_time_us = state->sum_waited_us / state->cnt_waited; + if (avg_wait_time_us == 0) { + avg_wait_time_us = 1; // prevent division by zero + } + DBUG_PRINT("ib_crypt", - ("thr_no: %u - update estimated_max_iops from %u to %u.", + ("thr_no: %u - update estimated_max_iops from %u to " + ULINTPF ".", state->thread_no, state->estimated_max_iops, 1000000 / avg_wait_time_us)); - if (avg_wait_time_us == 0) { - avg_wait_time_us = 1; // prevent division by zero - } - - state->estimated_max_iops = 1000000 / avg_wait_time_us; + state->estimated_max_iops = uint(1000000 / avg_wait_time_us); state->cnt_waited = 0; state->sum_waited_us = 0; } else { diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 43788012920..fba7d5d8e38 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -6772,7 +6772,8 @@ fil_iterate( /* If the original page is page_compressed, we need to decompress page before we can update it. */ if (page_compressed) { - fil_decompress_page(NULL, dst, size, NULL); + fil_decompress_page(NULL, dst, ulong(size), + NULL); updated = true; } diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc index 303ab5102fb..655460c57ac 100644 --- a/storage/xtradb/fil/fil0pagecompress.cc +++ b/storage/xtradb/fil/fil0pagecompress.cc @@ -300,7 +300,8 @@ fil_compress_page( #endif /* HAVE_SNAPPY */ case PAGE_ZLIB_ALGORITHM: - err = compress2(out_buf+header_len, (ulong*)&write_size, buf, len, comp_level); + err = compress2(out_buf+header_len, (ulong*)&write_size, buf, + uLong(len), comp_level); if (err != Z_OK) { /* If error we leave the actual page as it was */ @@ -364,7 +365,7 @@ fil_compress_page( uncomp_page = static_cast(ut_malloc(UNIV_PAGE_SIZE)); memcpy(comp_page, out_buf, UNIV_PAGE_SIZE); - fil_decompress_page(uncomp_page, comp_page, len, NULL); + fil_decompress_page(uncomp_page, comp_page, ulong(len), NULL); if(buf_page_is_corrupted(false, uncomp_page, 0, space)) { buf_page_print(uncomp_page, 0, BUF_PAGE_PRINT_NO_CRASH); -- cgit v1.2.1 From d23eb8e6d27dfc6450465baa2897db5ebd334ea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 21 Apr 2017 16:06:05 +0300 Subject: Follow-up to MDEV-12488: Fix some type mismatch in header files This reduces the number of compilation warnings on Windows. --- storage/innobase/include/dict0dict.ic | 99 ++++++++++++++++++++------------- storage/innobase/include/fil0crypt.h | 6 +- storage/innobase/include/fil0fil.ic | 37 +++++-------- storage/xtradb/include/dict0dict.ic | 101 ++++++++++++++++++++-------------- storage/xtradb/include/fil0crypt.h | 6 +- storage/xtradb/include/fil0fil.ic | 38 +++++-------- 6 files changed, 151 insertions(+), 136 deletions(-) (limited to 'storage') diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index dafc36f1cab..cf5a6a22d65 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1996, 2016, Oracle and/or its affiliates +Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -541,13 +541,15 @@ dict_tf_is_valid( /* Make sure there are no bits that we do not know about. */ if (unused != 0) { fprintf(stderr, - "InnoDB: Error: table unused flags are %ld" + "InnoDB: Error: table unused flags are " ULINTPF " in the data dictionary and are corrupted\n" "InnoDB: Error: data dictionary flags are\n" - "InnoDB: compact %ld atomic_blobs %ld\n" - "InnoDB: unused %ld data_dir %ld zip_ssize %ld\n" - "InnoDB: page_compression %ld page_compression_level %ld\n" - "InnoDB: atomic_writes %ld\n", + "InnoDB: compact " ULINTPF " atomic_blobs " ULINTPF + "\nInnoDB: unused " ULINTPF " data_dir " ULINTPF + " zip_ssize " ULINTPF + "\nInnoDB: page_compression " ULINTPF + " page_compression_level " ULINTPF + "\nInnoDB: atomic_writes " ULINTPF "\n", unused, compact, atomic_blobs, unused, data_dir, zip_ssize, page_compression, page_compression_level, atomic_writes @@ -563,13 +565,17 @@ dict_tf_is_valid( if (!compact) { fprintf(stderr, - "InnoDB: Error: table compact flags are %ld" + "InnoDB: Error: table compact flags are " + ULINTPF " in the data dictionary and are corrupted\n" "InnoDB: Error: data dictionary flags are\n" - "InnoDB: compact %ld atomic_blobs %ld\n" - "InnoDB: unused %ld data_dir %ld zip_ssize %ld\n" - "InnoDB: page_compression %ld page_compression_level %ld\n" - "InnoDB: atomic_writes %ld\n", + "InnoDB: compact " ULINTPF + " atomic_blobs " ULINTPF "\n" + "InnoDB: unused " ULINTPF + " data_dir " ULINTPF " zip_ssize " ULINTPF + "\nInnoDB: page_compression " ULINTPF + " page_compression_level " ULINTPF + "\nInnoDB: atomic_writes " ULINTPF "\n", compact, compact, atomic_blobs, unused, data_dir, zip_ssize, page_compression, page_compression_level, atomic_writes ); @@ -580,13 +586,15 @@ dict_tf_is_valid( /* Antelope does not support COMPRESSED row format. */ fprintf(stderr, - "InnoDB: Error: table flags are %ld" + "InnoDB: Error: table flags are " ULINTPF " in the data dictionary and are corrupted\n" "InnoDB: Error: data dictionary flags are\n" - "InnoDB: compact %ld atomic_blobs %ld\n" - "InnoDB: unused %ld data_dir %ld zip_ssize %ld\n" - "InnoDB: page_compression %ld page_compression_level %ld\n" - "InnoDB: atomic_writes %ld\n", + "InnoDB: compact " ULINTPF " atomic_blobs " ULINTPF + "\nInnoDB: unused " ULINTPF " data_dir " ULINTPF + " zip_ssize " ULINTPF + "\nInnoDB: page_compression " ULINTPF + " page_compression_level " ULINTPF + "\nInnoDB: atomic_writes " ULINTPF "\n", flags, compact, atomic_blobs, unused, data_dir, zip_ssize, page_compression, page_compression_level, atomic_writes ); @@ -603,12 +611,17 @@ dict_tf_is_valid( || zip_ssize > PAGE_ZIP_SSIZE_MAX) { fprintf(stderr, - "InnoDB: Error: table compact flags are %ld in the data dictionary and are corrupted\n" + "InnoDB: Error: table compact flags are " + ULINTPF + " in the data dictionary and are corrupted\n" "InnoDB: Error: data dictionary flags are\n" - "InnoDB: compact %ld atomic_blobs %ld\n" - "InnoDB: unused %ld data_dir %ld zip_ssize %ld\n" - "InnoDB: page_compression %ld page_compression_level %ld\n" - "InnoDB: atomic_writes %ld\n", + "InnoDB: compact " ULINTPF + " atomic_blobs " ULINTPF "\n" + "InnoDB: unused " ULINTPF + " data_dir " ULINTPF " zip_ssize " ULINTPF + "\nInnoDB: page_compression " ULINTPF + " page_compression_level " ULINTPF + "\nInnoDB: atomic_writes " ULINTPF "\n", flags, compact, atomic_blobs, unused, data_dir, zip_ssize, page_compression, page_compression_level, atomic_writes @@ -627,12 +640,16 @@ dict_tf_is_valid( || !atomic_blobs) { fprintf(stderr, - "InnoDB: Error: table flags are %ld in the data dictionary and are corrupted\n" + "InnoDB: Error: table flags are " ULINTPF + " in the data dictionary and are corrupted\n" "InnoDB: Error: data dictionary flags are\n" - "InnoDB: compact %ld atomic_blobs %ld\n" - "InnoDB: unused %ld data_dir %ld zip_ssize %ld\n" - "InnoDB: page_compression %ld page_compression_level %ld\n" - "InnoDB: atomic_writes %ld\n", + "InnoDB: compact " ULINTPF + " atomic_blobs " ULINTPF "\n" + "InnoDB: unused " ULINTPF + " data_dir " ULINTPF " zip_ssize " ULINTPF + "\nInnoDB: page_compression " ULINTPF + " page_compression_level " ULINTPF + "\nInnoDB: atomic_writes " ULINTPF "\n", flags, compact, atomic_blobs, unused, data_dir, zip_ssize, page_compression, page_compression_level, atomic_writes ); @@ -645,12 +662,16 @@ dict_tf_is_valid( if(atomic_writes > ATOMIC_WRITES_OFF) { fprintf(stderr, - "InnoDB: Error: table flags are %ld in the data dictionary and are corrupted\n" + "InnoDB: Error: table flags are " ULINTPF + " in the data dictionary and are corrupted\n" "InnoDB: Error: data dictionary flags are\n" - "InnoDB: compact %ld atomic_blobs %ld\n" - "InnoDB: unused %ld data_dir %ld zip_ssize %ld\n" - "InnoDB: page_compression %ld page_compression_level %ld\n" - "InnoDB: atomic_writes %ld\n", + "InnoDB: compact " ULINTPF + " atomic_blobs " ULINTPF "\n" + "InnoDB: unused " ULINTPF + " data_dir " ULINTPF " zip_ssize " ULINTPF + "\nInnoDB: page_compression " ULINTPF + " page_compression_level " ULINTPF + "\nInnoDB: atomic_writes " ULINTPF "\n", flags, compact, atomic_blobs, unused, data_dir, zip_ssize, page_compression, page_compression_level, atomic_writes ); @@ -695,7 +716,7 @@ dict_sys_tables_type_validate( if (redundant) { if (zip_ssize || atomic_blobs) { - fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=Redundant, zip_ssize %lu atomic_blobs %lu\n", + fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=Redundant, zip_ssize " ULINTPF " atomic_blobs " ULINTPF "\n", zip_ssize, atomic_blobs); return(ULINT_UNDEFINED); } @@ -703,7 +724,7 @@ dict_sys_tables_type_validate( /* Make sure there are no bits that we do not know about. */ if (unused) { - fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=%lu, unused %lu\n", + fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=" ULINTPF ", unused " ULINTPF "\n", type, unused); return(ULINT_UNDEFINED); } @@ -719,7 +740,7 @@ dict_sys_tables_type_validate( } else if (zip_ssize) { /* Antelope does not support COMPRESSED format. */ - fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=%lu, zip_ssize %lu\n", + fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=" ULINTPF ", zip_ssize " ULINTPF "\n", type, zip_ssize); return(ULINT_UNDEFINED); } @@ -730,14 +751,14 @@ dict_sys_tables_type_validate( should be in N_COLS, but we already know about the low_order_bit and DICT_N_COLS_COMPACT flags. */ if (!atomic_blobs) { - fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=%lu, zip_ssize %lu atomic_blobs %lu\n", + fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=" ULINTPF ", zip_ssize " ULINTPF " atomic_blobs " ULINTPF "\n", type, zip_ssize, atomic_blobs); return(ULINT_UNDEFINED); } /* Validate that the number is within allowed range. */ if (zip_ssize > PAGE_ZIP_SSIZE_MAX) { - fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=%lu, zip_ssize %lu max %d\n", + fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=" ULINTPF ", zip_ssize " ULINTPF " max %d\n", type, zip_ssize, PAGE_ZIP_SSIZE_MAX); return(ULINT_UNDEFINED); } @@ -755,8 +776,8 @@ dict_sys_tables_type_validate( low_order_bit and DICT_N_COLS_COMPACT flags. */ if (!atomic_blobs || !page_compression) { - fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=%lu, page_compression %lu page_compression_level %lu\n" - "InnoDB: Error: atomic_blobs %lu\n", + fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=" ULINTPF ", page_compression " ULINTPF " page_compression_level " ULINTPF "\n" + "InnoDB: Error: atomic_blobs " ULINTPF "\n", type, page_compression, page_compression_level, atomic_blobs); return(ULINT_UNDEFINED); } @@ -764,7 +785,7 @@ dict_sys_tables_type_validate( /* Validate that the atomic writes number is within allowed range. */ if (atomic_writes > ATOMIC_WRITES_OFF) { - fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=%lu, atomic_writes %lu\n", + fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=" ULINTPF ", atomic_writes " ULINTPF "\n", type, atomic_writes); return(ULINT_UNDEFINED); } diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index cfc2d850883..e7e9676aa3a 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -109,7 +109,7 @@ struct fil_space_crypt_t : st_encryption_scheme The object is expected to be placed in a buffer that has been zero-initialized. */ fil_space_crypt_t( - ulint new_type, + uint new_type, uint new_min_key_version, uint new_key_id, fil_encryption_t new_encryption) @@ -117,10 +117,10 @@ struct fil_space_crypt_t : st_encryption_scheme min_key_version(new_min_key_version), page0_offset(0), encryption(new_encryption), - key_found(), + mutex(), + key_found(new_min_key_version), rotate_state() { - key_found = new_min_key_version; key_id = new_key_id; my_random_bytes(iv, sizeof(iv)); mutex_create(fil_crypt_data_mutex_key, diff --git a/storage/innobase/include/fil0fil.ic b/storage/innobase/include/fil0fil.ic index bea274d2c86..b79167baead 100644 --- a/storage/innobase/include/fil0fil.ic +++ b/storage/innobase/include/fil0fil.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2015, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -137,7 +137,7 @@ fil_page_type_validate( page_type == FIL_PAGE_TYPE_ZBLOB2 || page_type == FIL_PAGE_TYPE_COMPRESSED))) { - uint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + ulint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED); bool page_compressed_encrypted = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); ulint space = mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); @@ -149,28 +149,17 @@ fil_page_type_validate( fil_system_exit(); /* Dump out the page info */ - fprintf(stderr, "InnoDB: Space %lu offset %lu name %s page_type %lu page_type_name %s\n" - "InnoDB: key_version %u page_compressed %d page_compressed_encrypted %d lsn %llu compressed_len %lu\n", - space, offset, rspace->name, page_type, fil_get_page_type_name(page_type), - key_version, page_compressed, page_compressed_encrypted, (ulonglong)lsn, compressed_len); - fflush(stderr); - - ut_ad(page_type == FIL_PAGE_PAGE_COMPRESSED || - page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED || - page_type == FIL_PAGE_INDEX || - page_type == FIL_PAGE_UNDO_LOG || - page_type == FIL_PAGE_INODE || - page_type == FIL_PAGE_IBUF_FREE_LIST || - page_type == FIL_PAGE_TYPE_ALLOCATED || - page_type == FIL_PAGE_IBUF_BITMAP || - page_type == FIL_PAGE_TYPE_SYS || - page_type == FIL_PAGE_TYPE_TRX_SYS || - page_type == FIL_PAGE_TYPE_FSP_HDR || - page_type == FIL_PAGE_TYPE_XDES || - page_type == FIL_PAGE_TYPE_BLOB || - page_type == FIL_PAGE_TYPE_ZBLOB || - page_type == FIL_PAGE_TYPE_ZBLOB2 || - page_type == FIL_PAGE_TYPE_COMPRESSED); + fprintf(stderr, "InnoDB: Page " ULINTPF ":" ULINTPF + " name %s page_type " ULINTPF " page_type_name %s\n" + "InnoDB: key_version " ULINTPF + " page_compressed %d page_compressed_encrypted %d lsn " + LSN_PF " compressed_len " ULINTPF "\n", + space, offset, rspace->name, page_type, + fil_get_page_type_name(page_type), + key_version, + page_compressed, page_compressed_encrypted, + lsn, compressed_len); + ut_error; return false; } diff --git a/storage/xtradb/include/dict0dict.ic b/storage/xtradb/include/dict0dict.ic index 81da2fa5580..e3de7a33123 100644 --- a/storage/xtradb/include/dict0dict.ic +++ b/storage/xtradb/include/dict0dict.ic @@ -540,15 +540,16 @@ dict_tf_is_valid( /* Make sure there are no bits that we do not know about. */ if (unused != 0) { - fprintf(stderr, - "InnoDB: Error: table unused flags are %ld" + "InnoDB: Error: table unused flags are " ULINTPF " in the data dictionary and are corrupted\n" "InnoDB: Error: data dictionary flags are\n" - "InnoDB: compact %ld atomic_blobs %ld\n" - "InnoDB: unused %ld data_dir %ld zip_ssize %ld\n" - "InnoDB: page_compression %ld page_compression_level %ld\n" - "InnoDB: atomic_writes %ld\n", + "InnoDB: compact " ULINTPF " atomic_blobs " ULINTPF + "\nInnoDB: unused " ULINTPF " data_dir " ULINTPF + " zip_ssize " ULINTPF + "\nInnoDB: page_compression " ULINTPF + " page_compression_level " ULINTPF + "\nInnoDB: atomic_writes " ULINTPF "\n", unused, compact, atomic_blobs, unused, data_dir, zip_ssize, page_compression, page_compression_level, atomic_writes @@ -564,17 +565,20 @@ dict_tf_is_valid( if (!compact) { fprintf(stderr, - "InnoDB: Error: table compact flags are %ld" + "InnoDB: Error: table compact flags are " + ULINTPF " in the data dictionary and are corrupted\n" "InnoDB: Error: data dictionary flags are\n" - "InnoDB: compact %ld atomic_blobs %ld\n" - "InnoDB: unused %ld data_dir %ld zip_ssize %ld\n" - "InnoDB: page_compression %ld page_compression_level %ld\n" - "InnoDB: atomic_writes %ld\n", + "InnoDB: compact " ULINTPF + " atomic_blobs " ULINTPF "\n" + "InnoDB: unused " ULINTPF + " data_dir " ULINTPF " zip_ssize " ULINTPF + "\nInnoDB: page_compression " ULINTPF + " page_compression_level " ULINTPF + "\nInnoDB: atomic_writes " ULINTPF "\n", compact, compact, atomic_blobs, unused, data_dir, zip_ssize, page_compression, page_compression_level, atomic_writes ); - return(false); } @@ -582,17 +586,18 @@ dict_tf_is_valid( /* Antelope does not support COMPRESSED row format. */ fprintf(stderr, - "InnoDB: Error: table flags are %ld" + "InnoDB: Error: table flags are " ULINTPF " in the data dictionary and are corrupted\n" "InnoDB: Error: data dictionary flags are\n" - "InnoDB: compact %ld atomic_blobs %ld\n" - "InnoDB: unused %ld data_dir %ld zip_ssize %ld\n" - "InnoDB: page_compression %ld page_compression_level %ld\n" - "InnoDB: atomic_writes %ld\n", + "InnoDB: compact " ULINTPF " atomic_blobs " ULINTPF + "\nInnoDB: unused " ULINTPF " data_dir " ULINTPF + " zip_ssize " ULINTPF + "\nInnoDB: page_compression " ULINTPF + " page_compression_level " ULINTPF + "\nInnoDB: atomic_writes " ULINTPF "\n", flags, compact, atomic_blobs, unused, data_dir, zip_ssize, page_compression, page_compression_level, atomic_writes ); - return(false); } @@ -606,12 +611,17 @@ dict_tf_is_valid( || zip_ssize > PAGE_ZIP_SSIZE_MAX) { fprintf(stderr, - "InnoDB: Error: table compact flags are %ld in the data dictionary and are corrupted\n" + "InnoDB: Error: table compact flags are " + ULINTPF + " in the data dictionary and are corrupted\n" "InnoDB: Error: data dictionary flags are\n" - "InnoDB: compact %ld atomic_blobs %ld\n" - "InnoDB: unused %ld data_dir %ld zip_ssize %ld\n" - "InnoDB: page_compression %ld page_compression_level %ld\n" - "InnoDB: atomic_writes %ld\n", + "InnoDB: compact " ULINTPF + " atomic_blobs " ULINTPF "\n" + "InnoDB: unused " ULINTPF + " data_dir " ULINTPF " zip_ssize " ULINTPF + "\nInnoDB: page_compression " ULINTPF + " page_compression_level " ULINTPF + "\nInnoDB: atomic_writes " ULINTPF "\n", flags, compact, atomic_blobs, unused, data_dir, zip_ssize, page_compression, page_compression_level, atomic_writes @@ -630,16 +640,19 @@ dict_tf_is_valid( || !atomic_blobs) { fprintf(stderr, - "InnoDB: Error: table flags are %ld in the data dictionary and are corrupted\n" + "InnoDB: Error: table flags are " ULINTPF + " in the data dictionary and are corrupted\n" "InnoDB: Error: data dictionary flags are\n" - "InnoDB: compact %ld atomic_blobs %ld\n" - "InnoDB: unused %ld data_dir %ld zip_ssize %ld\n" - "InnoDB: page_compression %ld page_compression_level %ld\n" - "InnoDB: atomic_writes %ld\n", + "InnoDB: compact " ULINTPF + " atomic_blobs " ULINTPF "\n" + "InnoDB: unused " ULINTPF + " data_dir " ULINTPF " zip_ssize " ULINTPF + "\nInnoDB: page_compression " ULINTPF + " page_compression_level " ULINTPF + "\nInnoDB: atomic_writes " ULINTPF "\n", flags, compact, atomic_blobs, unused, data_dir, zip_ssize, page_compression, page_compression_level, atomic_writes ); - return(false); } } @@ -649,12 +662,16 @@ dict_tf_is_valid( if(atomic_writes > ATOMIC_WRITES_OFF) { fprintf(stderr, - "InnoDB: Error: table flags are %ld in the data dictionary and are corrupted\n" + "InnoDB: Error: table flags are " ULINTPF + " in the data dictionary and are corrupted\n" "InnoDB: Error: data dictionary flags are\n" - "InnoDB: compact %ld atomic_blobs %ld\n" - "InnoDB: unused %ld data_dir %ld zip_ssize %ld\n" - "InnoDB: page_compression %ld page_compression_level %ld\n" - "InnoDB: atomic_writes %ld\n", + "InnoDB: compact " ULINTPF + " atomic_blobs " ULINTPF "\n" + "InnoDB: unused " ULINTPF + " data_dir " ULINTPF " zip_ssize " ULINTPF + "\nInnoDB: page_compression " ULINTPF + " page_compression_level " ULINTPF + "\nInnoDB: atomic_writes " ULINTPF "\n", flags, compact, atomic_blobs, unused, data_dir, zip_ssize, page_compression, page_compression_level, atomic_writes ); @@ -699,7 +716,7 @@ dict_sys_tables_type_validate( if (redundant) { if (zip_ssize || atomic_blobs) { - fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=Redundant, zip_ssize %lu atomic_blobs %lu\n", + fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=Redundant, zip_ssize " ULINTPF " atomic_blobs " ULINTPF "\n", zip_ssize, atomic_blobs); return(ULINT_UNDEFINED); } @@ -707,7 +724,7 @@ dict_sys_tables_type_validate( /* Make sure there are no bits that we do not know about. */ if (unused) { - fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=%lu, unused %lu\n", + fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=" ULINTPF ", unused " ULINTPF "\n", type, unused); return(ULINT_UNDEFINED); } @@ -723,7 +740,7 @@ dict_sys_tables_type_validate( } else if (zip_ssize) { /* Antelope does not support COMPRESSED format. */ - fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=%lu, zip_ssize %lu\n", + fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=" ULINTPF ", zip_ssize " ULINTPF "\n", type, zip_ssize); return(ULINT_UNDEFINED); } @@ -734,14 +751,14 @@ dict_sys_tables_type_validate( should be in N_COLS, but we already know about the low_order_bit and DICT_N_COLS_COMPACT flags. */ if (!atomic_blobs) { - fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=%lu, zip_ssize %lu atomic_blobs %lu\n", + fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=" ULINTPF ", zip_ssize " ULINTPF " atomic_blobs " ULINTPF "\n", type, zip_ssize, atomic_blobs); return(ULINT_UNDEFINED); } /* Validate that the number is within allowed range. */ if (zip_ssize > PAGE_ZIP_SSIZE_MAX) { - fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=%lu, zip_ssize %lu max %d\n", + fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=" ULINTPF ", zip_ssize " ULINTPF " max %d\n", type, zip_ssize, PAGE_ZIP_SSIZE_MAX); return(ULINT_UNDEFINED); } @@ -759,8 +776,8 @@ dict_sys_tables_type_validate( low_order_bit and DICT_N_COLS_COMPACT flags. */ if (!atomic_blobs || !page_compression) { - fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=%lu, page_compression %lu page_compression_level %lu\n" - "InnoDB: Error: atomic_blobs %lu\n", + fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=" ULINTPF ", page_compression " ULINTPF " page_compression_level " ULINTPF "\n" + "InnoDB: Error: atomic_blobs " ULINTPF "\n", type, page_compression, page_compression_level, atomic_blobs); return(ULINT_UNDEFINED); } @@ -768,7 +785,7 @@ dict_sys_tables_type_validate( /* Validate that the atomic writes number is within allowed range. */ if (atomic_writes > ATOMIC_WRITES_OFF) { - fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=%lu, atomic_writes %lu\n", + fprintf(stderr, "InnoDB: Error: SYS_TABLES::TYPE=" ULINTPF ", atomic_writes " ULINTPF "\n", type, atomic_writes); return(ULINT_UNDEFINED); } diff --git a/storage/xtradb/include/fil0crypt.h b/storage/xtradb/include/fil0crypt.h index cfc2d850883..e7e9676aa3a 100644 --- a/storage/xtradb/include/fil0crypt.h +++ b/storage/xtradb/include/fil0crypt.h @@ -109,7 +109,7 @@ struct fil_space_crypt_t : st_encryption_scheme The object is expected to be placed in a buffer that has been zero-initialized. */ fil_space_crypt_t( - ulint new_type, + uint new_type, uint new_min_key_version, uint new_key_id, fil_encryption_t new_encryption) @@ -117,10 +117,10 @@ struct fil_space_crypt_t : st_encryption_scheme min_key_version(new_min_key_version), page0_offset(0), encryption(new_encryption), - key_found(), + mutex(), + key_found(new_min_key_version), rotate_state() { - key_found = new_min_key_version; key_id = new_key_id; my_random_bytes(iv, sizeof(iv)); mutex_create(fil_crypt_data_mutex_key, diff --git a/storage/xtradb/include/fil0fil.ic b/storage/xtradb/include/fil0fil.ic index 1179eea8b8e..b79167baead 100644 --- a/storage/xtradb/include/fil0fil.ic +++ b/storage/xtradb/include/fil0fil.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2015, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -137,7 +137,7 @@ fil_page_type_validate( page_type == FIL_PAGE_TYPE_ZBLOB2 || page_type == FIL_PAGE_TYPE_COMPRESSED))) { - uint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + ulint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED); bool page_compressed_encrypted = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); ulint space = mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); @@ -148,30 +148,18 @@ fil_page_type_validate( fil_space_t* rspace = fil_space_get_by_id(space); fil_system_exit(); - /* Dump out the page info */ - fprintf(stderr, "InnoDB: Space %lu offset %lu name %s page_type %lu page_type_name %s\n" - "InnoDB: key_version %u page_compressed %d page_compressed_encrypted %d lsn %llu compressed_len %lu\n", - space, offset, rspace->name, page_type, fil_get_page_type_name(page_type), - key_version, page_compressed, page_compressed_encrypted, (ulonglong)lsn, compressed_len); - fflush(stderr); - - ut_ad(page_type == FIL_PAGE_PAGE_COMPRESSED || - page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED || - page_type == FIL_PAGE_INDEX || - page_type == FIL_PAGE_UNDO_LOG || - page_type == FIL_PAGE_INODE || - page_type == FIL_PAGE_IBUF_FREE_LIST || - page_type == FIL_PAGE_TYPE_ALLOCATED || - page_type == FIL_PAGE_IBUF_BITMAP || - page_type == FIL_PAGE_TYPE_SYS || - page_type == FIL_PAGE_TYPE_TRX_SYS || - page_type == FIL_PAGE_TYPE_FSP_HDR || - page_type == FIL_PAGE_TYPE_XDES || - page_type == FIL_PAGE_TYPE_BLOB || - page_type == FIL_PAGE_TYPE_ZBLOB || - page_type == FIL_PAGE_TYPE_ZBLOB2 || - page_type == FIL_PAGE_TYPE_COMPRESSED); + fprintf(stderr, "InnoDB: Page " ULINTPF ":" ULINTPF + " name %s page_type " ULINTPF " page_type_name %s\n" + "InnoDB: key_version " ULINTPF + " page_compressed %d page_compressed_encrypted %d lsn " + LSN_PF " compressed_len " ULINTPF "\n", + space, offset, rspace->name, page_type, + fil_get_page_type_name(page_type), + key_version, + page_compressed, page_compressed_encrypted, + lsn, compressed_len); + ut_error; return false; } -- cgit v1.2.1 From 555e52f3bc8d67d0cfe176d25e2fe0c73fdb2fd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 21 Apr 2017 11:52:25 +0300 Subject: MDEV-12467 encryption.create_or_replace hangs during DROP TABLE fil_crypt_thread(): Do invoke fil_crypt_complete_rotate_space() when the tablespace is about to be dropped. Also, remove a redundant check whether rotate_thread_t::space is NULL. It can only become NULL when fil_crypt_find_space_to_rotate() returns false, and in that case we would already have terminated the loop. fil_crypt_find_page_to_rotate(): Remove a redundant check for space->crypt_data == NULL. Once encryption metadata has been created for a tablespace, it cannot be removed without dropping the entire tablespace. --- storage/innobase/fil/fil0crypt.cc | 40 ++++++++++++++++++--------------------- storage/xtradb/fil/fil0crypt.cc | 40 ++++++++++++++++++--------------------- 2 files changed, 36 insertions(+), 44 deletions(-) (limited to 'storage') diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index b8892b4ec31..da8e538022d 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -1742,33 +1742,27 @@ fil_crypt_find_page_to_rotate( fil_space_crypt_t *crypt_data = space->crypt_data; - /* Space might already be dropped */ - if (crypt_data) { - mutex_enter(&crypt_data->mutex); - ut_ad(key_state->key_id == crypt_data->key_id); - - if (crypt_data->rotate_state.next_offset < - crypt_data->rotate_state.max_offset) { + mutex_enter(&crypt_data->mutex); + ut_ad(key_state->key_id == crypt_data->key_id); - state->offset = crypt_data->rotate_state.next_offset; - ulint remaining = crypt_data->rotate_state.max_offset - - crypt_data->rotate_state.next_offset; + bool found = crypt_data->rotate_state.max_offset >= + crypt_data->rotate_state.next_offset; - if (batch <= remaining) { - state->batch = batch; - } else { - state->batch = remaining; - } + if (found) { + state->offset = crypt_data->rotate_state.next_offset; + ulint remaining = crypt_data->rotate_state.max_offset - + crypt_data->rotate_state.next_offset; - crypt_data->rotate_state.next_offset += batch; - mutex_exit(&crypt_data->mutex); - return true; + if (batch <= remaining) { + state->batch = batch; + } else { + state->batch = remaining; } - - mutex_exit(&crypt_data->mutex); } - return false; + crypt_data->rotate_state.next_offset += batch; + mutex_exit(&crypt_data->mutex); + return found; } /*********************************************************************** @@ -2346,7 +2340,7 @@ DECLARE_THREAD(fil_crypt_thread)( fil_crypt_start_rotate_space(&new_state, &thr); /* iterate all pages (cooperativly with other threads) */ - while (!thr.should_shutdown() && thr.space && + while (!thr.should_shutdown() && fil_crypt_find_page_to_rotate(&new_state, &thr)) { /* rotate a (set) of pages */ @@ -2355,6 +2349,8 @@ DECLARE_THREAD(fil_crypt_thread)( /* If space is marked as stopping, release space and stop rotation. */ if (thr.space->is_stopping()) { + fil_crypt_complete_rotate_space( + &new_state, &thr); fil_space_release(thr.space); thr.space = NULL; break; diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index b8892b4ec31..da8e538022d 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -1742,33 +1742,27 @@ fil_crypt_find_page_to_rotate( fil_space_crypt_t *crypt_data = space->crypt_data; - /* Space might already be dropped */ - if (crypt_data) { - mutex_enter(&crypt_data->mutex); - ut_ad(key_state->key_id == crypt_data->key_id); - - if (crypt_data->rotate_state.next_offset < - crypt_data->rotate_state.max_offset) { + mutex_enter(&crypt_data->mutex); + ut_ad(key_state->key_id == crypt_data->key_id); - state->offset = crypt_data->rotate_state.next_offset; - ulint remaining = crypt_data->rotate_state.max_offset - - crypt_data->rotate_state.next_offset; + bool found = crypt_data->rotate_state.max_offset >= + crypt_data->rotate_state.next_offset; - if (batch <= remaining) { - state->batch = batch; - } else { - state->batch = remaining; - } + if (found) { + state->offset = crypt_data->rotate_state.next_offset; + ulint remaining = crypt_data->rotate_state.max_offset - + crypt_data->rotate_state.next_offset; - crypt_data->rotate_state.next_offset += batch; - mutex_exit(&crypt_data->mutex); - return true; + if (batch <= remaining) { + state->batch = batch; + } else { + state->batch = remaining; } - - mutex_exit(&crypt_data->mutex); } - return false; + crypt_data->rotate_state.next_offset += batch; + mutex_exit(&crypt_data->mutex); + return found; } /*********************************************************************** @@ -2346,7 +2340,7 @@ DECLARE_THREAD(fil_crypt_thread)( fil_crypt_start_rotate_space(&new_state, &thr); /* iterate all pages (cooperativly with other threads) */ - while (!thr.should_shutdown() && thr.space && + while (!thr.should_shutdown() && fil_crypt_find_page_to_rotate(&new_state, &thr)) { /* rotate a (set) of pages */ @@ -2355,6 +2349,8 @@ DECLARE_THREAD(fil_crypt_thread)( /* If space is marked as stopping, release space and stop rotation. */ if (thr.space->is_stopping()) { + fil_crypt_complete_rotate_space( + &new_state, &thr); fil_space_release(thr.space); thr.space = NULL; break; -- cgit v1.2.1 From 996c7d5cb5e369de178da3482f3dfc36d57fd6a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 21 Apr 2017 17:32:02 +0300 Subject: MDEV-12545 Reduce the amount of fil_space_t lookups buf_flush_write_block_low(): Acquire the tablespace reference once, and pass it to lower-level functions. This is only a start; further calls may be removed later. --- storage/innobase/buf/buf0buf.cc | 98 +++++++++++---------------- storage/innobase/buf/buf0dblwr.cc | 4 +- storage/innobase/buf/buf0flu.cc | 16 +++-- storage/innobase/fil/fil0fil.cc | 76 +++++++++++++-------- storage/innobase/fil/fil0pagecompress.cc | 63 +++++++++--------- storage/innobase/include/buf0buf.h | 18 ++--- storage/innobase/include/fil0fil.h | 29 ++++---- storage/innobase/include/fil0pagecompress.h | 30 +-------- storage/innobase/include/fsp0pagecompress.ic | 63 ------------------ storage/xtradb/buf/buf0buf.cc | 99 +++++++++++----------------- storage/xtradb/buf/buf0dblwr.cc | 4 +- storage/xtradb/buf/buf0flu.cc | 16 +++-- storage/xtradb/fil/fil0fil.cc | 76 +++++++++++++-------- storage/xtradb/fil/fil0pagecompress.cc | 62 +++++++++-------- storage/xtradb/include/buf0buf.h | 18 ++--- storage/xtradb/include/fil0fil.h | 27 ++++---- storage/xtradb/include/fil0pagecompress.h | 30 +-------- storage/xtradb/include/fsp0pagecompress.ic | 63 ------------------ 18 files changed, 309 insertions(+), 483 deletions(-) (limited to 'storage') diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 2133457fcbb..bb567119a13 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -6080,71 +6080,54 @@ buf_pool_reserve_tmp_slot( return (free_slot); } -/********************************************************************//** -Encrypts a buffer page right before it's flushed to disk -@param[in,out] bpage Page control block -@param[in,out] src_frame Source page -@param[in] space_id Tablespace id -@return either unencrypted source page or decrypted page. -*/ +/** Encryption and page_compression hook that is called just before +a page is written to disk. +@param[in,out] space tablespace +@param[in,out] bpage buffer page +@param[in] src_frame physical page frame that is being encrypted +@return page frame to be written to file +(may be src_frame or an encrypted/compressed copy of it) */ UNIV_INTERN byte* buf_page_encrypt_before_write( + fil_space_t* space, buf_page_t* bpage, - byte* src_frame, - ulint space_id) + byte* src_frame) { + ut_ad(space->id == bpage->space); bpage->real_size = UNIV_PAGE_SIZE; fil_page_type_validate(src_frame); - if (bpage->offset == 0) { + switch (bpage->offset) { + case 0: /* Page 0 of a tablespace is not encrypted/compressed */ ut_ad(bpage->key_version == 0); return src_frame; - } - - if (bpage->space == TRX_SYS_SPACE && bpage->offset == TRX_SYS_PAGE_NO) { - /* don't encrypt/compress page as it contains address to dblwr buffer */ - bpage->key_version = 0; - return src_frame; - } - - fil_space_t* space = fil_space_acquire_silent(space_id); - - /* Tablespace must exist during write operation */ - if (!space) { - /* This could be true on discard if we have injected a error - case e.g. in innodb.innodb-wl5522-debug-zip so that space - is already marked as stop_new_ops = true. */ - return src_frame; + case TRX_SYS_PAGE_NO: + if (bpage->space == TRX_SYS_SPACE) { + /* don't encrypt/compress page as it contains + address to dblwr buffer */ + bpage->key_version = 0; + return src_frame; + } } fil_space_crypt_t* crypt_data = space->crypt_data; - bool encrypted = true; - - if (space->crypt_data != NULL && space->crypt_data->not_encrypted()) { - /* Encryption is disabled */ - encrypted = false; - } + const bool encrypted = crypt_data + && !crypt_data->not_encrypted() + && crypt_data->type != CRYPT_SCHEME_UNENCRYPTED + && (!crypt_data->is_default_encryption() + || srv_encrypt_tables); - if (!srv_encrypt_tables && (crypt_data == NULL || crypt_data->is_default_encryption())) { - /* Encryption is disabled */ - encrypted = false; - } - - /* Is encryption needed? */ - if (crypt_data == NULL || crypt_data->type == CRYPT_SCHEME_UNENCRYPTED) { - /* An unencrypted table */ + if (!encrypted) { bpage->key_version = 0; - encrypted = false; } - bool page_compressed = fil_space_is_page_compressed(bpage->space); + bool page_compressed = FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags); if (!encrypted && !page_compressed) { /* No need to encrypt or page compress the page */ - fil_space_release(space); return src_frame; } @@ -6172,25 +6155,21 @@ buf_page_encrypt_before_write( bpage->real_size = page_size; slot->out_buf = dst_frame = tmp; -#ifdef UNIV_DEBUG - fil_page_type_validate(tmp); -#endif - + ut_d(fil_page_type_validate(tmp)); } else { /* First we compress the page content */ ulint out_len = 0; - ulint block_size = fil_space_get_block_size(bpage->space, bpage->offset, page_size); - - byte *tmp = fil_compress_page(bpage->space, - (byte *)src_frame, - slot->comp_buf, - page_size, - fil_space_get_page_compression_level(bpage->space), - block_size, - encrypted, - &out_len, - IF_LZO(slot->lzo_mem, NULL) - ); + + byte *tmp = fil_compress_page( + space, + (byte *)src_frame, + slot->comp_buf, + page_size, + fsp_flags_get_page_compression_level(space->flags), + fil_space_get_block_size(space, bpage->offset), + encrypted, + &out_len, + IF_LZO(slot->lzo_mem, NULL)); bpage->real_size = out_len; @@ -6215,7 +6194,6 @@ buf_page_encrypt_before_write( fil_page_type_validate(dst_frame); #endif - fil_space_release(space); // return dst_frame which will be written return dst_frame; } diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index f99679f6fec..cedbff96a67 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -986,7 +986,7 @@ flush: srv_stats.dblwr_writes.inc(); /* Now flush the doublewrite buffer data to disk */ - fil_flush(TRX_SYS_SPACE); + fil_flush(ulint(TRX_SYS_SPACE)); /* We know that the writes have been flushed to disk now and in recovery we will find them in the doublewrite buffer @@ -1231,7 +1231,7 @@ retry: } /* Now flush the doublewrite buffer data to disk */ - fil_flush(TRX_SYS_SPACE); + fil_flush(ulint(TRX_SYS_SPACE)); /* We know that the write has been flushed to disk now and during recovery we will find it in the doublewrite buffer diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 3f03ee8256f..87a5386ce25 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -831,11 +831,12 @@ buf_flush_write_block_low( buf_flush_t flush_type, /*!< in: type of flush */ bool sync) /*!< in: true if sync IO request */ { + fil_space_t* space = fil_space_acquire(bpage->space, true); + if (!space) { + return; + } ulint zip_size = buf_page_get_zip_size(bpage); page_t* frame = NULL; - ulint space_id = buf_page_get_space(bpage); - atomic_writes_t awrites = fil_space_get_atomic_writes(space_id); - #ifdef UNIV_DEBUG buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); ut_ad(!buf_pool_mutex_own(buf_pool)); @@ -906,7 +907,7 @@ buf_flush_write_block_low( break; } - frame = buf_page_encrypt_before_write(bpage, frame, space_id); + frame = buf_page_encrypt_before_write(space, bpage, frame); if (!srv_use_doublewrite_buf || !buf_dblwr) { fil_io(OS_FILE_WRITE | OS_AIO_SIMULATED_WAKE_LATER, @@ -928,7 +929,8 @@ buf_flush_write_block_low( atomic writes should be used, no doublewrite buffer is used. */ - if (awrites == ATOMIC_WRITES_ON) { + if (fsp_flags_get_atomic_writes(space->flags) + == ATOMIC_WRITES_ON) { fil_io(OS_FILE_WRITE | OS_AIO_SIMULATED_WAKE_LATER, FALSE, buf_page_get_space(bpage), @@ -952,13 +954,15 @@ buf_flush_write_block_low( are working on. */ if (sync) { ut_ad(flush_type == BUF_FLUSH_SINGLE_PAGE); - fil_flush(buf_page_get_space(bpage)); + fil_flush(space); /* true means we want to evict this page from the LRU list as well. */ buf_page_io_complete(bpage, true); } + fil_space_release(space); + /* Increment the counter of I/O operations used for selecting LRU policy. */ buf_LRU_stat_inc_io(); diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index ceda86e16b6..2a812fa8a9f 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -5956,31 +5956,34 @@ fil_space_get_node( return (node); } -/********************************************************************//** -Return block size of node in file space -@return file block size */ + +/** Determine the block size of the data file. +@param[in] space tablespace +@param[in] offset page number +@return block size */ UNIV_INTERN ulint -fil_space_get_block_size( -/*=====================*/ - ulint space_id, - ulint block_offset, - ulint len) +fil_space_get_block_size(const fil_space_t* space, unsigned offset) { - ulint block_size = 512; - ut_ad(!mutex_own(&fil_system->mutex)); - - mutex_enter(&fil_system->mutex); - fil_space_t* space = fil_space_get_space(space_id); + ut_ad(space->n_pending_ops > 0); - if (space) { - fil_node_t* node = fil_space_get_node(space, space_id, &block_offset, 0, len); + ulint block_size = 512; - if (node) { - block_size = node->file_block_size; + for (fil_node_t* node = UT_LIST_GET_FIRST(space->chain); + node != NULL; + node = UT_LIST_GET_NEXT(chain, node)) { + block_size = node->file_block_size; + if (node->size > offset) { + break; } + offset -= node->size; + } + + /* Currently supporting block size up to 4K, + fall back to default if bigger requested. */ + if (block_size > 4096) { + block_size = 512; } - mutex_exit(&fil_system->mutex); return block_size; } @@ -6334,14 +6337,29 @@ fil_flush( mutex_exit(&fil_system->mutex); } -/**********************************************************************//** -Flushes to disk the writes in file spaces of the given type possibly cached by -the OS. */ +/** Flush a tablespace. +@param[in,out] space tablespace to flush */ UNIV_INTERN void -fil_flush_file_spaces( -/*==================*/ - ulint purpose) /*!< in: FIL_TABLESPACE, FIL_LOG */ +fil_flush(fil_space_t* space) +{ + ut_ad(space->n_pending_ops > 0); + + if (!space->is_stopping()) { + mutex_enter(&fil_system->mutex); + if (!space->is_stopping()) { + fil_flush_low(space); + } + mutex_exit(&fil_system->mutex); + } +} + +/** Flush to disk the writes in file spaces of the given type +possibly cached by the OS. +@param[in] purpose FIL_TYPE_TABLESPACE or FIL_TYPE_LOG */ +UNIV_INTERN +void +fil_flush_file_spaces(ulint purpose) { fil_space_t* space; ulint* space_ids; @@ -6769,12 +6787,14 @@ fil_iterate( if (page_compressed) { ulint len = 0; - fil_compress_page(space_id, + + fil_compress_page( + NULL, src, NULL, size, - fil_space_get_page_compression_level(space_id), - fil_space_get_block_size(space_id, offset, size), + 0,/* FIXME: compression level */ + 512,/* FIXME: use proper block size */ encrypted, &len, NULL); @@ -6785,6 +6805,8 @@ fil_iterate( /* If tablespace is encrypted, encrypt page before we write it back. Note that we should not encrypt the buffer that is in buffer pool. */ + /* NOTE: At this stage of IMPORT the + buffer pool is not being used at all! */ if (decrypted && encrypted) { byte *dest = writeptr + (i * size); ulint space = mach_read_from_4( diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index 42f894f08d7..73df69beb90 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -91,8 +91,7 @@ UNIV_INTERN byte* fil_compress_page( /*==============*/ - ulint space_id, /*!< in: tablespace id of the - table. */ + fil_space_t* space, /*!< in,out: tablespace (NULL during IMPORT) */ byte* buf, /*!< in: buffer from which to write; in aio this must be appropriately aligned */ byte* out_buf, /*!< out: compressed buffer */ @@ -110,9 +109,12 @@ fil_compress_page( ulint write_size=0; /* Cache to avoid change during function execution */ ulint comp_method = innodb_compression_algorithm; - ulint orig_page_type; bool allocated=false; + /* page_compression does not apply to tables or tablespaces + that use ROW_FORMAT=COMPRESSED */ + ut_ad(!space || !FSP_FLAGS_GET_ZIP_SSIZE(space->flags)); + if (encrypted) { header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE; } @@ -133,21 +135,14 @@ fil_compress_page( ut_ad(len); ut_ad(out_len); - /* read original page type */ - orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); - - fil_system_enter(); - fil_space_t* space = fil_space_get_by_id(space_id); - fil_system_exit(); - /* Let's not compress file space header or extent descriptor */ - if (orig_page_type == 0 || - orig_page_type == FIL_PAGE_TYPE_FSP_HDR || - orig_page_type == FIL_PAGE_TYPE_XDES || - orig_page_type == FIL_PAGE_PAGE_COMPRESSED) { + switch (fil_page_get_type(buf)) { + case 0: + case FIL_PAGE_TYPE_FSP_HDR: + case FIL_PAGE_TYPE_XDES: + case FIL_PAGE_PAGE_COMPRESSED: *out_len = len; - goto err_exit; } @@ -157,11 +152,11 @@ fil_compress_page( comp_level = page_zip_level; } -#ifdef UNIV_PAGECOMPRESS_DEBUG - ib_logf(IB_LOG_LEVEL_INFO, - "Preparing for compress for space %lu name %s len %lu.", - space_id, fil_space_name(space), len); -#endif /* UNIV_PAGECOMPRESS_DEBUG */ + DBUG_PRINT("compress", + ("Preparing for space " ULINTPF " '%s' len " ULINTPF, + space ? space->id : 0, + space ? space->name : "(import)", + len)); write_size = UNIV_PAGE_SIZE - header_len; @@ -306,10 +301,13 @@ fil_compress_page( if (err != Z_OK) { /* If error we leave the actual page as it was */ - if (space->printed_compression_failure == false) { + if (space && !space->printed_compression_failure) { ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space %lu name %s len %lu rt %d write %lu.", - space_id, fil_space_name(space), len, err, write_size); + "Compression failed for space " + ULINTPF " name %s len " ULINTPF + " rt %d write " ULINTPF ".", + space->id, space->name, len, err, + write_size); space->printed_compression_failure = true; } @@ -367,9 +365,8 @@ fil_compress_page( fil_decompress_page(uncomp_page, comp_page, ulong(len), NULL); - if(buf_page_is_corrupted(false, uncomp_page, 0, space)) { - buf_page_print(uncomp_page, 0, BUF_PAGE_PRINT_NO_CRASH); - ut_error; + if (buf_page_is_corrupted(false, uncomp_page, 0, space)) { + buf_page_print(uncomp_page, 0, 0); } ut_free(comp_page); @@ -388,7 +385,6 @@ fil_compress_page( /* Actual write needs to be alligned on block size */ if (write_size % block_size) { size_t tmp = write_size; - write_size = (size_t)ut_uint64_align_up((ib_uint64_t)write_size, block_size); /* Clean up the end of buffer */ memset(out_buf+tmp, 0, write_size - tmp); @@ -398,11 +394,12 @@ fil_compress_page( #endif } -#ifdef UNIV_PAGECOMPRESS_DEBUG - ib_logf(IB_LOG_LEVEL_INFO, - "Compression succeeded for space %lu name %s len %lu out_len %lu.", - space_id, fil_space_name(space), len, write_size); -#endif /* UNIV_PAGECOMPRESS_DEBUG */ + DBUG_PRINT("compress", + ("Succeeded for space " ULINTPF + " '%s' len " ULINTPF " out_len " ULINTPF, + space ? space->id : 0, + space ? space->name : "(import)", + len, write_size)); srv_stats.page_compression_saved.add((len - write_size)); srv_stats.pages_page_compressed.inc(); diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 643e8c731e5..b4d39351370 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1502,17 +1502,19 @@ buf_flush_update_zip_checksum( #endif /* !UNIV_HOTBACKUP */ -/********************************************************************//** -The hook that is called just before a page is written to disk. -The function encrypts the content of the page and returns a pointer -to a frame that will be written instead of the real frame. */ +/** Encryption and page_compression hook that is called just before +a page is written to disk. +@param[in,out] space tablespace +@param[in,out] bpage buffer page +@param[in] src_frame physical page frame that is being encrypted +@return page frame to be written to file +(may be src_frame or an encrypted/compressed copy of it) */ UNIV_INTERN byte* buf_page_encrypt_before_write( -/*==========================*/ - buf_page_t* page, /*!< in/out: buffer page to be flushed */ - byte* frame, /*!< in: src frame */ - ulint space_id); /*!< in: space id */ + fil_space_t* space, + buf_page_t* bpage, + byte* src_frame); /********************************************************************** The hook that is called after page is written to disk. diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 3d7ae49f8ae..ba9e5029c6d 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1135,14 +1135,18 @@ fil_flush( /*======*/ ulint space_id); /*!< in: file space id (this can be a group of log files or a tablespace of the database) */ -/**********************************************************************//** -Flushes to disk writes in file spaces of the given type possibly cached by -the OS. */ +/** Flush a tablespace. +@param[in,out] space tablespace to flush */ UNIV_INTERN void -fil_flush_file_spaces( -/*==================*/ - ulint purpose); /*!< in: FIL_TABLESPACE, FIL_LOG */ +fil_flush(fil_space_t* space); + +/** Flush to disk the writes in file spaces of the given type +possibly cached by the OS. +@param[in] purpose FIL_TYPE_TABLESPACE or FIL_TYPE_LOG */ +UNIV_INTERN +void +fil_flush_file_spaces(ulint purpose); /******************************************************************//** Checks the consistency of the tablespace cache. @return TRUE if ok */ @@ -1430,18 +1434,13 @@ fil_get_next_space_safe( /*====================*/ ulint id); /*!< in: previous space id */ - -/*******************************************************************//** -Returns the block size of the file space +/** Determine the block size of the data file. +@param[in] space tablespace +@param[in] offset page number @return block size */ UNIV_INTERN ulint -fil_space_get_block_size( -/*=====================*/ - ulint id, /*!< in: space id */ - ulint offset, /*!< in: page offset */ - ulint len); /*!< in: page len */ - +fil_space_get_block_size(const fil_space_t* space, unsigned offset); #endif /* UNIV_INNOCHECKSUM */ #ifndef UNIV_INNOCHECKSUM diff --git a/storage/innobase/include/fil0pagecompress.h b/storage/innobase/include/fil0pagecompress.h index 1fe5cb66bf6..73667c5420e 100644 --- a/storage/innobase/include/fil0pagecompress.h +++ b/storage/innobase/include/fil0pagecompress.h @@ -30,33 +30,6 @@ atomic writes information to table space. Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com ***********************************************************************/ -/*******************************************************************//** -Returns the page compression level flag of the space, or 0 if the space -is not compressed. The tablespace must be cached in the memory cache. -@return page compression level if page compressed, ULINT_UNDEFINED if space not found */ -UNIV_INLINE -ulint -fil_space_get_page_compression_level( -/*=================================*/ - ulint id); /*!< in: space id */ -/*******************************************************************//** -Returns the page compression flag of the space, or false if the space -is not compressed. The tablespace must be cached in the memory cache. -@return true if page compressed, false if not or space not found */ -UNIV_INLINE -bool -fil_space_is_page_compressed( -/*=========================*/ - ulint id); /*!< in: space id */ -/*******************************************************************//** -Returns the atomic writes flag of the space, or false if the space -is not using atomic writes. The tablespace must be cached in the memory cache. -@return atomic write table option value */ -UNIV_INLINE -atomic_writes_t -fil_space_get_atomic_writes( -/*=========================*/ - ulint id); /*!< in: space id */ /*******************************************************************//** Find out wheather the page is index page or not @return true if page type index page, false if not */ @@ -84,8 +57,7 @@ UNIV_INTERN byte* fil_compress_page( /*==============*/ - ulint space_id, /*!< in: tablespace id of the - table. */ + fil_space_t* space, /*!< in,out: tablespace (NULL during IMPORT) */ byte* buf, /*!< in: buffer from which to write; in aio this must be appropriately aligned */ byte* out_buf, /*!< out: compressed buffer */ diff --git a/storage/innobase/include/fsp0pagecompress.ic b/storage/innobase/include/fsp0pagecompress.ic index 48163277feb..14f968e319e 100644 --- a/storage/innobase/include/fsp0pagecompress.ic +++ b/storage/innobase/include/fsp0pagecompress.ic @@ -85,47 +85,6 @@ fil_page_is_compressed_encrypted( return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); } -#ifndef UNIV_INNOCHECKSUM -/*******************************************************************//** -Returns the page compression level of the space, or 0 if the space -is not compressed. The tablespace must be cached in the memory cache. -@return page compression level, 0 if space not found */ -UNIV_INLINE -ulint -fil_space_get_page_compression_level( -/*=================================*/ - ulint id) /*!< in: space id */ -{ - ulint flags; - - flags = fil_space_get_flags(id); - - if (flags && flags != ULINT_UNDEFINED) { - - return(fsp_flags_get_page_compression_level(flags)); - } - - return(0); -} - -/*******************************************************************//** -Extract the page compression from space. -@return true if space is page compressed, false if space is not found -or space is not page compressed. */ -UNIV_INLINE -bool -fil_space_is_page_compressed( -/*=========================*/ - ulint id) /*!< in: space id */ -{ - ulint flags = fil_space_get_flags(id); - - return(flags != ULINT_UNDEFINED - && FSP_FLAGS_HAS_PAGE_COMPRESSION(flags)); -} - -#endif /* UNIV_INNOCHECKSUM */ - /****************************************************************//** Get the name of the compression algorithm used for page compression. @@ -165,28 +124,6 @@ fil_get_compression_alg_name( } #ifndef UNIV_INNOCHECKSUM -/*******************************************************************//** -Returns the atomic writes flag of the space, or false if the space -is not using atomic writes. The tablespace must be cached in the memory cache. -@return atomic writes table option value */ -UNIV_INLINE -atomic_writes_t -fil_space_get_atomic_writes( -/*========================*/ - ulint id) /*!< in: space id */ -{ - ulint flags; - - flags = fil_space_get_flags(id); - - if (flags && flags != ULINT_UNDEFINED) { - - return((atomic_writes_t)fsp_flags_get_atomic_writes(flags)); - } - - return((atomic_writes_t)0); -} - /*******************************************************************//** Find out wheather the page is page compressed with lzo method @return true if page is page compressed with lzo method, false if not */ diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 7f77f96c2fe..394d2be6838 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -6241,70 +6241,54 @@ buf_pool_reserve_tmp_slot( return (free_slot); } -/********************************************************************//** -Encrypts a buffer page right before it's flushed to disk -@param[in,out] bpage Page control block -@param[in,out] src_frame Source page -@param[in] space_id Tablespace id -@return either unencrypted source page or decrypted page. -*/ +/** Encryption and page_compression hook that is called just before +a page is written to disk. +@param[in,out] space tablespace +@param[in,out] bpage buffer page +@param[in] src_frame physical page frame that is being encrypted +@return page frame to be written to file +(may be src_frame or an encrypted/compressed copy of it) */ +UNIV_INTERN byte* buf_page_encrypt_before_write( + fil_space_t* space, buf_page_t* bpage, - byte* src_frame, - ulint space_id) + byte* src_frame) { + ut_ad(space->id == bpage->space); bpage->real_size = UNIV_PAGE_SIZE; fil_page_type_validate(src_frame); - if (bpage->offset == 0) { + switch (bpage->offset) { + case 0: /* Page 0 of a tablespace is not encrypted/compressed */ ut_ad(bpage->key_version == 0); return src_frame; - } - - if (bpage->space == TRX_SYS_SPACE && bpage->offset == TRX_SYS_PAGE_NO) { - /* don't encrypt/compress page as it contains address to dblwr buffer */ - bpage->key_version = 0; - return src_frame; - } - - fil_space_t* space = fil_space_acquire_silent(space_id); - - /* Tablespace must exist during write operation */ - if (!space) { - /* This could be true on discard if we have injected a error - case e.g. in innodb.innodb-wl5522-debug-zip so that space - is already marked as stop_new_ops = true. */ - return src_frame; + case TRX_SYS_PAGE_NO: + if (bpage->space == TRX_SYS_SPACE) { + /* don't encrypt/compress page as it contains + address to dblwr buffer */ + bpage->key_version = 0; + return src_frame; + } } fil_space_crypt_t* crypt_data = space->crypt_data; - bool encrypted = true; - - if (space->crypt_data != NULL && space->crypt_data->not_encrypted()) { - /* Encryption is disabled */ - encrypted = false; - } + const bool encrypted = crypt_data + && !crypt_data->not_encrypted() + && crypt_data->type != CRYPT_SCHEME_UNENCRYPTED + && (!crypt_data->is_default_encryption() + || srv_encrypt_tables); - if (!srv_encrypt_tables && (crypt_data == NULL || crypt_data->is_default_encryption())) { - /* Encryption is disabled */ - encrypted = false; - } - - /* Is encryption needed? */ - if (crypt_data == NULL || crypt_data->type == CRYPT_SCHEME_UNENCRYPTED) { - /* An unencrypted table */ + if (!encrypted) { bpage->key_version = 0; - encrypted = false; } - bool page_compressed = fil_space_is_page_compressed(bpage->space); + bool page_compressed = FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags); if (!encrypted && !page_compressed) { /* No need to encrypt or page compress the page */ - fil_space_release(space); return src_frame; } @@ -6332,25 +6316,21 @@ buf_page_encrypt_before_write( bpage->real_size = page_size; slot->out_buf = dst_frame = tmp; -#ifdef UNIV_DEBUG - fil_page_type_validate(tmp); -#endif - + ut_d(fil_page_type_validate(tmp)); } else { /* First we compress the page content */ ulint out_len = 0; - ulint block_size = fil_space_get_block_size(bpage->space, bpage->offset, page_size); - - byte *tmp = fil_compress_page(bpage->space, - (byte *)src_frame, - slot->comp_buf, - page_size, - fil_space_get_page_compression_level(bpage->space), - block_size, - encrypted, - &out_len, - IF_LZO(slot->lzo_mem, NULL) - ); + + byte *tmp = fil_compress_page( + space, + (byte *)src_frame, + slot->comp_buf, + page_size, + fsp_flags_get_page_compression_level(space->flags), + fil_space_get_block_size(space, bpage->offset), + encrypted, + &out_len, + IF_LZO(slot->lzo_mem, NULL)); bpage->real_size = out_len; @@ -6375,7 +6355,6 @@ buf_page_encrypt_before_write( fil_page_type_validate(dst_frame); #endif - fil_space_release(space); // return dst_frame which will be written return dst_frame; } diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index ddef13ef8ea..55c5e4d543a 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -1001,7 +1001,7 @@ flush: srv_stats.dblwr_writes.inc(); /* Now flush the doublewrite buffer data to disk */ - fil_flush(TRX_SYS_SPACE); + fil_flush(ulint(TRX_SYS_SPACE)); /* We know that the writes have been flushed to disk now and in recovery we will find them in the doublewrite buffer @@ -1249,7 +1249,7 @@ retry: } /* Now flush the doublewrite buffer data to disk */ - fil_flush(TRX_SYS_SPACE); + fil_flush(ulint(TRX_SYS_SPACE)); /* We know that the write has been flushed to disk now and during recovery we will find it in the doublewrite buffer diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index 4804d71a51f..f71bdb9f69a 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -873,11 +873,12 @@ buf_flush_write_block_low( buf_flush_t flush_type, /*!< in: type of flush */ bool sync) /*!< in: true if sync IO request */ { + fil_space_t* space = fil_space_acquire(bpage->space, true); + if (!space) { + return; + } ulint zip_size = buf_page_get_zip_size(bpage); page_t* frame = NULL; - ulint space_id = buf_page_get_space(bpage); - atomic_writes_t awrites = fil_space_get_atomic_writes(space_id); - #ifdef UNIV_DEBUG buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); ut_ad(!mutex_own(&buf_pool->LRU_list_mutex)); @@ -947,7 +948,7 @@ buf_flush_write_block_low( break; } - frame = buf_page_encrypt_before_write(bpage, frame, space_id); + frame = buf_page_encrypt_before_write(space, bpage, frame); if (!srv_use_doublewrite_buf || !buf_dblwr) { fil_io(OS_FILE_WRITE | OS_AIO_SIMULATED_WAKE_LATER, @@ -968,7 +969,8 @@ buf_flush_write_block_low( atomic writes should be used, no doublewrite buffer is used. */ - if (awrites == ATOMIC_WRITES_ON) { + if (fsp_flags_get_atomic_writes(space->flags) + == ATOMIC_WRITES_ON) { fil_io(OS_FILE_WRITE | OS_AIO_SIMULATED_WAKE_LATER, FALSE, buf_page_get_space(bpage), @@ -991,10 +993,12 @@ buf_flush_write_block_low( are working on. */ if (sync) { ut_ad(flush_type == BUF_FLUSH_SINGLE_PAGE); - fil_flush(buf_page_get_space(bpage)); + fil_flush(space); buf_page_io_complete(bpage); } + fil_space_release(space); + /* Increment the counter of I/O operations used for selecting LRU policy. */ buf_LRU_stat_inc_io(); diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index fba7d5d8e38..e504ab3947e 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -5993,31 +5993,34 @@ fil_space_get_node( return (node); } -/********************************************************************//** -Return block size of node in file space -@return file block size */ + +/** Determine the block size of the data file. +@param[in] space tablespace +@param[in] offset page number +@return block size */ UNIV_INTERN ulint -fil_space_get_block_size( -/*=====================*/ - ulint space_id, - ulint block_offset, - ulint len) +fil_space_get_block_size(const fil_space_t* space, unsigned offset) { - ulint block_size = 512; - ut_ad(!mutex_own(&fil_system->mutex)); - - mutex_enter(&fil_system->mutex); - fil_space_t* space = fil_space_get_space(space_id); + ut_ad(space->n_pending_ops > 0); - if (space) { - fil_node_t* node = fil_space_get_node(space, space_id, &block_offset, 0, len); + ulint block_size = 512; - if (node) { - block_size = node->file_block_size; + for (fil_node_t* node = UT_LIST_GET_FIRST(space->chain); + node != NULL; + node = UT_LIST_GET_NEXT(chain, node)) { + block_size = node->file_block_size; + if (node->size > offset) { + break; } + offset -= node->size; + } + + /* Currently supporting block size up to 4K, + fall back to default if bigger requested. */ + if (block_size > 4096) { + block_size = 512; } - mutex_exit(&fil_system->mutex); return block_size; } @@ -6398,14 +6401,29 @@ fil_flush( mutex_exit(&fil_system->mutex); } -/**********************************************************************//** -Flushes to disk the writes in file spaces of the given type possibly cached by -the OS. */ +/** Flush a tablespace. +@param[in,out] space tablespace to flush */ UNIV_INTERN void -fil_flush_file_spaces( -/*==================*/ - ulint purpose) /*!< in: FIL_TABLESPACE, FIL_LOG */ +fil_flush(fil_space_t* space) +{ + ut_ad(space->n_pending_ops > 0); + + if (!space->is_stopping()) { + mutex_enter(&fil_system->mutex); + if (!space->is_stopping()) { + fil_flush_low(space); + } + mutex_exit(&fil_system->mutex); + } +} + +/** Flush to disk the writes in file spaces of the given type +possibly cached by the OS. +@param[in] purpose FIL_TYPE_TABLESPACE or FIL_TYPE_LOG */ +UNIV_INTERN +void +fil_flush_file_spaces(ulint purpose) { fil_space_t* space; ulint* space_ids; @@ -6833,12 +6851,14 @@ fil_iterate( if (page_compressed) { ulint len = 0; - fil_compress_page(space_id, + + fil_compress_page( + NULL, src, NULL, size, - fil_space_get_page_compression_level(space_id), - fil_space_get_block_size(space_id, offset, size), + 0,/* FIXME: compression level */ + 512,/* FIXME: use proper block size */ encrypted, &len, NULL); @@ -6849,6 +6869,8 @@ fil_iterate( /* If tablespace is encrypted, encrypt page before we write it back. Note that we should not encrypt the buffer that is in buffer pool. */ + /* NOTE: At this stage of IMPORT the + buffer pool is not being used at all! */ if (decrypted && encrypted) { byte *dest = writeptr + (i * size); ulint space = mach_read_from_4( diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc index 655460c57ac..73df69beb90 100644 --- a/storage/xtradb/fil/fil0pagecompress.cc +++ b/storage/xtradb/fil/fil0pagecompress.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -91,8 +91,7 @@ UNIV_INTERN byte* fil_compress_page( /*==============*/ - ulint space_id, /*!< in: tablespace id of the - table. */ + fil_space_t* space, /*!< in,out: tablespace (NULL during IMPORT) */ byte* buf, /*!< in: buffer from which to write; in aio this must be appropriately aligned */ byte* out_buf, /*!< out: compressed buffer */ @@ -110,9 +109,12 @@ fil_compress_page( ulint write_size=0; /* Cache to avoid change during function execution */ ulint comp_method = innodb_compression_algorithm; - ulint orig_page_type; bool allocated=false; + /* page_compression does not apply to tables or tablespaces + that use ROW_FORMAT=COMPRESSED */ + ut_ad(!space || !FSP_FLAGS_GET_ZIP_SSIZE(space->flags)); + if (encrypted) { header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE; } @@ -133,21 +135,14 @@ fil_compress_page( ut_ad(len); ut_ad(out_len); - /* read original page type */ - orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); - - fil_system_enter(); - fil_space_t* space = fil_space_get_by_id(space_id); - fil_system_exit(); - /* Let's not compress file space header or extent descriptor */ - if (orig_page_type == 0 || - orig_page_type == FIL_PAGE_TYPE_FSP_HDR || - orig_page_type == FIL_PAGE_TYPE_XDES || - orig_page_type == FIL_PAGE_PAGE_COMPRESSED) { + switch (fil_page_get_type(buf)) { + case 0: + case FIL_PAGE_TYPE_FSP_HDR: + case FIL_PAGE_TYPE_XDES: + case FIL_PAGE_PAGE_COMPRESSED: *out_len = len; - goto err_exit; } @@ -157,11 +152,11 @@ fil_compress_page( comp_level = page_zip_level; } -#ifdef UNIV_PAGECOMPRESS_DEBUG - ib_logf(IB_LOG_LEVEL_INFO, - "Preparing for compress for space %lu name %s len %lu.", - space_id, fil_space_name(space), len); -#endif /* UNIV_PAGECOMPRESS_DEBUG */ + DBUG_PRINT("compress", + ("Preparing for space " ULINTPF " '%s' len " ULINTPF, + space ? space->id : 0, + space ? space->name : "(import)", + len)); write_size = UNIV_PAGE_SIZE - header_len; @@ -306,10 +301,13 @@ fil_compress_page( if (err != Z_OK) { /* If error we leave the actual page as it was */ - if (space->printed_compression_failure == false) { + if (space && !space->printed_compression_failure) { ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space %lu name %s len %lu rt %d write %lu.", - space_id, fil_space_name(space), len, err, write_size); + "Compression failed for space " + ULINTPF " name %s len " ULINTPF + " rt %d write " ULINTPF ".", + space->id, space->name, len, err, + write_size); space->printed_compression_failure = true; } @@ -367,9 +365,8 @@ fil_compress_page( fil_decompress_page(uncomp_page, comp_page, ulong(len), NULL); - if(buf_page_is_corrupted(false, uncomp_page, 0, space)) { - buf_page_print(uncomp_page, 0, BUF_PAGE_PRINT_NO_CRASH); - ut_error; + if (buf_page_is_corrupted(false, uncomp_page, 0, space)) { + buf_page_print(uncomp_page, 0, 0); } ut_free(comp_page); @@ -397,11 +394,12 @@ fil_compress_page( #endif } -#ifdef UNIV_PAGECOMPRESS_DEBUG - ib_logf(IB_LOG_LEVEL_INFO, - "Compression succeeded for space %lu name %s len %lu out_len %lu.", - space_id, fil_space_name(space), len, write_size); -#endif /* UNIV_PAGECOMPRESS_DEBUG */ + DBUG_PRINT("compress", + ("Succeeded for space " ULINTPF + " '%s' len " ULINTPF " out_len " ULINTPF, + space ? space->id : 0, + space ? space->name : "(import)", + len, write_size)); srv_stats.page_compression_saved.add((len - write_size)); srv_stats.pages_page_compressed.inc(); diff --git a/storage/xtradb/include/buf0buf.h b/storage/xtradb/include/buf0buf.h index 1774d9445ff..4a632e2345f 100644 --- a/storage/xtradb/include/buf0buf.h +++ b/storage/xtradb/include/buf0buf.h @@ -1534,17 +1534,19 @@ buf_own_zip_mutex_for_page( MY_ATTRIBUTE((nonnull,warn_unused_result)); #endif /* UNIV_DEBUG */ -/********************************************************************//** -The hook that is called just before a page is written to disk. -The function encrypts the content of the page and returns a pointer -to a frame that will be written instead of the real frame. */ +/** Encryption and page_compression hook that is called just before +a page is written to disk. +@param[in,out] space tablespace +@param[in,out] bpage buffer page +@param[in] src_frame physical page frame that is being encrypted +@return page frame to be written to file +(may be src_frame or an encrypted/compressed copy of it) */ UNIV_INTERN byte* buf_page_encrypt_before_write( -/*==========================*/ - buf_page_t* page, /*!< in/out: buffer page to be flushed */ - byte* frame, /*!< in: src frame */ - ulint space_id); /*!< in: space id */ + fil_space_t* space, + buf_page_t* bpage, + byte* src_frame); /********************************************************************** The hook that is called after page is written to disk. diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 698039afede..fe7cc60d1e4 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -1120,16 +1120,13 @@ _fil_io( #define fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size) \ _fil_io(type, sync, space_id, zip_size, block_offset, byte_offset, len, buf, message, write_size, NULL) -/*******************************************************************//** -Returns the block size of the file space +/** Determine the block size of the data file. +@param[in] space tablespace +@param[in] offset page number @return block size */ UNIV_INTERN ulint -fil_space_get_block_size( -/*=====================*/ - ulint id, /*!< in: space id */ - ulint offset, /*!< in: page offset */ - ulint len); /*!< in: page len */ +fil_space_get_block_size(const fil_space_t* space, unsigned offset); /**********************************************************************//** Waits for an aio operation to complete. This function is used to write the @@ -1151,14 +1148,18 @@ fil_flush( /*======*/ ulint space_id); /*!< in: file space id (this can be a group of log files or a tablespace of the database) */ -/**********************************************************************//** -Flushes to disk writes in file spaces of the given type possibly cached by -the OS. */ +/** Flush a tablespace. +@param[in,out] space tablespace to flush */ UNIV_INTERN void -fil_flush_file_spaces( -/*==================*/ - ulint purpose); /*!< in: FIL_TABLESPACE, FIL_LOG */ +fil_flush(fil_space_t* space); + +/** Flush to disk the writes in file spaces of the given type +possibly cached by the OS. +@param[in] purpose FIL_TYPE_TABLESPACE or FIL_TYPE_LOG */ +UNIV_INTERN +void +fil_flush_file_spaces(ulint purpose); /******************************************************************//** Checks the consistency of the tablespace cache. @return TRUE if ok */ diff --git a/storage/xtradb/include/fil0pagecompress.h b/storage/xtradb/include/fil0pagecompress.h index 1fe5cb66bf6..73667c5420e 100644 --- a/storage/xtradb/include/fil0pagecompress.h +++ b/storage/xtradb/include/fil0pagecompress.h @@ -30,33 +30,6 @@ atomic writes information to table space. Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com ***********************************************************************/ -/*******************************************************************//** -Returns the page compression level flag of the space, or 0 if the space -is not compressed. The tablespace must be cached in the memory cache. -@return page compression level if page compressed, ULINT_UNDEFINED if space not found */ -UNIV_INLINE -ulint -fil_space_get_page_compression_level( -/*=================================*/ - ulint id); /*!< in: space id */ -/*******************************************************************//** -Returns the page compression flag of the space, or false if the space -is not compressed. The tablespace must be cached in the memory cache. -@return true if page compressed, false if not or space not found */ -UNIV_INLINE -bool -fil_space_is_page_compressed( -/*=========================*/ - ulint id); /*!< in: space id */ -/*******************************************************************//** -Returns the atomic writes flag of the space, or false if the space -is not using atomic writes. The tablespace must be cached in the memory cache. -@return atomic write table option value */ -UNIV_INLINE -atomic_writes_t -fil_space_get_atomic_writes( -/*=========================*/ - ulint id); /*!< in: space id */ /*******************************************************************//** Find out wheather the page is index page or not @return true if page type index page, false if not */ @@ -84,8 +57,7 @@ UNIV_INTERN byte* fil_compress_page( /*==============*/ - ulint space_id, /*!< in: tablespace id of the - table. */ + fil_space_t* space, /*!< in,out: tablespace (NULL during IMPORT) */ byte* buf, /*!< in: buffer from which to write; in aio this must be appropriately aligned */ byte* out_buf, /*!< out: compressed buffer */ diff --git a/storage/xtradb/include/fsp0pagecompress.ic b/storage/xtradb/include/fsp0pagecompress.ic index 48163277feb..14f968e319e 100644 --- a/storage/xtradb/include/fsp0pagecompress.ic +++ b/storage/xtradb/include/fsp0pagecompress.ic @@ -85,47 +85,6 @@ fil_page_is_compressed_encrypted( return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); } -#ifndef UNIV_INNOCHECKSUM -/*******************************************************************//** -Returns the page compression level of the space, or 0 if the space -is not compressed. The tablespace must be cached in the memory cache. -@return page compression level, 0 if space not found */ -UNIV_INLINE -ulint -fil_space_get_page_compression_level( -/*=================================*/ - ulint id) /*!< in: space id */ -{ - ulint flags; - - flags = fil_space_get_flags(id); - - if (flags && flags != ULINT_UNDEFINED) { - - return(fsp_flags_get_page_compression_level(flags)); - } - - return(0); -} - -/*******************************************************************//** -Extract the page compression from space. -@return true if space is page compressed, false if space is not found -or space is not page compressed. */ -UNIV_INLINE -bool -fil_space_is_page_compressed( -/*=========================*/ - ulint id) /*!< in: space id */ -{ - ulint flags = fil_space_get_flags(id); - - return(flags != ULINT_UNDEFINED - && FSP_FLAGS_HAS_PAGE_COMPRESSION(flags)); -} - -#endif /* UNIV_INNOCHECKSUM */ - /****************************************************************//** Get the name of the compression algorithm used for page compression. @@ -165,28 +124,6 @@ fil_get_compression_alg_name( } #ifndef UNIV_INNOCHECKSUM -/*******************************************************************//** -Returns the atomic writes flag of the space, or false if the space -is not using atomic writes. The tablespace must be cached in the memory cache. -@return atomic writes table option value */ -UNIV_INLINE -atomic_writes_t -fil_space_get_atomic_writes( -/*========================*/ - ulint id) /*!< in: space id */ -{ - ulint flags; - - flags = fil_space_get_flags(id); - - if (flags && flags != ULINT_UNDEFINED) { - - return((atomic_writes_t)fsp_flags_get_atomic_writes(flags)); - } - - return((atomic_writes_t)0); -} - /*******************************************************************//** Find out wheather the page is page compressed with lzo method @return true if page is page compressed with lzo method, false if not */ -- cgit v1.2.1 From 200ef5134483c05442b76b7fdda01297d43a6985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 21 Apr 2017 18:29:50 +0300 Subject: Fix a compilation error --- storage/innobase/fil/fil0pagecompress.cc | 67 +++++++++++++++++++------------- storage/innobase/include/fil0fil.h | 10 +---- storage/innobase/include/fil0fil.ic | 22 ----------- storage/xtradb/fil/fil0pagecompress.cc | 67 +++++++++++++++++++------------- storage/xtradb/include/fil0fil.h | 10 +---- storage/xtradb/include/fil0fil.ic | 22 ----------- 6 files changed, 84 insertions(+), 114 deletions(-) (limited to 'storage') diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index 73df69beb90..8b2449983df 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -171,12 +171,15 @@ fil_compress_page( /* If error we leave the actual page as it was */ #ifndef UNIV_PAGECOMPRESS_DEBUG - if (space->printed_compression_failure == false) { + if (space && !space->printed_compression_failure) { + space->printed_compression_failure = true; #endif ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space %lu name %s len %lu rt %d write %lu.", - space_id, fil_space_name(space), len, err, write_size); - space->printed_compression_failure = true; + "Compression failed for space " ULINTPF + " name %s len " ULINTPF + " err %d write_size " ULINTPF ".", + space->id, space->name, len, + err, write_size); #ifndef UNIV_PAGECOMPRESS_DEBUG } #endif @@ -192,11 +195,14 @@ fil_compress_page( buf, len, out_buf+header_len, &write_size, lzo_mem); if (err != LZO_E_OK || write_size > UNIV_PAGE_SIZE-header_len) { - if (space->printed_compression_failure == false) { - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space %lu name %s len %lu err %d write_size %lu.", - space_id, fil_space_name(space), len, err, write_size); + if (space && !space->printed_compression_failure) { space->printed_compression_failure = true; + ib_logf(IB_LOG_LEVEL_WARN, + "Compression failed for space " ULINTPF + " name %s len " ULINTPF + " err %d write_size " ULINTPF ".", + space->id, space->name, len, + err, write_size); } srv_stats.pages_page_compression_error.inc(); @@ -221,11 +227,14 @@ fil_compress_page( (size_t)write_size); if (err != LZMA_OK || out_pos > UNIV_PAGE_SIZE-header_len) { - if (space->printed_compression_failure == false) { - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space %lu name %s len %lu err %d write_size %lu", - space_id, fil_space_name(space), len, err, out_pos); + if (space && !space->printed_compression_failure) { space->printed_compression_failure = true; + ib_logf(IB_LOG_LEVEL_WARN, + "Compression failed for space " ULINTPF + " name %s len " ULINTPF + " err %d write_size " ULINTPF ".", + space->id, space->name, len, + err, out_pos); } srv_stats.pages_page_compression_error.inc(); @@ -252,11 +261,14 @@ fil_compress_page( 0); if (err != BZ_OK || write_size > UNIV_PAGE_SIZE-header_len) { - if (space->printed_compression_failure == false) { - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space %lu name %s len %lu err %d write_size %lu.", - space_id, fil_space_name(space), len, err, write_size); + if (space && !space->printed_compression_failure) { space->printed_compression_failure = true; + ib_logf(IB_LOG_LEVEL_WARN, + "Compression failed for space " ULINTPF + " name %s len " ULINTPF + " err %d write_size " ULINTPF ".", + space->id, space->name, len, + err, write_size); } srv_stats.pages_page_compression_error.inc(); @@ -279,11 +291,14 @@ fil_compress_page( (size_t*)&write_size); if (cstatus != SNAPPY_OK || write_size > UNIV_PAGE_SIZE-header_len) { - if (space->printed_compression_failure == false) { - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space %lu name %s len %lu err %d write_size %lu.", - space_id, fil_space_name(space), len, (int)cstatus, write_size); + if (space && !space->printed_compression_failure) { space->printed_compression_failure = true; + ib_logf(IB_LOG_LEVEL_WARN, + "Compression failed for space " ULINTPF + " name %s len " ULINTPF + " err %d write_size " ULINTPF ".", + space->id, space->name, len, + (int)cstatus, write_size); } srv_stats.pages_page_compression_error.inc(); @@ -302,13 +317,13 @@ fil_compress_page( /* If error we leave the actual page as it was */ if (space && !space->printed_compression_failure) { - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " - ULINTPF " name %s len " ULINTPF - " rt %d write " ULINTPF ".", - space->id, space->name, len, err, - write_size); space->printed_compression_failure = true; + ib_logf(IB_LOG_LEVEL_WARN, + "Compression failed for space " ULINTPF + " name %s len " ULINTPF + " rt %d write_size " ULINTPF ".", + space->id, space->name, len, + err, write_size); } srv_stats.pages_page_compression_error.inc(); diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index ba9e5029c6d..69594b4ad1d 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1366,14 +1366,6 @@ fil_user_tablespace_restore_page( ulint page_no); /* in: page_no to obtain from double write buffer */ -/*******************************************************************//** -Return space flags */ -UNIV_INLINE -ulint -fil_space_flags( -/*===========*/ - fil_space_t* space); /*!< in: space */ - /*******************************************************************//** Returns a pointer to the file_space_t that is in the memory cache associated with a space id. diff --git a/storage/innobase/include/fil0fil.ic b/storage/innobase/include/fil0fil.ic index b79167baead..6c2504c9f8c 100644 --- a/storage/innobase/include/fil0fil.ic +++ b/storage/innobase/include/fil0fil.ic @@ -26,28 +26,6 @@ Created 31/03/2015 Jan Lindström #ifndef fil0fil_ic #define fil0fil_ic -/*******************************************************************//** -Return space name */ -UNIV_INLINE -char* -fil_space_name( -/*===========*/ - fil_space_t* space) /*!< in: space */ -{ - return (space->name); -} - -/*******************************************************************//** -Return space flags */ -UNIV_INLINE -ulint -fil_space_flags( -/*===========*/ - fil_space_t* space) /*!< in: space */ -{ - return (space->flags); -} - /*******************************************************************//** Return page type name */ UNIV_INLINE diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc index 73df69beb90..8b2449983df 100644 --- a/storage/xtradb/fil/fil0pagecompress.cc +++ b/storage/xtradb/fil/fil0pagecompress.cc @@ -171,12 +171,15 @@ fil_compress_page( /* If error we leave the actual page as it was */ #ifndef UNIV_PAGECOMPRESS_DEBUG - if (space->printed_compression_failure == false) { + if (space && !space->printed_compression_failure) { + space->printed_compression_failure = true; #endif ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space %lu name %s len %lu rt %d write %lu.", - space_id, fil_space_name(space), len, err, write_size); - space->printed_compression_failure = true; + "Compression failed for space " ULINTPF + " name %s len " ULINTPF + " err %d write_size " ULINTPF ".", + space->id, space->name, len, + err, write_size); #ifndef UNIV_PAGECOMPRESS_DEBUG } #endif @@ -192,11 +195,14 @@ fil_compress_page( buf, len, out_buf+header_len, &write_size, lzo_mem); if (err != LZO_E_OK || write_size > UNIV_PAGE_SIZE-header_len) { - if (space->printed_compression_failure == false) { - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space %lu name %s len %lu err %d write_size %lu.", - space_id, fil_space_name(space), len, err, write_size); + if (space && !space->printed_compression_failure) { space->printed_compression_failure = true; + ib_logf(IB_LOG_LEVEL_WARN, + "Compression failed for space " ULINTPF + " name %s len " ULINTPF + " err %d write_size " ULINTPF ".", + space->id, space->name, len, + err, write_size); } srv_stats.pages_page_compression_error.inc(); @@ -221,11 +227,14 @@ fil_compress_page( (size_t)write_size); if (err != LZMA_OK || out_pos > UNIV_PAGE_SIZE-header_len) { - if (space->printed_compression_failure == false) { - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space %lu name %s len %lu err %d write_size %lu", - space_id, fil_space_name(space), len, err, out_pos); + if (space && !space->printed_compression_failure) { space->printed_compression_failure = true; + ib_logf(IB_LOG_LEVEL_WARN, + "Compression failed for space " ULINTPF + " name %s len " ULINTPF + " err %d write_size " ULINTPF ".", + space->id, space->name, len, + err, out_pos); } srv_stats.pages_page_compression_error.inc(); @@ -252,11 +261,14 @@ fil_compress_page( 0); if (err != BZ_OK || write_size > UNIV_PAGE_SIZE-header_len) { - if (space->printed_compression_failure == false) { - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space %lu name %s len %lu err %d write_size %lu.", - space_id, fil_space_name(space), len, err, write_size); + if (space && !space->printed_compression_failure) { space->printed_compression_failure = true; + ib_logf(IB_LOG_LEVEL_WARN, + "Compression failed for space " ULINTPF + " name %s len " ULINTPF + " err %d write_size " ULINTPF ".", + space->id, space->name, len, + err, write_size); } srv_stats.pages_page_compression_error.inc(); @@ -279,11 +291,14 @@ fil_compress_page( (size_t*)&write_size); if (cstatus != SNAPPY_OK || write_size > UNIV_PAGE_SIZE-header_len) { - if (space->printed_compression_failure == false) { - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space %lu name %s len %lu err %d write_size %lu.", - space_id, fil_space_name(space), len, (int)cstatus, write_size); + if (space && !space->printed_compression_failure) { space->printed_compression_failure = true; + ib_logf(IB_LOG_LEVEL_WARN, + "Compression failed for space " ULINTPF + " name %s len " ULINTPF + " err %d write_size " ULINTPF ".", + space->id, space->name, len, + (int)cstatus, write_size); } srv_stats.pages_page_compression_error.inc(); @@ -302,13 +317,13 @@ fil_compress_page( /* If error we leave the actual page as it was */ if (space && !space->printed_compression_failure) { - ib_logf(IB_LOG_LEVEL_WARN, - "Compression failed for space " - ULINTPF " name %s len " ULINTPF - " rt %d write " ULINTPF ".", - space->id, space->name, len, err, - write_size); space->printed_compression_failure = true; + ib_logf(IB_LOG_LEVEL_WARN, + "Compression failed for space " ULINTPF + " name %s len " ULINTPF + " rt %d write_size " ULINTPF ".", + space->id, space->name, len, + err, write_size); } srv_stats.pages_page_compression_error.inc(); diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index fe7cc60d1e4..d73a68d9d34 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1460,14 +1460,6 @@ fil_get_next_space_safe( #endif /* UNIV_INNOCHECKSUM */ -/*******************************************************************//** -Return space flags */ -UNIV_INLINE -ulint -fil_space_flags( -/*===========*/ - fil_space_t* space); /*!< in: space */ - /****************************************************************//** Does error handling when a file operation fails. @return TRUE if we should retry the operation */ diff --git a/storage/xtradb/include/fil0fil.ic b/storage/xtradb/include/fil0fil.ic index b79167baead..6c2504c9f8c 100644 --- a/storage/xtradb/include/fil0fil.ic +++ b/storage/xtradb/include/fil0fil.ic @@ -26,28 +26,6 @@ Created 31/03/2015 Jan Lindström #ifndef fil0fil_ic #define fil0fil_ic -/*******************************************************************//** -Return space name */ -UNIV_INLINE -char* -fil_space_name( -/*===========*/ - fil_space_t* space) /*!< in: space */ -{ - return (space->name); -} - -/*******************************************************************//** -Return space flags */ -UNIV_INLINE -ulint -fil_space_flags( -/*===========*/ - fil_space_t* space) /*!< in: space */ -{ - return (space->flags); -} - /*******************************************************************//** Return page type name */ UNIV_INLINE -- cgit v1.2.1 From 7d0ac3ade7fb9f1276e87faa7099832d01329200 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 24 Apr 2017 13:02:08 +1000 Subject: MDEV-12488: Remove type mismatch in InnoDB printf-like calls To fix OSX error: /Users/travis/build/grooverdan/mariadb-server/storage/xtradb/sync/sync0arr.cc:530:5: warning: format specifies type 'unsigned long' but the argument has type 'os_thread_id_t' (aka '_opaque_pthread_t *') [-Wformat] mutex->thread_id, ^~~~~~~~~~~~~~~~ Signed-off-by: Daniel Black --- storage/xtradb/sync/sync0arr.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'storage') diff --git a/storage/xtradb/sync/sync0arr.cc b/storage/xtradb/sync/sync0arr.cc index ae8032da167..134d16ae58e 100644 --- a/storage/xtradb/sync/sync0arr.cc +++ b/storage/xtradb/sync/sync0arr.cc @@ -527,7 +527,7 @@ sync_array_cell_print( "waiters flag " ULINTPF "\n", (void*) mutex, mutex->cmutex_name, (ulong) mutex->lock_word, - mutex->thread_id, + os_thread_pf(mutex->thread_id), mutex->file_name, mutex->line, mutex->waiters); } -- cgit v1.2.1 From 4b5a9d8e0fdb8a98a3f5509ed9704650523b3b10 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Fri, 15 Jul 2016 14:39:37 +0530 Subject: Bug #23475211 COMBINING ALTER OPERATIONS TRIGGERS TABLE REBUILD Problem: ======= Inplace alter algorithm determines the table to be rebuild if the table undergoes row format change, key block size if handler flag contains only change table create option. If alter with inplace ignore flag operations and change table create options then it leads to table rebuild operation. Solution: ======== During the check for rebuild, ignore the inplace ignore flag and check for table create options. Reviewed-by: Jimmy Yang Reviewed-by: Marko Makela RB: 13172 --- storage/innobase/handler/handler0alter.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'storage') diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index a5b919b424d..3ede403b5b3 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -400,7 +400,10 @@ innobase_need_rebuild( const Alter_inplace_info* ha_alter_info, const TABLE* altered_table) { - if (ha_alter_info->handler_flags + Alter_inplace_info::HA_ALTER_FLAGS alter_inplace_flags = + ha_alter_info->handler_flags & ~(INNOBASE_INPLACE_IGNORE); + + if (alter_inplace_flags == Alter_inplace_info::CHANGE_CREATE_OPTION && !(ha_alter_info->create_info->used_fields & (HA_CREATE_USED_ROW_FORMAT @@ -6082,7 +6085,7 @@ err_exit: } if (!(ha_alter_info->handler_flags & INNOBASE_ALTER_DATA) - || (ha_alter_info->handler_flags + || ((ha_alter_info->handler_flags & ~INNOBASE_INPLACE_IGNORE) == Alter_inplace_info::CHANGE_CREATE_OPTION && !innobase_need_rebuild(ha_alter_info, table))) { @@ -6346,7 +6349,7 @@ ok_exit: DBUG_RETURN(false); } - if (ha_alter_info->handler_flags + if ((ha_alter_info->handler_flags & ~INNOBASE_INPLACE_IGNORE) == Alter_inplace_info::CHANGE_CREATE_OPTION && !innobase_need_rebuild(ha_alter_info, table)) { goto ok_exit; -- cgit v1.2.1 From 9ce1ea6f51e05ecd0cc03b7224be23ebfa1bb4d2 Mon Sep 17 00:00:00 2001 From: Shaohua Wang Date: Wed, 27 Jul 2016 03:43:52 +0200 Subject: BUG#24009272 SEGFAULT WITH CREATE+SELECT FROM IS+DROP FTS TABLE CONCURRENTLY Analysis: When we access fts_internal_tbl_name in i_s_fts_config_fill (), it can be set to NULL by another session. Solution: Define fts_internal_tbl_name2 for global variable innodb_ft_aux_table, if it's NULL, set fts_internal_tbl_name to "default". Reviewed-by: Jimmy Yang RB: 13401 --- storage/innobase/fts/fts0fts.cc | 1 + storage/innobase/handler/ha_innodb.cc | 41 ++++++++++++++++++++++++++++++++--- storage/innobase/handler/i_s.cc | 2 ++ storage/innobase/include/fts0fts.h | 1 + 4 files changed, 42 insertions(+), 3 deletions(-) (limited to 'storage') diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index dd0e036170c..67a71a80417 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -93,6 +93,7 @@ static const ulint FTS_DEADLOCK_RETRY_WAIT = 100000; /** variable to record innodb_fts_internal_tbl_name for information schema table INNODB_FTS_INSERTED etc. */ char* fts_internal_tbl_name = NULL; +char* fts_internal_tbl_name2 = NULL; /** InnoDB default stopword list: There are different versions of stopwords, the stop words listed diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 3e00e5e2ad2..44c4beb2445 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -18773,6 +18773,41 @@ innodb_internal_table_validate( } #ifdef BTR_CUR_HASH_ADAPT +/****************************************************************//** +Update global variable "fts_internal_tbl_name" with the "saved" +stopword table name value. This function is registered as a callback +with MySQL. */ +static +void +innodb_internal_table_update( +/*=========================*/ + THD* thd, /*!< in: thread handle */ + struct st_mysql_sys_var* var, /*!< in: pointer to + system variable */ + void* var_ptr,/*!< out: where the + formal string goes */ + const void* save) /*!< in: immediate result + from check function */ +{ + const char* table_name; + char* old; + + ut_a(save != NULL); + ut_a(var_ptr != NULL); + + table_name = *static_cast(save); + old = *(char**) var_ptr; + *(char**) var_ptr = table_name ? my_strdup(table_name, MYF(0)) : NULL; + my_free(old); + + fts_internal_tbl_name2 = *(char**) var_ptr; + if (fts_internal_tbl_name2 == NULL) { + fts_internal_tbl_name = const_cast("default"); + } else { + fts_internal_tbl_name = fts_internal_tbl_name2; + } +} + /****************************************************************//** Update the system variable innodb_adaptive_hash_index using the "saved" value. This function is registered as a callback with MySQL. */ @@ -21226,11 +21261,11 @@ static MYSQL_SYSVAR_BOOL(disable_sort_file_cache, srv_disable_sort_file_cache, "Whether to disable OS system file cache for sort I/O", NULL, NULL, FALSE); -static MYSQL_SYSVAR_STR(ft_aux_table, fts_internal_tbl_name, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC, +static MYSQL_SYSVAR_STR(ft_aux_table, fts_internal_tbl_name2, + PLUGIN_VAR_RQCMDARG, "FTS internal auxiliary table to be checked", innodb_internal_table_validate, - NULL, NULL); + innodb_internal_table_update, NULL); static MYSQL_SYSVAR_ULONG(ft_cache_size, fts_max_cache_size, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 6bf16573efd..65197e402aa 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -3955,6 +3955,8 @@ i_s_fts_config_fill( DBUG_RETURN(0); } + DEBUG_SYNC_C("i_s_fts_config_fille_check"); + fields = table->field; /* Prevent DDL to drop fts aux tables. */ diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 21354b1b605..966f28bde3f 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -409,6 +409,7 @@ extern bool fts_need_sync; /** Variable specifying the table that has Fulltext index to display its content through information schema table */ extern char* fts_internal_tbl_name; +extern char* fts_internal_tbl_name2; #define fts_que_graph_free(graph) \ do { \ -- cgit v1.2.1 From b862c7972b9784d1522b444e83cb2349e848c1bb Mon Sep 17 00:00:00 2001 From: Shaohua Wang Date: Wed, 27 Jul 2016 09:37:20 +0200 Subject: BUG#24315031 FAILING ASSERTION: !TABLE->CAN_BE_EVICTED Analysis: the old table is dropped, just after it's added into drop list, and new table with the same name is created, then we try to drop the new table in background. Solution: Don't drop a table in background if table->to_be_dropped is false. Reviewed-by: Jimmy Yang RB: 13414 --- storage/innobase/row/row0mysql.cc | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'storage') diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index d285352dadb..da3dd42fddc 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -2888,6 +2888,10 @@ loop: return(n_tables + n_tables_dropped); } + DBUG_EXECUTE_IF("row_drop_tables_in_background_sleep", + os_thread_sleep(5000000); + ); + table = dict_table_open_on_name(drop->table_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); @@ -2898,6 +2902,16 @@ loop: goto already_dropped; } + if (!table->to_be_dropped) { + /* There is a scenario: the old table is dropped + just after it's added into drop list, and new + table with the same name is created, then we try + to drop the new table in background. */ + dict_table_close(table, FALSE, FALSE); + + goto already_dropped; + } + ut_a(!table->can_be_evicted); dict_table_close(table, FALSE, FALSE); @@ -3766,6 +3780,13 @@ row_drop_table_for_mysql( } } + + DBUG_EXECUTE_IF("row_drop_table_add_to_background", + row_add_table_to_background_drop_list(table->name.m_name); + err = DB_SUCCESS; + goto funct_exit; + ); + /* TODO: could we replace the counter n_foreign_key_checks_running with lock checks on the table? Acquire here an exclusive lock on the table, and rewrite lock0lock.cc and the lock wait in srv0srv.cc so that -- cgit v1.2.1 From 1a5ca702b1301b12bf97636bd8d168120a188681 Mon Sep 17 00:00:00 2001 From: Shaohua Wang Date: Wed, 27 Jul 2016 10:39:19 +0200 Subject: Followup: BUG#23479595 SEGMENTATION FAULT WHEN SELECT FTS INDEX TABLES IN INFORMATION SCHEMA BUG#23742339 FAILING ASSERTION: SYM_NODE->TABLE != NULL Analysis: When we access fts aux tables in information schema,the fts aux tables are dropped by DROP DATABASE in another session. Solution: Drop parent table if it's a fts aux table, and drop table will drop fts aux tables together. Reviewed-by: Jimmy Yang RB: 13264 --- storage/innobase/fts/fts0fts.cc | 30 ++++++++++++++++++++++++++++++ storage/innobase/include/fts0fts.h | 9 +++++++++ storage/innobase/row/row0mysql.cc | 13 +++++++++++++ 3 files changed, 52 insertions(+) (limited to 'storage') diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 67a71a80417..79020fb4442 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -6497,6 +6497,36 @@ fts_check_corrupt_index( return(0); } +/* Get parent table name if it's a fts aux table +@param[in] aux_table_name aux table name +@param[in] aux_table_len aux table length +@return parent table name, or NULL */ +char* +fts_get_parent_table_name( + const char* aux_table_name, + ulint aux_table_len) +{ + fts_aux_table_t aux_table; + char* parent_table_name = NULL; + + if (fts_is_aux_table_name(&aux_table, aux_table_name, aux_table_len)) { + dict_table_t* parent_table; + + parent_table = dict_table_open_on_id( + aux_table.parent_id, TRUE, DICT_TABLE_OP_NORMAL); + + if (parent_table != NULL) { + parent_table_name = mem_strdupl( + parent_table->name.m_name, + strlen(parent_table->name.m_name)); + + dict_table_close(parent_table, TRUE, FALSE); + } + } + + return(parent_table_name); +} + /** Check the validity of the parent table. @param[in] aux_table auxiliary table @return true if it is a valid table or false if it is not */ diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 966f28bde3f..f1d53165cdd 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -843,6 +843,15 @@ fts_init_doc_id( /*============*/ const dict_table_t* table); /*!< in: table */ +/* Get parent table name if it's a fts aux table +@param[in] aux_table_name aux table name +@param[in] aux_table_len aux table length +@return parent table name, or NULL */ +char* +fts_get_parent_table_name( + const char* aux_table_name, + ulint aux_table_len); + /******************************************************************//** compare two character string according to their charset. */ extern diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index da3dd42fddc..b92794138cc 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -4293,6 +4293,19 @@ loop: row_mysql_lock_data_dictionary(trx); while ((table_name = dict_get_first_table_name_in_db(name))) { + /* Drop parent table if it is a fts aux table, to + avoid accessing dropped fts aux tables in information + scheam when parent table still exists. + Note: Drop parent table will drop fts aux tables. */ + char* parent_table_name; + parent_table_name = fts_get_parent_table_name( + table_name, strlen(table_name)); + + if (parent_table_name != NULL) { + ut_free(table_name); + table_name = parent_table_name; + } + ut_a(memcmp(table_name, name, namelen) == 0); table = dict_table_open_on_name( -- cgit v1.2.1 From d3a2f60e1a80a70f38ac8e86f297169bad812415 Mon Sep 17 00:00:00 2001 From: Shaohua Wang Date: Thu, 28 Jul 2016 13:08:52 +0800 Subject: BUG#23477773 OPTION TO TURN OFF/ON DEADLOCK CHECKER Backport WL#9383 INNODB: ADD AN OPTION TO TURN OFF/ON DEADLOCK CHECKER (rb#12873) to 5.7. --- storage/innobase/handler/ha_innodb.cc | 8 ++++ storage/innobase/include/lock0lock.h | 5 ++- storage/innobase/lock/lock0lock.cc | 77 ++++++++++++++++------------------- 3 files changed, 46 insertions(+), 44 deletions(-) (limited to 'storage') diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 44c4beb2445..5d8ff4ff493 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -21246,6 +21246,13 @@ static MYSQL_SYSVAR_ULONG(concurrency_tickets, srv_n_free_tickets_to_enter, "Number of times a thread is allowed to enter InnoDB within the same SQL query after it has once got the ticket", NULL, NULL, 5000L, 1L, ~0UL, 0); +static MYSQL_SYSVAR_BOOL(deadlock_detect, innobase_deadlock_detect, + PLUGIN_VAR_NOCMDARG, + "Enable/disable InnoDB deadlock detector (default ON)." + " if set to OFF, deadlock detection is skipped," + " and we rely on innodb_lock_wait_timeout in case of deadlock.", + NULL, NULL, TRUE); + static MYSQL_SYSVAR_LONG(fill_factor, innobase_fill_factor, PLUGIN_VAR_RQCMDARG, "Percentage of B-tree page filled during bulk insert", @@ -21954,6 +21961,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(lock_schedule_algorithm), MYSQL_SYSVAR(locks_unsafe_for_binlog), MYSQL_SYSVAR(lock_wait_timeout), + MYSQL_SYSVAR(deadlock_detect), MYSQL_SYSVAR(page_size), MYSQL_SYSVAR(log_buffer_size), MYSQL_SYSVAR(log_file_size), diff --git a/storage/innobase/include/lock0lock.h b/storage/innobase/include/lock0lock.h index ca6a5286cde..e718189062d 100644 --- a/storage/innobase/include/lock0lock.h +++ b/storage/innobase/include/lock0lock.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -55,6 +55,9 @@ extern ulong innodb_lock_schedule_algorithm; // Forward declaration class ReadView; +/** The value of innodb_deadlock_detect */ +extern my_bool innobase_deadlock_detect; + /*********************************************************************//** Gets the size of a lock struct. @return size in bytes */ diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 1068adc6462..800868bef94 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -56,6 +56,9 @@ Created 5/7/1996 Heikki Tuuri /** Lock scheduling algorithm */ ulong innodb_lock_schedule_algorithm = INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS; +/** The value of innodb_deadlock_detect */ +my_bool innobase_deadlock_detect; + /** Total number of cached record locks */ static const ulint REC_LOCK_CACHE = 8; @@ -124,7 +127,7 @@ public: @return id of transaction chosen as victim or 0 */ static const trx_t* check_and_resolve( const lock_t* lock, - const trx_t* trx); + trx_t* trx); private: /** Do a shallow copy. Default destructor OK. @@ -2136,25 +2139,8 @@ RecLock::deadlock_check(lock_t* lock) ut_ad(lock->trx == m_trx); ut_ad(trx_mutex_own(m_trx)); - bool async_rollback = m_trx->in_innodb & TRX_FORCE_ROLLBACK_ASYNC; - - /* This is safe, because DeadlockChecker::check_and_resolve() - is invoked when a lock wait is enqueued for the currently - running transaction. Because m_trx is a running transaction - (it is not currently suspended because of a lock wait), - its state can only be changed by this thread, which is - currently associated with the transaction. */ - - trx_mutex_exit(m_trx); - - /* If transaction is marked for ASYNC rollback then we should - not allow it to wait for another lock causing possible deadlock. - We return current transaction as deadlock victim here. */ - - const trx_t* victim_trx = async_rollback ? m_trx - : DeadlockChecker::check_and_resolve(lock, m_trx); - - trx_mutex_enter(m_trx); + const trx_t* victim_trx = + DeadlockChecker::check_and_resolve(lock, m_trx); /* Check the outcome of the deadlock test. It is possible that the transaction that blocked our lock was rolled back and we @@ -4653,25 +4639,8 @@ lock_table_enqueue_waiting( /* Enqueue the lock request that will wait to be granted */ lock = lock_table_create(c_lock, table, mode | LOCK_WAIT, trx); - bool async_rollback = trx->in_innodb & TRX_FORCE_ROLLBACK_ASYNC; - /* Release the mutex to obey the latching order. - This is safe, because DeadlockChecker::check_and_resolve() - is invoked when a lock wait is enqueued for the currently - running transaction. Because trx is a running transaction - (it is not currently suspended because of a lock wait), - its state can only be changed by this thread, which is - currently associated with the transaction. */ - - trx_mutex_exit(trx); - - /* If transaction is marked for ASYNC rollback then we should - not allow it to wait for another lock causing possible deadlock. - We return current transaction as deadlock victim here. */ - - const trx_t* victim_trx = async_rollback ? trx - : DeadlockChecker::check_and_resolve(lock, trx); - - trx_mutex_enter(trx); + const trx_t* victim_trx = + DeadlockChecker::check_and_resolve(lock, trx); if (victim_trx != 0) { ut_ad(victim_trx == trx); @@ -8441,17 +8410,37 @@ and rolling it back. It will attempt to resolve all deadlocks. The returned transaction id will be the joining transaction instance or NULL if some other transaction was chosen as a victim and rolled back or no deadlock found. -@param lock lock the transaction is requesting -@param trx transaction requesting the lock +@param[in] lock lock the transaction is requesting +@param[in,out] trx transaction requesting the lock @return transaction instanace chosen as victim or 0 */ const trx_t* -DeadlockChecker::check_and_resolve(const lock_t* lock, const trx_t* trx) +DeadlockChecker::check_and_resolve(const lock_t* lock, trx_t* trx) { ut_ad(lock_mutex_own()); + ut_ad(trx_mutex_own(trx)); check_trx_state(trx); ut_ad(!srv_read_only_mode); + /* If transaction is marked for ASYNC rollback then we should + not allow it to wait for another lock causing possible deadlock. + We return current transaction as deadlock victim here. */ + if (trx->in_innodb & TRX_FORCE_ROLLBACK_ASYNC) { + return(trx); + } else if (!innobase_deadlock_detect) { + return(NULL); + } + + /* Release the mutex to obey the latching order. + This is safe, because DeadlockChecker::check_and_resolve() + is invoked when a lock wait is enqueued for the currently + running transaction. Because m_trx is a running transaction + (it is not currently suspended because of a lock wait), + its state can only be changed by this thread, which is + currently associated with the transaction. */ + + trx_mutex_exit(trx); + const trx_t* victim_trx; THD* start_mysql_thd; bool report_waits = false; @@ -8491,7 +8480,7 @@ DeadlockChecker::check_and_resolve(const lock_t* lock, const trx_t* trx) break; - } else if (victim_trx != 0 && victim_trx != trx) { + } else if (victim_trx != NULL && victim_trx != trx) { ut_ad(victim_trx == checker.m_wait_lock->trx); @@ -8512,6 +8501,8 @@ DeadlockChecker::check_and_resolve(const lock_t* lock, const trx_t* trx) lock_deadlock_found = true; } + trx_mutex_enter(trx); + return(victim_trx); } -- cgit v1.2.1 From a930c0aa8647c5d8d1de2c4a22dd30c264310428 Mon Sep 17 00:00:00 2001 From: Shaohua Wang Date: Fri, 5 Aug 2016 09:05:18 +0200 Subject: BUG#23760086 INNODB: ASSERTION FAILURE: MACH0DATA.IC:56:(N | 0XFFFFUL) <= 0XFFFFUL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Analysis: In row_log_table_delete(), extern size could be greater than 2 bytes int if there are enough index on blob columns. Solution: Use 4 bytes int other than 2 bytes for extern size. Reviewed-by: Marko Mäkelä RB: 13573 --- storage/innobase/row/row0log.cc | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'storage') diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc index 7e6975a67f1..a76b6ddd11f 100644 --- a/storage/innobase/row/row0log.cc +++ b/storage/innobase/row/row0log.cc @@ -649,7 +649,7 @@ row_log_table_delete( &old_pk_extra_size); ut_ad(old_pk_extra_size < 0x100); - mrec_size = 4 + old_pk_size; + mrec_size = 6 + old_pk_size; /* Log enough prefix of the BLOB unless both the old and new table are in COMPACT or REDUNDANT format, @@ -686,8 +686,8 @@ row_log_table_delete( *b++ = static_cast(old_pk_extra_size); /* Log the size of external prefix we saved */ - mach_write_to_2(b, ext_size); - b += 2; + mach_write_to_4(b, ext_size); + b += 4; rec_convert_dtuple_to_temp( b + old_pk_extra_size, new_index, @@ -2441,14 +2441,14 @@ row_log_table_apply_op( break; case ROW_T_DELETE: - /* 1 (extra_size) + 2 (ext_size) + at least 1 (payload) */ - if (mrec + 4 >= mrec_end) { + /* 1 (extra_size) + 4 (ext_size) + at least 1 (payload) */ + if (mrec + 6 >= mrec_end) { return(NULL); } extra_size = *mrec++; - ext_size = mach_read_from_2(mrec); - mrec += 2; + ext_size = mach_read_from_4(mrec); + mrec += 4; ut_ad(mrec < mrec_end); /* We assume extra_size < 0x100 for the PRIMARY KEY prefix. @@ -2459,6 +2459,10 @@ row_log_table_apply_op( rec_init_offsets_temp(mrec, new_index, offsets); next_mrec = mrec + rec_offs_data_size(offsets) + ext_size; if (log->table->n_v_cols) { + if (next_mrec + 2 >= mrec_end) { + return(NULL); + } + next_mrec += mach_read_from_2(next_mrec); } -- cgit v1.2.1 From f5759bd8d8a5074ed4cbf8c368b6ba753019e62b Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Wed, 10 Aug 2016 14:47:36 +0530 Subject: Bug #24347476 HIGH PRIORITY TRX FAILED TO KILL LOW PRIORITY TRX WHEN FOREIGN KEYS ARE INVOLVED Problem: ======= High priority transaction can't able to kill the blocking transaction when foreign keys are involved. trx_kill_blocking() missing while checking the foreign key constraint. Fix: === Add trx_kill_blocking() while checking for the foreign key constraint. Reviewed-by: Debarun Banerjee RB: 13579 --- storage/innobase/row/row0ins.cc | 2 ++ 1 file changed, 2 insertions(+) (limited to 'storage') diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index 9626645ebf2..8a180844ce4 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -1878,6 +1878,8 @@ do_possible_lock_wait: my_atomic_addlint( &check_table->n_foreign_key_checks_running, 1); + trx_kill_blocking(trx); + lock_wait_suspend_thread(thr); thr->lock_state = QUE_THR_LOCK_NOLOCK; -- cgit v1.2.1 From 92f7f81b6b0a617c4e1c128ffee34961b9fb6799 Mon Sep 17 00:00:00 2001 From: Shaohua Wang Date: Tue, 16 Aug 2016 03:33:21 +0200 Subject: BUG#24331265 MEMORY LEAK IN SOME INNODB FTS TESTS We don't free some memory used by fts optimize thread in release build. Patch approved by Allen.Lai over IM. --- storage/innobase/fts/fts0opt.cc | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'storage') diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index f1e717c353a..820e82dc600 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -578,9 +578,6 @@ fts_zip_read_word( fts_zip_t* zip, /*!< in: Zip state + data */ fts_string_t* word) /*!< out: uncompressed word */ { -#ifdef UNIV_DEBUG - ulint i; -#endif short len = 0; void* null = NULL; byte* ptr = word->f_str; @@ -656,10 +653,9 @@ fts_zip_read_word( } } -#ifdef UNIV_DEBUG /* All blocks must be freed at end of inflate. */ if (zip->status != Z_OK) { - for (i = 0; i < ib_vector_size(zip->blocks); ++i) { + for (ulint i = 0; i < ib_vector_size(zip->blocks); ++i) { if (ib_vector_getp(zip->blocks, i)) { ut_free(ib_vector_getp(zip->blocks, i)); ib_vector_set(zip->blocks, i, &null); @@ -670,7 +666,6 @@ fts_zip_read_word( if (ptr != NULL) { ut_ad(word->f_len == strlen((char*) ptr)); } -#endif /* UNIV_DEBUG */ return(zip->status == Z_OK || zip->status == Z_STREAM_END ? ptr : NULL); } -- cgit v1.2.1 From 01389ee8ac2632db663be16828ba70363f1ea08d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 8 Aug 2016 14:19:32 +0300 Subject: Bug#24397406 INNODB: ALGORITHM=INPLACE FAILS TO PROMOTE UNIQUE KEY TO CLUSTERED INDEX When a table has no PRIMARY KEY, but there is a UNIQUE INDEX defined on NOT NULL columns that are not column prefixes, that unique index must be treated as the primary key. This property was being violated by InnoDB when a column was changed to NOT NULL, such that a UNIQUE INDEX on that column became eligible to being treated as a primary key. innobase_create_key_defs(): Instead of checking each ADD [UNIQUE] INDEX request, check if a GEN_CLUST_INDEX can be replaced with any unique index in the altered_table definition. So, we can have new_primary even if n_add==0. prepare_inplace_alter_table_dict(): When the table is not being rebuilt, assert that TABLE_SHARE::primary_key is not changing. RB: 13595 Reviewed-by: Kevin Lewis --- storage/innobase/handler/handler0alter.cc | 42 ++++++++++++------------------- 1 file changed, 16 insertions(+), 26 deletions(-) (limited to 'storage') diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 3ede403b5b3..8acc335b127 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -2469,6 +2469,7 @@ innobase_fts_check_doc_id_index_in_def( return(FTS_NOT_EXIST_DOC_ID_INDEX); } + /*******************************************************************//** Create an index table where indexes are ordered as follows: @@ -2537,35 +2538,16 @@ innobase_create_key_defs( (only prefix/part of the column is indexed), MySQL will treat the index as a PRIMARY KEY unless the table already has one. */ - if (n_add > 0 && !new_primary && got_default_clust - && (key_info[*add].flags & HA_NOSAME) - && !(key_info[*add].flags & HA_KEY_HAS_PART_KEY_SEG)) { - uint key_part = key_info[*add].user_defined_key_parts; - - new_primary = true; - - while (key_part--) { - const uint maybe_null - = key_info[*add].key_part[key_part].key_type - & FIELDFLAG_MAYBE_NULL; - bool is_v - = innobase_is_v_fld( - key_info[*add].key_part[key_part].field); - DBUG_ASSERT(!maybe_null - == !key_info[*add].key_part[key_part]. - field->real_maybe_null()); - - if (maybe_null || is_v) { - new_primary = false; - break; - } - } + ut_ad(altered_table->s->primary_key == 0 + || altered_table->s->primary_key == MAX_KEY); + + if (got_default_clust && !new_primary) { + new_primary = (altered_table->s->primary_key != MAX_KEY); } const bool rebuild = new_primary || add_fts_doc_id || innobase_need_rebuild(ha_alter_info, table); - /* Reserve one more space if new_primary is true, and we might need to add the FTS_DOC_ID_INDEX */ indexdef = indexdefs = static_cast( @@ -2579,8 +2561,14 @@ innobase_create_key_defs( ulint primary_key_number; if (new_primary) { - DBUG_ASSERT(n_add > 0); - primary_key_number = *add; + if (n_add == 0) { + DBUG_ASSERT(got_default_clust); + DBUG_ASSERT(altered_table->s->primary_key + == 0); + primary_key_number = 0; + } else { + primary_key_number = *add; + } } else if (got_default_clust) { /* Create the GEN_CLUST_INDEX */ index_def_t* index = indexdef++; @@ -4785,6 +4773,8 @@ new_clustered_failed: ctx->add_cols = add_cols; } else { DBUG_ASSERT(!innobase_need_rebuild(ha_alter_info, old_table)); + DBUG_ASSERT(old_table->s->primary_key + == altered_table->s->primary_key); for (dict_index_t* index = dict_table_get_first_index(user_table); -- cgit v1.2.1 From 223eb5fb9a28c11898a3789515d37abe5dceb856 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Fri, 26 Aug 2016 11:00:44 +0530 Subject: Bug #20989615 INNODB AUTO_INCREMENT PRODUCES SAME VALUE TWICE Problem: ======= Autoincrement value gives duplicate values because of the following reasons. (1) In InnoDB handler function, current autoincrement value is not changed based on newly set auto_increment_increment or auto_increment_offset variable. (2) Handler function does the rounding logic and changes the current autoincrement value and InnoDB doesn't aware of the change in current autoincrement value. Solution: ======== Fix the problem(1), InnoDB always respect the auto_increment_increment and auto_increment_offset value in case of current autoincrement value. By fixing the problem (2), handler layer won't change any current autoincrement value. Reviewed-by: Jimmy Yang RB: 13748 --- storage/innobase/handler/ha_innodb.cc | 31 +++++++++++++++++++++++++++++++ storage/innobase/row/row0mysql.cc | 2 ++ 2 files changed, 33 insertions(+) (limited to 'storage') diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 5d8ff4ff493..6dc625a30d7 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -17667,6 +17667,37 @@ ha_innobase::get_auto_increment( whether we update the table autoinc counter or not. */ ulonglong col_max_value = innobase_get_int_col_max_value(table->next_number_field); + /** The following logic is needed to avoid duplicate key error + for autoincrement column. + + (1) InnoDB gives the current autoincrement value with respect + to increment and offset value. + + (2) Basically it does compute_next_insert_id() logic inside InnoDB + to avoid the current auto increment value changed by handler layer. + + (3) It is restricted only for insert operations. */ + + if (increment > 1 && thd_sql_command(m_user_thd) != SQLCOM_ALTER_TABLE + && autoinc < col_max_value) { + + ulonglong prev_auto_inc = autoinc; + + autoinc = ((autoinc - 1) + increment - offset)/ increment; + + autoinc = autoinc * increment + offset; + + /* If autoinc exceeds the col_max_value then reset + to old autoinc value. Because in case of non-strict + sql mode, boundary value is not considered as error. */ + + if (autoinc >= col_max_value) { + autoinc = prev_auto_inc; + } + + ut_ad(autoinc > 0); + } + /* Called for the first time ? */ if (trx->n_autoinc_rows == 0) { diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index b92794138cc..d34b42e50ba 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1505,6 +1505,8 @@ run_again: row_ins_step(thr); + DEBUG_SYNC_C("ib_after_row_insert_step"); + err = trx->error_state; if (err != DB_SUCCESS) { -- cgit v1.2.1 From 6f5f720848c867987407a15b0a91ba8052944fd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 30 Aug 2016 08:34:49 +0300 Subject: Bug#22018745 CORRUPTION IN ONLINE TABLE REBUILD (ROW_FORMAT=REDUNDANT, INDEXED VIRTUAL COLUMN) Apparently, WL#8149 QA did not cover the code changes made to online table rebuild (which was introduced in MySQL 5.6.8 by WL#6255) for ROW_FORMAT=REDUNDANT tables. row_log_table_low_redundant(): Log the new values of indexed virtual columns (ventry) only once. row_log_table_low(): Assert that if o_ventry is specified, the logged operation must not be ROW_T_INSERT, and ventry must be specified as well. row_log_table_low(): When computing the size of old_pk, pass v_entry=NULL to rec_get_converted_size_temp(), to be consistent with the subsequent call to rec_convert_dtuple_to_temp() for logging old_pk. Assert that old_pk never contains information on virtual columns, thus proving that this change is a no-op. RB: 13822 Reviewed-by: Jimmy Yang --- storage/innobase/row/row0log.cc | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'storage') diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc index a76b6ddd11f..776d1891e6b 100644 --- a/storage/innobase/row/row0log.cc +++ b/storage/innobase/row/row0log.cc @@ -820,12 +820,9 @@ row_log_table_low_redundant( mrec_size = ROW_LOG_HEADER_SIZE + size + (extra_size >= 0x80); - if (ventry && ventry->n_v_fields > 0) { - ulint v_extra = 0; - mrec_size += rec_get_converted_size_temp( - index, NULL, 0, ventry, &v_extra); - + if (num_v) { if (o_ventry) { + ulint v_extra = 0; mrec_size += rec_get_converted_size_temp( index, NULL, 0, o_ventry, &v_extra); } @@ -878,11 +875,7 @@ row_log_table_low_redundant( ventry); b += size; - if (ventry && ventry->n_v_fields > 0) { - rec_convert_dtuple_to_temp( - b, new_index, NULL, 0, ventry); - b += mach_read_from_2(b); - + if (num_v) { if (o_ventry) { rec_convert_dtuple_to_temp( b, new_index, NULL, 0, o_ventry); @@ -943,6 +936,13 @@ row_log_table_low( ut_ad(fil_page_get_type(page_align(rec)) == FIL_PAGE_INDEX); ut_ad(page_is_leaf(page_align(rec))); ut_ad(!page_is_comp(page_align(rec)) == !rec_offs_comp(offsets)); + /* old_pk=row_log_table_get_pk() [not needed in INSERT] is a prefix + of the clustered index record (PRIMARY KEY,DB_TRX_ID,DB_ROLL_PTR), + with no information on virtual columns */ + ut_ad(!old_pk || !insert); + ut_ad(!old_pk || old_pk->n_v_fields == 0); + ut_ad(!o_ventry || !insert); + ut_ad(!o_ventry || ventry); if (dict_index_is_corrupted(index) || !dict_index_is_online_ddl(index) @@ -996,7 +996,7 @@ row_log_table_low( old_pk_size = rec_get_converted_size_temp( new_index, old_pk->fields, old_pk->n_fields, - old_pk, &old_pk_extra_size); + NULL, &old_pk_extra_size); ut_ad(old_pk_extra_size < 0x100); mrec_size += 1/*old_pk_extra_size*/ + old_pk_size; } -- cgit v1.2.1 From 44b1fb36147391aab447571fabeb5e1e1ffc0fcc Mon Sep 17 00:00:00 2001 From: Aditya A Date: Wed, 31 Aug 2016 20:16:48 +0530 Subject: WL9513 Bug#23333990 PERSISTENT INDEX STATISTICS UPDATE BEFORE TRANSACTION IS COMMITTED PROBLEM By design stats estimation always reading uncommitted data. In this scenario an uncommitted transaction has deleted all rows in the table. In Innodb uncommitted delete records are marked as delete but not actually removed from Btree until the transaction has committed or a read view for the rows is present.While calculating persistent stats we were ignoring the delete marked records,since all the records are delete marked we were estimating the number of rows present in the table as zero which leads to bad plans in other transaction operating on the table. Fix Introduced a system variable called innodb_stats_include_delete_marked which when enabled includes delete marked records for stat calculations . --- storage/innobase/dict/dict0stats.cc | 12 +++++++++--- storage/innobase/handler/ha_innodb.cc | 7 +++++++ storage/innobase/include/srv0srv.h | 1 + storage/innobase/srv/srv0srv.cc | 1 + 4 files changed, 18 insertions(+), 3 deletions(-) (limited to 'storage') diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index 47bab4141de..577e989c1ca 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -1152,7 +1152,8 @@ dict_stats_analyze_index_level( them away) which brings non-determinism. We skip only leaf-level delete marks because delete marks on non-leaf level do not make sense. */ - if (level == 0 && + + if (level == 0 && srv_stats_include_delete_marked ? 0: rec_get_deleted_flag( rec, page_is_comp(btr_pcur_get_page(&pcur)))) { @@ -1176,7 +1177,6 @@ dict_stats_analyze_index_level( continue; } - rec_offsets = rec_get_offsets( rec, index, rec_offsets, n_uniq, &heap); @@ -1334,8 +1334,12 @@ enum page_scan_method_t { the given page and count the number of distinct ones, also ignore delete marked records */ - QUIT_ON_FIRST_NON_BORING/* quit when the first record that differs + QUIT_ON_FIRST_NON_BORING,/* quit when the first record that differs from its right neighbor is found */ + COUNT_ALL_NON_BORING_INCLUDE_DEL_MARKED/* scan all records on + the given page and count the number of + distinct ones, include delete marked + records */ }; /* @} */ @@ -1608,6 +1612,8 @@ dict_stats_analyze_index_below_cur( offsets_rec = dict_stats_scan_page( &rec, offsets1, offsets2, index, page, n_prefix, + srv_stats_include_delete_marked ? + COUNT_ALL_NON_BORING_INCLUDE_DEL_MARKED: COUNT_ALL_NON_BORING_AND_SKIP_DEL_MARKED, n_diff, n_external_pages); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 6dc625a30d7..dbf37ee613a 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -20769,6 +20769,12 @@ static MYSQL_SYSVAR_BOOL(use_fallocate, innobase_use_fallocate, "Use posix_fallocate() to allocate files. DEPRECATED, has no effect.", NULL, NULL, FALSE); +static MYSQL_SYSVAR_BOOL(stats_include_delete_marked, + srv_stats_include_delete_marked, + PLUGIN_VAR_OPCMDARG, + "Include delete marked records when calculating persistent statistics", + NULL, NULL, FALSE); + static MYSQL_SYSVAR_ULONG(io_capacity, srv_io_capacity, PLUGIN_VAR_RQCMDARG, "Number of IOPs the server can do. Tunes the background IO rate", @@ -21965,6 +21971,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(temp_data_file_path), MYSQL_SYSVAR(data_home_dir), MYSQL_SYSVAR(doublewrite), + MYSQL_SYSVAR(stats_include_delete_marked), MYSQL_SYSVAR(use_atomic_writes), MYSQL_SYSVAR(use_fallocate), MYSQL_SYSVAR(fast_shutdown), diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index eb08c478965..c3c59cd93d4 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -472,6 +472,7 @@ extern unsigned long long srv_stats_transient_sample_pages; extern my_bool srv_stats_persistent; extern unsigned long long srv_stats_persistent_sample_pages; extern my_bool srv_stats_auto_recalc; +extern my_bool srv_stats_include_delete_marked; extern unsigned long long srv_stats_modified_counter; extern my_bool srv_stats_sample_traditional; diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index c0b218dd6dd..820b10bf726 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -373,6 +373,7 @@ this many index pages, there are 2 ways to calculate statistics: table/index are not found in the innodb database */ unsigned long long srv_stats_transient_sample_pages = 8; my_bool srv_stats_persistent = TRUE; +my_bool srv_stats_include_delete_marked = FALSE; unsigned long long srv_stats_persistent_sample_pages = 20; my_bool srv_stats_auto_recalc = TRUE; -- cgit v1.2.1 From d7d8c23654c282c0bbf1aa90324fc5c9531c413c Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Mon, 10 Apr 2017 11:05:50 +1000 Subject: MDEV-12469: static_assert cannot be determined on bigendian Signed-off-by: Daniel Black --- storage/rocksdb/rdb_datadic.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'storage') diff --git a/storage/rocksdb/rdb_datadic.cc b/storage/rocksdb/rdb_datadic.cc index 255a54cbdce..da4eead99b8 100644 --- a/storage/rocksdb/rdb_datadic.cc +++ b/storage/rocksdb/rdb_datadic.cc @@ -1303,7 +1303,7 @@ static int rdb_unpack_floating_point( // On little-endian, swap the bytes around swap_func(dst, tmp); #else - static_assert(swap_func == nullptr, "Assuming that no swapping is needed."); + DBUG_ASSERT(swap_func == nullptr); #endif return UNPACK_SUCCESS; -- cgit v1.2.1 From 9a848ee1827db1b6f062dc80917385de43f8ec4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 25 Apr 2017 07:15:23 +0300 Subject: Fix the -DWITH_INNODB_AHI=OFF build --- storage/innobase/handler/ha_innodb.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'storage') diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index dbf37ee613a..12ea9af99d3 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -18803,7 +18803,6 @@ innodb_internal_table_validate( return(ret); } -#ifdef BTR_CUR_HASH_ADAPT /****************************************************************//** Update global variable "fts_internal_tbl_name" with the "saved" stopword table name value. This function is registered as a callback @@ -18839,6 +18838,7 @@ innodb_internal_table_update( } } +#ifdef BTR_CUR_HASH_ADAPT /****************************************************************//** Update the system variable innodb_adaptive_hash_index using the "saved" value. This function is registered as a callback with MySQL. */ -- cgit v1.2.1 From 93078c9cb16a079a2d5f261fc95a0ecf99ac48d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 31 Aug 2016 09:21:13 +0300 Subject: Bug#24444831 MY_ERROR(ER_INNODB_ONLINE_LOG_TOO_BIG) CALLED WITH INVALID INDEX NAME This bug was introduced in MySQL 5.6.8 with WL#6255. When an error occurs while rebuilding a table that only has a hidden GEN_CLUST_INDEX inside InnoDB, ha_alter_info->key_info_buffer would be invalid and should not be dereferenced. get_error_key_name(): Get the name of an erroneous key. Avoid dereferencing ha_alter_info->key_info_buffer when no keys exist in the SQL layer. ha_innobase::inplace_alter_table(), ha_innobase::commit_try_rebuild(): Invoke get_error_key_name() for reporting ER_INNODB_ONLINE_LOG_TOO_BIG or ER_INDEX_CORRUPT. RB: 13834 Reviewed-by: Jimmy Yang --- storage/innobase/handler/handler0alter.cc | 41 +++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 13 deletions(-) (limited to 'storage') diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 8acc335b127..8f619c98187 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -6299,6 +6299,26 @@ alter_templ_needs_rebuild( return(false); } +/** Get the name of an erroneous key. +@param[in] error_key_num InnoDB number of the erroneus key +@param[in] ha_alter_info changes that were being performed +@param[in] table InnoDB table +@return the name of the erroneous key */ +static +const char* +get_error_key_name( + ulint error_key_num, + const Alter_inplace_info* ha_alter_info, + const dict_table_t* table) +{ + if (error_key_num == ULINT_UNDEFINED) { + return(FTS_DOC_ID_INDEX_NAME); + } else if (ha_alter_info->key_count == 0) { + return(dict_table_get_first_index(table)->name); + } else { + return(ha_alter_info->key_info_buffer[error_key_num].name); + } +} /** Alter the table structure in-place with operations specified using Alter_inplace_info. @@ -6499,17 +6519,13 @@ oom: case DB_ONLINE_LOG_TOO_BIG: DBUG_ASSERT(ctx->online); my_error(ER_INNODB_ONLINE_LOG_TOO_BIG, MYF(0), - (m_prebuilt->trx->error_key_num == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[ - m_prebuilt->trx->error_key_num].name); + get_error_key_name(m_prebuilt->trx->error_key_num, + ha_alter_info, m_prebuilt->table)); break; case DB_INDEX_CORRUPT: my_error(ER_INDEX_CORRUPT, MYF(0), - (m_prebuilt->trx->error_key_num == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[ - m_prebuilt->trx->error_key_num].name); + get_error_key_name(m_prebuilt->trx->error_key_num, + ha_alter_info, m_prebuilt->table)); break; case DB_DECRYPTION_FAILED: { String str; @@ -7743,14 +7759,13 @@ commit_try_rebuild( DBUG_RETURN(true); case DB_ONLINE_LOG_TOO_BIG: my_error(ER_INNODB_ONLINE_LOG_TOO_BIG, MYF(0), - ha_alter_info->key_info_buffer[0].name); + get_error_key_name(err_key, ha_alter_info, + rebuilt_table)); DBUG_RETURN(true); case DB_INDEX_CORRUPT: my_error(ER_INDEX_CORRUPT, MYF(0), - (err_key == ULINT_UNDEFINED) - ? FTS_DOC_ID_INDEX_NAME - : ha_alter_info->key_info_buffer[err_key] - .name); + get_error_key_name(err_key, ha_alter_info, + rebuilt_table)); DBUG_RETURN(true); default: my_error_innodb(error, table_name, user_table->flags); -- cgit v1.2.1 From a6adf567fd64b5566948ca029f68c613b6b703cc Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Thu, 1 Sep 2016 19:53:04 +0530 Subject: Bug #23533396 ASSERTION !M_PREBUILT->TRX->CHECK_FOREIGNS Analysis: ======== A foreign key constraint cannot reference a secondary index defined on a generated virtual column. While adding new index/drop existing column, server internally drops the internal foreign key index and it leads to choose the virtual secondary index as foreign key index. But innodb doesn't allow foreign key constraint reference to secondary virtual index. Fix: === Allow foreign key constraint refer to secondary index defined on a generated virutal column. Reviewed-by: Jimmy Yang RB: 13586 --- storage/innobase/handler/handler0alter.cc | 1 - 1 file changed, 1 deletion(-) (limited to 'storage') diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 8f619c98187..90a866b4714 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -1296,7 +1296,6 @@ innobase_find_fk_index( while (index != NULL) { if (!(index->type & DICT_FTS) - && !dict_index_has_virtual(index) && dict_foreign_qualify_index( table, col_names, columns, n_cols, index, NULL, true, 0, -- cgit v1.2.1 From e63ead68bf4ef14f836181c834aa010d471abe9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 2 Sep 2016 17:28:54 +0300 Subject: Bug#24346574 PAGE CLEANER THREAD, ASSERT BLOCK->N_POINTERS == 0 btr_search_drop_page_hash_index(): Do not return before ensuring that block->index=NULL, even if !btr_search_enabled. We would typically still skip acquiring the AHI latch when the AHI is disabled, because block->index would already be NULL. Only if the AHI is in the process of being disabled, we would wait for the AHI latch and then notice that block->index=NULL and return. The above bug was a regression caused in MySQL 5.7.9 by the fix of Bug#21407023: DISABLING AHI SHOULD AVOID TAKING AHI LATCH The rest of this patch improves diagnostics by adding assertions. assert_block_ahi_valid(): A debug predicate for checking that block->n_pointers!=0 implies block->index!=NULL. assert_block_ahi_empty(): A debug predicate for checking that block->n_pointers==0. buf_block_init(): Instead of assigning block->n_pointers=0, assert_block_ahi_empty(block). buf_pool_clear_hash_index(): Clarify comments, and assign block->n_pointers=0 before assigning block->index=NULL. The wrong ordering could make block->n_pointers appear incorrect in debug assertions. This bug was introduced in MySQL 5.1.52 by Bug#13006367 62487: INNODB TAKES 3 MINUTES TO CLEAN UP THE ADAPTIVE HASH INDEX AT SHUTDOWN i_s_innodb_buffer_page_get_info(): Add a comment that the IS_HASHED column in the INFORMATION_SCHEMA views INNODB_BUFFER_POOL_PAGE and INNODB_BUFFER_PAGE_LRU may show false positives (there may be no pointers after all.) ha_insert_for_fold_func(), ha_delete_hash_node(), ha_search_and_update_if_found_func(): Use atomics for updating buf_block_t::n_pointers. While buf_block_t::index is always protected by btr_search_x_lock(index), in ha_insert_for_fold_func() the n_pointers-- may belong to another dict_index_t whose btr_search_latches[] we are not holding. RB: 13879 Reviewed-by: Jimmy Yang --- storage/innobase/btr/btr0cur.cc | 2 ++ storage/innobase/btr/btr0sea.cc | 37 +++++++++++------------- storage/innobase/buf/buf0buf.cc | 27 ++++++++++++------ storage/innobase/buf/buf0lru.cc | 57 +++++++++++++++++++++++-------------- storage/innobase/ha/ha0ha.cc | 29 +++++++++++++------ storage/innobase/handler/i_s.cc | 4 +++ storage/innobase/ibuf/ibuf0ibuf.cc | 4 +++ storage/innobase/include/buf0buf.h | 49 +++++++++++++++++++++++++------ storage/innobase/include/buf0buf.ic | 1 + 9 files changed, 143 insertions(+), 67 deletions(-) (limited to 'storage') diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 39d6508adf3..773b03775be 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -3683,6 +3683,8 @@ btr_cur_update_in_place( btr_search_x_lock(index); } + + assert_block_ahi_valid(block); #endif /* BTR_CUR_HASH_ADAPT */ row_upd_rec_in_place(rec, index, offsets, update, page_zip); diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index f7a430591ac..3ae9e95819a 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -611,6 +611,7 @@ btr_search_update_hash_ref( || rw_lock_own(&(block->lock), RW_LOCK_X)); ut_ad(page_align(btr_cur_get_rec(cursor)) == buf_block_get_frame(block)); + assert_block_ahi_valid(block); index = block->index; @@ -1122,14 +1123,13 @@ btr_search_drop_page_hash_index(buf_block_t* block) rw_lock_t* latch; btr_search_t* info; - if (!btr_search_enabled) { - return; - } - retry: /* Do a dirty check on block->index, return if the block is not in the adaptive hash index. */ index = block->index; + /* This debug check uses a dirty read that could theoretically cause + false positives while buf_pool_clear_hash_index() is executing. */ + assert_block_ahi_valid(block); if (index == NULL) { return; @@ -1156,6 +1156,7 @@ retry: ut_ad(!btr_search_own_any(RW_LOCK_X)); rw_lock_s_lock(latch); + assert_block_ahi_valid(block); if (block->index == NULL) { rw_lock_s_unlock(latch); @@ -1172,6 +1173,7 @@ retry: #ifdef MYSQL_INDEX_DISABLE_AHI ut_ad(!index->disable_ahi); #endif + ut_ad(btr_search_enabled); ut_ad(block->page.id.space() == index->space); ut_a(index_id == index->id); @@ -1290,23 +1292,8 @@ next_rec: MONITOR_INC_VALUE(MONITOR_ADAPTIVE_HASH_ROW_REMOVED, n_cached); cleanup: -#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG - if (UNIV_UNLIKELY(block->n_pointers)) { - /* Corruption */ - ib::error() << "Corruption of adaptive hash index." - << " After dropping, the hash index to a page of " - << index->name - << ", still " << block->n_pointers - << " hash nodes remain."; - rw_lock_x_unlock(latch); - - ut_ad(btr_search_validate()); - } else { - rw_lock_x_unlock(latch); - } -#else /* UNIV_AHI_DEBUG || UNIV_DEBUG */ + assert_block_ahi_valid(block); rw_lock_x_unlock(latch); -#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ ut_free(folds); } @@ -1533,6 +1520,7 @@ btr_search_build_page_hash_index( have to take care not to increment the counter in that case. */ if (!block->index) { + assert_block_ahi_empty(block); index->search_info->ref_count++; } @@ -1551,6 +1539,7 @@ btr_search_build_page_hash_index( MONITOR_INC(MONITOR_ADAPTIVE_HASH_PAGE_ADDED); MONITOR_INC_VALUE(MONITOR_ADAPTIVE_HASH_ROW_ADDED, n_cached); exit_func: + assert_block_ahi_valid(block); btr_search_x_unlock(index); ut_free(folds); @@ -1590,6 +1579,8 @@ btr_search_move_or_delete_hash_entries( ut_a(!block->index || block->index == index); ut_a(!(new_block->index || block->index) || !dict_index_is_ibuf(index)); + assert_block_ahi_valid(block); + assert_block_ahi_valid(new_block); if (new_block->index) { @@ -1650,6 +1641,7 @@ btr_search_update_hash_on_delete(btr_cur_t* cursor) ut_ad(rw_lock_own(&(block->lock), RW_LOCK_X)); + assert_block_ahi_valid(block); index = block->index; if (!index) { @@ -1674,6 +1666,7 @@ btr_search_update_hash_on_delete(btr_cur_t* cursor) } btr_search_x_lock(index); + assert_block_ahi_valid(block); if (block->index) { ut_a(block->index == index); @@ -1684,6 +1677,8 @@ btr_search_update_hash_on_delete(btr_cur_t* cursor) MONITOR_INC( MONITOR_ADAPTIVE_HASH_ROW_REMOVE_NOT_FOUND); } + + assert_block_ahi_valid(block); } btr_search_x_unlock(index); @@ -1747,6 +1742,7 @@ btr_search_update_hash_node_on_insert(btr_cur_t* cursor) } func_exit: + assert_block_ahi_valid(block); btr_search_x_unlock(index); } else { btr_search_x_unlock(index); @@ -1791,6 +1787,7 @@ btr_search_update_hash_on_insert(btr_cur_t* cursor) block = btr_cur_get_block(cursor); ut_ad(rw_lock_own(&(block->lock), RW_LOCK_X)); + assert_block_ahi_valid(block); index = block->index; diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index d2902580924..cc567ec73c8 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1496,6 +1496,10 @@ buf_block_init( { UNIV_MEM_DESC(frame, UNIV_PAGE_SIZE); + /* This function should only be executed at database startup or by + buf_pool_resize(). Either way, adaptive hash index must not exist. */ + assert_block_ahi_empty(block); + block->frame = frame; block->page.buf_pool_index = buf_pool_index(buf_pool); @@ -1526,11 +1530,6 @@ buf_block_init( ut_d(block->in_unzip_LRU_list = FALSE); ut_d(block->in_withdraw_list = FALSE); -#ifdef BTR_CUR_HASH_ADAPT -# if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG - block->n_pointers = 0; -# endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ -#endif /* BTR_CUR_HASH_ADAPT */ page_zip_des_init(&block->page.zip); mutex_create(LATCH_ID_BUF_BLOCK_MUTEX, &block->mutex); @@ -2244,6 +2243,10 @@ buf_page_realloc( /* set other flags of buf_block_t */ #ifdef BTR_CUR_HASH_ADAPT + /* This code should only be executed by buf_pool_resize(), + while the adaptive hash index is disabled. */ + assert_block_ahi_empty(block); + assert_block_ahi_empty(new_block); ut_ad(!block->index); new_block->index = NULL; new_block->n_hash_helps = 0; @@ -3210,20 +3213,23 @@ buf_pool_clear_hash_index() for (; i--; block++) { dict_index_t* index = block->index; + assert_block_ahi_valid(block); /* We can set block->index = NULL - when we have an x-latch on search latch; - see the comment in buf0buf.h */ + and block->n_pointers = 0 + when btr_search_own_all(RW_LOCK_X); + see the comments in buf0buf.h */ if (!index) { - /* Not hashed */ continue; } - block->index = NULL; + ut_ad(buf_block_get_state(block) + == BUF_BLOCK_FILE_PAGE); # if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG block->n_pointers = 0; # endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ + block->index = NULL; } } } @@ -3915,6 +3921,9 @@ buf_block_init_low( { block->skip_flush_check = false; #ifdef BTR_CUR_HASH_ADAPT + /* No adaptive hash index entries may point to a previously + unused (and now freshly allocated) block. */ + assert_block_ahi_empty(block); block->index = NULL; block->n_hash_helps = 0; diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index c492ec60494..10a8561d38d 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -295,21 +295,29 @@ next_page: continue; } - mutex_enter(&((buf_block_t*) bpage)->mutex); + buf_block_t* block = reinterpret_cast(bpage); - { - bool skip = bpage->buf_fix_count > 0 - || !((buf_block_t*) bpage)->index; + mutex_enter(&block->mutex); - mutex_exit(&((buf_block_t*) bpage)->mutex); + /* This debug check uses a dirty read that could + theoretically cause false positives while + buf_pool_clear_hash_index() is executing. + (Other conflicting access paths to the adaptive hash + index should not be possible, because when a + tablespace is being discarded or dropped, there must + be no concurrect access to the contained tables.) */ + assert_block_ahi_valid(block); - if (skip) { - /* Skip this block, because there are - no adaptive hash index entries - pointing to it, or because we cannot - drop them due to the buffer-fix. */ - goto next_page; - } + bool skip = bpage->buf_fix_count > 0 || !block->index; + + mutex_exit(&block->mutex); + + if (skip) { + /* Skip this block, because there are + no adaptive hash index entries + pointing to it, or because we cannot + drop them due to the buffer-fix. */ + goto next_page; } /* Store the page number so that we can drop the hash @@ -800,6 +808,17 @@ scan_again: bpage->id, bpage->size); goto scan_again; + } else { + /* This debug check uses a dirty read that could + theoretically cause false positives while + buf_pool_clear_hash_index() is executing, + if the writes to block->index=NULL and + block->n_pointers=0 are reordered. + (Other conflicting access paths to the adaptive hash + index should not be possible, because when a + tablespace is being discarded or dropped, there must + be no concurrect access to the contained tables.) */ + assert_block_ahi_empty((buf_block_t*) bpage); } #endif /* BTR_CUR_HASH_ADAPT */ @@ -1156,6 +1175,9 @@ buf_LRU_get_free_only( || !buf_block_will_withdrawn(buf_pool, block)) { /* found valid free block */ buf_page_mutex_enter(block); + /* No adaptive hash index entries may point to + a free block. */ + assert_block_ahi_empty(block); buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); UNIV_MEM_ALLOC(block->frame, UNIV_PAGE_SIZE); @@ -2030,17 +2052,10 @@ buf_LRU_block_free_non_file_page( case BUF_BLOCK_READY_FOR_USE: break; default: - ib::error() << "Block:" << block - << " incorrect state:" << buf_get_state_name(block) - << " in buf_LRU_block_free_non_file_page"; - return; /* Continue */ + ut_error; } -#ifdef BTR_CUR_HASH_ADAPT -# if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG - ut_a(block->n_pointers == 0); -# endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ -#endif /* BTR_CUR_HASH_ADAPT */ + assert_block_ahi_empty(block); ut_ad(!block->page.in_free_list); ut_ad(!block->page.in_flush_list); ut_ad(!block->page.in_LRU_list); diff --git a/storage/innobase/ha/ha0ha.cc b/storage/innobase/ha/ha0ha.cc index 5822bd6755c..f620db6f62e 100644 --- a/storage/innobase/ha/ha0ha.cc +++ b/storage/innobase/ha/ha0ha.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -189,6 +190,12 @@ ha_clear( } #ifdef BTR_CUR_HASH_ADAPT +# if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG +/** Maximum number of records in a page */ +static const lint MAX_N_POINTERS + = UNIV_PAGE_SIZE_MAX / REC_N_NEW_EXTRA_BYTES; +# endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ + /*************************************************************//** Inserts an entry into a hash table. If an entry with the same fold number is found, its node is updated to point to the new data, and no new node @@ -235,9 +242,11 @@ ha_insert_for_fold_func( buf_block_t* prev_block = prev_node->block; ut_a(prev_block->frame == page_align(prev_node->data)); - ut_a(prev_block->n_pointers > 0); - prev_block->n_pointers--; - block->n_pointers++; + ut_a(my_atomic_addlint( + &prev_block->n_pointers, -1) + < MAX_N_POINTERS); + ut_a(my_atomic_addlint(&block->n_pointers, 1) + < MAX_N_POINTERS); } prev_node->block = block; @@ -268,7 +277,8 @@ ha_insert_for_fold_func( #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG if (table->adaptive) { - block->n_pointers++; + ut_a(my_atomic_addlint(&block->n_pointers, 1) + < MAX_N_POINTERS); } #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ @@ -329,8 +339,8 @@ ha_delete_hash_node( #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG if (table->adaptive) { ut_a(del_node->block->frame = page_align(del_node->data)); - ut_a(del_node->block->n_pointers > 0); - del_node->block->n_pointers--; + ut_a(my_atomic_addlint(&del_node->block->n_pointers, -1) + < MAX_N_POINTERS); } #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ @@ -372,9 +382,10 @@ ha_search_and_update_if_found_func( if (node) { #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG if (table->adaptive) { - ut_a(node->block->n_pointers > 0); - node->block->n_pointers--; - new_block->n_pointers++; + ut_a(my_atomic_addlint(&node->block->n_pointers, -1) + < MAX_N_POINTERS); + ut_a(my_atomic_addlint(&new_block->n_pointers, 1) + < MAX_N_POINTERS); } node->block = new_block; diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 65197e402aa..3d764ef6e7a 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -5149,6 +5149,10 @@ i_s_innodb_buffer_page_get_info( block = reinterpret_cast(bpage); frame = block->frame; #ifdef BTR_CUR_HASH_ADAPT + /* Note: this may be a false positive, that + is, block->index will not always be set to + NULL when the last adaptive hash index + reference is dropped. */ page_info->hashed = (block->index != NULL); #endif /* BTR_CUR_HASH_ADAPT */ } else { diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 7a4dc610694..c7ffecd00c1 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -3953,7 +3953,11 @@ ibuf_insert_to_index_page( ut_ad(ibuf_inside(mtr)); ut_ad(dtuple_check_typed(entry)); #ifdef BTR_CUR_HASH_ADAPT + /* A change buffer merge must occur before users are granted + any access to the page. No adaptive hash index entries may + point to a freshly read page. */ ut_ad(!block->index); + assert_block_ahi_empty(block); #endif /* BTR_CUR_HASH_ADAPT */ ut_ad(mtr->is_named_space(block->page.id.space())); diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index c0b635ab8c9..aaadd544315 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1836,23 +1836,53 @@ struct buf_block_t{ /* @} */ /** @name Hash search fields - These 5 fields may only be modified when we have - an x-latch on search system AND - - we are holding an s-latch or x-latch on buf_block_t::lock or - - we know that buf_block_t::buf_fix_count == 0. + These 5 fields may only be modified when: + we are holding the appropriate x-latch in btr_search_latches[], and + one of the following holds: + (1) the block state is BUF_BLOCK_FILE_PAGE, and + we are holding an s-latch or x-latch on buf_block_t::lock, or + (2) buf_block_t::buf_fix_count == 0, or + (3) the block state is BUF_BLOCK_REMOVE_HASH. An exception to this is when we init or create a page in the buffer pool in buf0buf.cc. - Another exception is that assigning block->index = NULL - is allowed whenever holding an x-latch on search system. */ + Another exception for buf_pool_clear_hash_index() is that + assigning block->index = NULL (and block->n_pointers = 0) + is allowed whenever btr_search_own_all(RW_LOCK_X). + + Another exception is that ha_insert_for_fold_func() may + decrement n_pointers without holding the appropriate latch + in btr_search_latches[]. Thus, n_pointers must be + protected by atomic memory access. + + This implies that the fields may be read without race + condition whenever any of the following hold: + - the btr_search_latches[] s-latch or x-latch is being held, or + - the block state is not BUF_BLOCK_FILE_PAGE or BUF_BLOCK_REMOVE_HASH, + and holding some latch prevents the state from changing to that. + + Some use of assert_block_ahi_empty() or assert_block_ahi_valid() + is prone to race conditions while buf_pool_clear_hash_index() is + executing (the adaptive hash index is being disabled). Such use + is explicitly commented. */ /* @{ */ # if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG ulint n_pointers; /*!< used in debugging: the number of pointers in the adaptive hash index - pointing to this frame */ + pointing to this frame; + protected by atomic memory access + or btr_search_own_all(). */ +# define assert_block_ahi_empty(block) \ + ut_a(my_atomic_addlint(&(block)->n_pointers, 0) == 0) +# define assert_block_ahi_valid(block) \ + ut_a((block)->index \ + || my_atomic_addlint(&(block)->n_pointers, 0) == 0) +# else /* UNIV_AHI_DEBUG || UNIV_DEBUG */ +# define assert_block_ahi_empty(block) /* nothing */ +# define assert_block_ahi_valid(block) /* nothing */ # endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ unsigned curr_n_fields:10;/*!< prefix length for hash indexing: number of full fields */ @@ -1868,11 +1898,14 @@ struct buf_block_t{ complete, though: there may have been hash collisions, record deletions, etc. */ + /* @} */ +#else /* BTR_CUR_HASH_ADAPT */ +# define assert_block_ahi_empty(block) /* nothing */ +# define assert_block_ahi_valid(block) /* nothing */ #endif /* BTR_CUR_HASH_ADAPT */ bool skip_flush_check; /*!< Skip check in buf_dblwr_check_block during bulk load, protected by lock.*/ - /* @} */ # ifdef UNIV_DEBUG /** @name Debug fields */ /* @{ */ diff --git a/storage/innobase/include/buf0buf.ic b/storage/innobase/include/buf0buf.ic index 429c7fd3ba6..f22dcc48a01 100644 --- a/storage/innobase/include/buf0buf.ic +++ b/storage/innobase/include/buf0buf.ic @@ -920,6 +920,7 @@ buf_block_modify_clock_inc( RW_LOCK_FLAG_X | RW_LOCK_FLAG_SX)); } #endif /* UNIV_DEBUG */ + assert_block_ahi_valid(block); block->modify_clock++; } -- cgit v1.2.1 From 472b5f0d1ff53cf9889758f6c2f11fb93ad15e6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 6 Sep 2016 14:57:16 +0300 Subject: Follow-up to Bug#24346574 PAGE CLEANER THREAD, ASSERT BLOCK->N_POINTERS == 0 Silence the Valgrind warnings on instrumented builds (-DWITH_VALGRIND). assert_block_ahi_empty_on_init(): A variant of assert_block_ahi_empty() that declares n_pointers initialized and then asserts that n_pointers==0. In Valgrind-instrumented builds, InnoDB declares allocated memory uninitialized. --- storage/innobase/buf/buf0buf.cc | 6 +++--- storage/innobase/include/buf0buf.h | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) (limited to 'storage') diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index cc567ec73c8..0a227f6b68c 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1498,7 +1498,7 @@ buf_block_init( /* This function should only be executed at database startup or by buf_pool_resize(). Either way, adaptive hash index must not exist. */ - assert_block_ahi_empty(block); + assert_block_ahi_empty_on_init(block); block->frame = frame; @@ -2246,7 +2246,7 @@ buf_page_realloc( /* This code should only be executed by buf_pool_resize(), while the adaptive hash index is disabled. */ assert_block_ahi_empty(block); - assert_block_ahi_empty(new_block); + assert_block_ahi_empty_on_init(new_block); ut_ad(!block->index); new_block->index = NULL; new_block->n_hash_helps = 0; @@ -3923,7 +3923,7 @@ buf_block_init_low( #ifdef BTR_CUR_HASH_ADAPT /* No adaptive hash index entries may point to a previously unused (and now freshly allocated) block. */ - assert_block_ahi_empty(block); + assert_block_ahi_empty_on_init(block); block->index = NULL; block->n_hash_helps = 0; diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index aaadd544315..6832c133d58 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1877,11 +1877,16 @@ struct buf_block_t{ or btr_search_own_all(). */ # define assert_block_ahi_empty(block) \ ut_a(my_atomic_addlint(&(block)->n_pointers, 0) == 0) +# define assert_block_ahi_empty_on_init(block) do { \ + UNIV_MEM_VALID(&(block)->n_pointers, sizeof (block)->n_pointers); \ + assert_block_ahi_empty(block); \ +} while (0) # define assert_block_ahi_valid(block) \ ut_a((block)->index \ || my_atomic_addlint(&(block)->n_pointers, 0) == 0) # else /* UNIV_AHI_DEBUG || UNIV_DEBUG */ # define assert_block_ahi_empty(block) /* nothing */ +# define assert_block_ahi_empty_on_init(block) /* nothing */ # define assert_block_ahi_valid(block) /* nothing */ # endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ unsigned curr_n_fields:10;/*!< prefix length for hash indexing: @@ -1901,6 +1906,7 @@ struct buf_block_t{ /* @} */ #else /* BTR_CUR_HASH_ADAPT */ # define assert_block_ahi_empty(block) /* nothing */ +# define assert_block_ahi_empty_on_init(block) /* nothing */ # define assert_block_ahi_valid(block) /* nothing */ #endif /* BTR_CUR_HASH_ADAPT */ bool skip_flush_check; -- cgit v1.2.1 From 23ea4360fd1e93fad3dc391dd94f2d1592eb7220 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 25 Apr 2017 09:37:27 +0300 Subject: Remove TRX_SYS_OLD_N_RSEGS --- storage/innobase/include/trx0sys.h | 4 ---- storage/innobase/trx/trx0sys.cc | 10 ++++------ 2 files changed, 4 insertions(+), 10 deletions(-) (limited to 'storage') diff --git a/storage/innobase/include/trx0sys.h b/storage/innobase/include/trx0sys.h index b867600653d..c361a9bac55 100644 --- a/storage/innobase/include/trx0sys.h +++ b/storage/innobase/include/trx0sys.h @@ -403,10 +403,6 @@ byte, therefore 128; each slot is currently 8 bytes in size. If you want to raise the level to 256 then you will need to fix some assertions that impose the 7 bit restriction. e.g., mach_write_to_3() */ #define TRX_SYS_N_RSEGS 128 -/* Originally, InnoDB defined TRX_SYS_N_RSEGS as 256 but created only one -rollback segment. It initialized some arrays with this number of entries. -We must remember this limit in order to keep file compatibility. */ -#define TRX_SYS_OLD_N_RSEGS 256 /** Maximum length of MySQL binlog file name, in bytes. */ #define TRX_SYS_MYSQL_LOG_NAME_LEN 512 diff --git a/storage/innobase/trx/trx0sys.cc b/storage/innobase/trx/trx0sys.cc index 19d4a228eda..80aab87edaf 100644 --- a/storage/innobase/trx/trx0sys.cc +++ b/storage/innobase/trx/trx0sys.cc @@ -446,7 +446,6 @@ trx_sysf_create( page_t* page; ulint page_no; byte* ptr; - ulint len; ut_ad(mtr); @@ -481,13 +480,12 @@ trx_sysf_create( mach_write_to_8(sys_header + TRX_SYS_TRX_ID_STORE, 1); /* Reset the rollback segment slots. Old versions of InnoDB - define TRX_SYS_N_RSEGS as 256 (TRX_SYS_OLD_N_RSEGS) and expect + (before MySQL 5.5) define TRX_SYS_N_RSEGS as 256 and expect that the whole array is initialized. */ ptr = TRX_SYS_RSEGS + sys_header; - len = ut_max(TRX_SYS_OLD_N_RSEGS, TRX_SYS_N_RSEGS) - * TRX_SYS_RSEG_SLOT_SIZE; - memset(ptr, 0xff, len); - ptr += len; + compile_time_assert(256 >= TRX_SYS_N_RSEGS); + memset(ptr, 0xff, 256 * TRX_SYS_RSEG_SLOT_SIZE); + ptr += 256 * TRX_SYS_RSEG_SLOT_SIZE; ut_a(ptr <= page + (UNIV_PAGE_SIZE - FIL_PAGE_DATA_END)); /* Initialize all of the page. This part used to be uninitialized. */ -- cgit v1.2.1 From 206ecb79a5f208547ac30ddd3fa108d208198eb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 25 Apr 2017 09:26:01 +0300 Subject: Follow-up to MDEV-12289: Support innodb_undo_tablespaces=127 MySQL 5.7 reduced the maximum number of innodb_undo_tablespaces from 126 to 95 when it reserved 32 persistent rollback segments for the temporary undo logs. Since MDEV-12289 restored all 128 persistent rollback segments for persistent undo logs, the reasonable maximum value of innodb_undo_tablespaces is 127 (not 126 or 95). This is because out of the 128 rollback segments, the first one will always be created in the system tablespace and the remaining ones can be created in dedicated undo tablespaces. --- storage/innobase/handler/ha_innodb.cc | 6 +++--- storage/innobase/include/srv0srv.h | 3 +++ storage/innobase/include/srv0start.h | 2 -- storage/innobase/include/trx0rseg.h | 5 ++++- storage/innobase/include/trx0rseg.ic | 8 ++++++-- storage/innobase/include/trx0sys.h | 2 ++ storage/innobase/srv/srv0start.cc | 20 ++++++++++---------- storage/innobase/trx/trx0sys.cc | 2 +- 8 files changed, 29 insertions(+), 19 deletions(-) (limited to 'storage') diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 12ea9af99d3..5a5af11ec92 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -4487,7 +4487,7 @@ innobase_change_buffering_inited_ok: os_thread_sleep(20); } - srv_was_started = TRUE; + srv_was_started = true; /* Adjust the innodb_undo_logs config object */ innobase_undo_logs_init_default_max(); @@ -21477,11 +21477,11 @@ static MYSQL_SYSVAR_STR(undo_directory, srv_undo_dir, static MYSQL_SYSVAR_ULONG(undo_tablespaces, srv_undo_tablespaces, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "Number of undo tablespaces to use. ", + "Number of undo tablespaces to use.", NULL, NULL, 0L, /* Default seting */ 0L, /* Minimum value */ - 95L, 0); /* Maximum value */ + TRX_SYS_MAX_UNDO_SPACES, 0); /* Maximum value */ static MYSQL_SYSVAR_ULONG(undo_logs, srv_undo_logs, PLUGIN_VAR_OPCMDARG, diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index c3c59cd93d4..8e3659a50e1 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -301,6 +301,9 @@ extern long srv_mtflush_threads; /* If this flag is TRUE, then we will use multi threaded flush. */ extern my_bool srv_use_mtflush; +/** TRUE if the server was successfully started */ +extern bool srv_was_started; + /** Server undo tablespaces directory, can be absolute path. */ extern char* srv_undo_dir; diff --git a/storage/innobase/include/srv0start.h b/storage/innobase/include/srv0start.h index d7a5922c954..4f2f4a312ff 100644 --- a/storage/innobase/include/srv0start.h +++ b/storage/innobase/include/srv0start.h @@ -121,8 +121,6 @@ extern lsn_t srv_start_lsn; extern bool srv_is_being_started; /** TRUE if SYS_TABLESPACES is available for lookups */ extern bool srv_sys_tablespaces_open; -/** TRUE if the server was successfully started */ -extern ibool srv_was_started; /** TRUE if the server is being started, before rolling back any incomplete transactions */ extern bool srv_startup_is_before_trx_rollback_phase; diff --git a/storage/innobase/include/trx0rseg.h b/storage/innobase/include/trx0rseg.h index 95774cbf476..4c162526384 100644 --- a/storage/innobase/include/trx0rseg.h +++ b/storage/innobase/include/trx0rseg.h @@ -200,7 +200,10 @@ struct trx_rseg_t { bool is_persistent() const { ut_ad(space == SRV_TMP_SPACE_ID - || space <= srv_undo_tablespaces); + || space <= TRX_SYS_MAX_UNDO_SPACES); + ut_ad(space == SRV_TMP_SPACE_ID + || space <= srv_undo_tablespaces_active + || !srv_was_started); return(space != SRV_TMP_SPACE_ID); } }; diff --git a/storage/innobase/include/trx0rseg.ic b/storage/innobase/include/trx0rseg.ic index 0a33c747668..45ee3ef8d66 100644 --- a/storage/innobase/include/trx0rseg.ic +++ b/storage/innobase/include/trx0rseg.ic @@ -42,7 +42,9 @@ trx_rsegf_get( buf_block_t* block; trx_rsegf_t* header; - ut_ad(space <= srv_undo_tablespaces || space == SRV_TMP_SPACE_ID); + ut_ad(space <= srv_undo_tablespaces_active || space == SRV_TMP_SPACE_ID + || !srv_was_started); + ut_ad(space <= TRX_SYS_MAX_UNDO_SPACES || space == SRV_TMP_SPACE_ID); block = buf_page_get( page_id_t(space, page_no), univ_page_size, RW_X_LATCH, mtr); @@ -69,7 +71,9 @@ trx_rsegf_get_new( buf_block_t* block; trx_rsegf_t* header; - ut_ad(space <= srv_undo_tablespaces || space == SRV_TMP_SPACE_ID); + ut_ad(space <= srv_undo_tablespaces_active || space == SRV_TMP_SPACE_ID + || !srv_was_started); + ut_ad(space <= TRX_SYS_MAX_UNDO_SPACES || space == SRV_TMP_SPACE_ID); block = buf_page_get( page_id_t(space, page_no), univ_page_size, RW_X_LATCH, mtr); diff --git a/storage/innobase/include/trx0sys.h b/storage/innobase/include/trx0sys.h index c361a9bac55..bf8cf2481eb 100644 --- a/storage/innobase/include/trx0sys.h +++ b/storage/innobase/include/trx0sys.h @@ -403,6 +403,8 @@ byte, therefore 128; each slot is currently 8 bytes in size. If you want to raise the level to 256 then you will need to fix some assertions that impose the 7 bit restriction. e.g., mach_write_to_3() */ #define TRX_SYS_N_RSEGS 128 +/** Maximum number of undo tablespaces (not counting the system tablespace) */ +#define TRX_SYS_MAX_UNDO_SPACES (TRX_SYS_N_RSEGS - 1) /** Maximum length of MySQL binlog file name, in bytes. */ #define TRX_SYS_MYSQL_LOG_NAME_LEN 512 diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 9e89dfda833..ef0d88b4316 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -121,22 +121,22 @@ lsn_t srv_start_lsn; lsn_t srv_shutdown_lsn; /** TRUE if a raw partition is in use */ -ibool srv_start_raw_disk_in_use = FALSE; +ibool srv_start_raw_disk_in_use; /** Number of IO threads to use */ -ulint srv_n_file_io_threads = 0; +ulint srv_n_file_io_threads; /** TRUE if the server is being started, before rolling back any incomplete transactions */ -bool srv_startup_is_before_trx_rollback_phase = false; +bool srv_startup_is_before_trx_rollback_phase; /** TRUE if the server is being started */ -bool srv_is_being_started = false; +bool srv_is_being_started; /** TRUE if SYS_TABLESPACES is available for lookups */ -bool srv_sys_tablespaces_open = false; +bool srv_sys_tablespaces_open; /** TRUE if the server was successfully started */ -ibool srv_was_started = FALSE; +bool srv_was_started; /** TRUE if innobase_start_or_create_for_mysql() has been called */ -static ibool srv_start_has_been_called = FALSE; +static bool srv_start_has_been_called; #ifdef UNIV_DEBUG /** InnoDB system tablespace to set during recovery */ UNIV_INTERN uint srv_sys_space_size_debug; @@ -1519,7 +1519,7 @@ innobase_start_or_create_for_mysql(void) " once during the process lifetime."; } - srv_start_has_been_called = TRUE; + srv_start_has_been_called = true; srv_is_being_started = true; @@ -2889,8 +2889,8 @@ innodb_shutdown() } srv_start_state = SRV_START_STATE_NONE; - srv_was_started = FALSE; - srv_start_has_been_called = FALSE; + srv_was_started = false; + srv_start_has_been_called = false; } #if 0 // TODO: Enable this in WL#6608 diff --git a/storage/innobase/trx/trx0sys.cc b/storage/innobase/trx/trx0sys.cc index 80aab87edaf..1ce9f49ba30 100644 --- a/storage/innobase/trx/trx0sys.cc +++ b/storage/innobase/trx/trx0sys.cc @@ -879,7 +879,7 @@ trx_sys_create_rsegs() srv_undo_logs determines how many of the srv_available_undo_logs rollback segments may be used for logging new transactions. */ - ut_ad(srv_undo_tablespaces < TRX_SYS_N_RSEGS); + ut_ad(srv_undo_tablespaces <= TRX_SYS_MAX_UNDO_SPACES); ut_ad(srv_undo_logs <= TRX_SYS_N_RSEGS); if (srv_read_only_mode) { -- cgit v1.2.1 From bdfa49f6e23e1f6b8dbd1a5c9a782c4e65ab6869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 25 Apr 2017 10:11:40 +0300 Subject: Remove redundant initialization of some InnoDB startup parameters The InnoDB startup parameters will be initialized via pointers in innobase_system_variables[]. Remove some redundant link-time initialization. --- storage/innobase/handler/ha_innodb.cc | 44 +++++++++++++++++------------------ storage/innobase/srv/srv0srv.cc | 26 ++++++++++----------- 2 files changed, 35 insertions(+), 35 deletions(-) (limited to 'storage') diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 5a5af11ec92..e2be027b8e9 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -215,46 +215,46 @@ static uint innobase_old_blocks_pct; /* The default values for the following char* start-up parameters are determined in innobase_init below: */ -static char* innobase_data_home_dir = NULL; -static char* innobase_data_file_path = NULL; -static char* innobase_temp_data_file_path = NULL; -static char* innobase_file_format_name = NULL; -static char* innobase_change_buffering = NULL; -static char* innobase_enable_monitor_counter = NULL; -static char* innobase_disable_monitor_counter = NULL; -static char* innobase_reset_monitor_counter = NULL; -static char* innobase_reset_all_monitor_counter = NULL; +static char* innobase_data_home_dir; +static char* innobase_data_file_path; +static char* innobase_temp_data_file_path; +static char* innobase_file_format_name; +static char* innobase_change_buffering; +static char* innobase_enable_monitor_counter; +static char* innobase_disable_monitor_counter; +static char* innobase_reset_monitor_counter; +static char* innobase_reset_all_monitor_counter; /* The highest file format being used in the database. The value can be set by user, however, it will be adjusted to the newer file format if a table of such format is created/opened. */ -char* innobase_file_format_max = NULL; +char* innobase_file_format_max; /** Default value of innodb_file_format */ static const char* innodb_file_format_default = "Barracuda"; /** Default value of innodb_file_format_max */ static const char* innodb_file_format_max_default = "Antelope"; -static char* innobase_file_flush_method = NULL; +static char* innobase_file_flush_method; /* This variable can be set in the server configure file, specifying stopword table to be used */ -static char* innobase_server_stopword_table = NULL; +static char* innobase_server_stopword_table; /* Below we have boolean-valued start-up parameters, and their default values */ -static my_bool innobase_file_format_check = TRUE; -static my_bool innobase_use_atomic_writes = TRUE; +static my_bool innobase_file_format_check; +static my_bool innobase_use_atomic_writes; static my_bool innobase_use_fallocate; -static my_bool innobase_use_doublewrite = TRUE; -static my_bool innobase_use_checksums = TRUE; -static my_bool innobase_locks_unsafe_for_binlog = FALSE; -static my_bool innobase_rollback_on_timeout = FALSE; -static my_bool innobase_create_status_file = FALSE; -my_bool innobase_stats_on_metadata = TRUE; -static my_bool innobase_large_prefix = FALSE; -static my_bool innodb_optimize_fulltext_only = FALSE; +static my_bool innobase_use_doublewrite; +static my_bool innobase_use_checksums; +static my_bool innobase_locks_unsafe_for_binlog; +static my_bool innobase_rollback_on_timeout; +static my_bool innobase_create_status_file; +my_bool innobase_stats_on_metadata; +static my_bool innobase_large_prefix; +static my_bool innodb_optimize_fulltext_only; static char* innodb_version_str = (char*) INNODB_VERSION_STR; diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 820b10bf726..f63c930865f 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -107,34 +107,34 @@ const char srv_mysql50_table_name_prefix[10] = "#mysql50#"; /* The following three are dir paths which are catenated before file names, where the file name itself may also contain a path */ -char* srv_data_home = NULL; +char* srv_data_home; /** Rollback files directory, can be absolute. */ -char* srv_undo_dir = NULL; +char* srv_undo_dir; /** The number of tablespaces to use for rollback segments. */ -ulong srv_undo_tablespaces = 0; +ulong srv_undo_tablespaces; /** The number of UNDO tablespaces that are open and ready to use. */ -ulint srv_undo_tablespaces_open = 0; +ulint srv_undo_tablespaces_open; /** The number of UNDO tablespaces that are active (hosting some rollback segment). It is quite possible that some of the tablespaces doesn't host any of the rollback-segment based on configuration used. */ -ulint srv_undo_tablespaces_active = 0; +ulint srv_undo_tablespaces_active; /* The number of rollback segments to use */ -ulong srv_undo_logs = 1; +ulong srv_undo_logs; /** Rate at which UNDO records should be purged. */ -ulong srv_purge_rseg_truncate_frequency = 128; +ulong srv_purge_rseg_truncate_frequency; /** Enable or Disable Truncate of UNDO tablespace. Note: If enabled then UNDO tablespace will be selected for truncate. While Server waits for undo-tablespace to truncate if user disables it, truncate action is completed but no new tablespace is marked for truncate (action is never aborted). */ -my_bool srv_undo_log_truncate = FALSE; +my_bool srv_undo_log_truncate; /** Maximum size of undo tablespace. */ unsigned long long srv_max_undo_log_size; @@ -166,9 +166,9 @@ my_bool high_level_read_only; /** Place locks to records only i.e. do not use next-key locking except on duplicate key checking and foreign key checking */ -ibool srv_locks_unsafe_for_binlog = FALSE; +ibool srv_locks_unsafe_for_binlog; /** Sort buffer size in index creation */ -ulong srv_sort_buf_size = 1048576; +ulong srv_sort_buf_size; /** Maximum modification log file size for online index creation */ unsigned long long srv_online_max_size; @@ -176,13 +176,13 @@ unsigned long long srv_online_max_size; OS (provided we compiled Innobase with it in), otherwise we will use simulated aio we build below with threads. Currently we support native aio on windows and linux */ -my_bool srv_use_native_aio = TRUE; -my_bool srv_numa_interleave = FALSE; +my_bool srv_use_native_aio; +my_bool srv_numa_interleave; /* If this flag is TRUE, then we will use fallocate(PUCH_HOLE) to the pages */ UNIV_INTERN my_bool srv_use_trim; /* If this flag is TRUE, then we disable doublewrite buffer */ -UNIV_INTERN my_bool srv_use_atomic_writes = FALSE; +UNIV_INTERN my_bool srv_use_atomic_writes; /* If this flag IS TRUE, then we use this algorithm for page compressing the pages */ UNIV_INTERN ulong innodb_compression_algorithm = PAGE_ZLIB_ALGORITHM; /* Number of threads used for multi-threaded flush */ -- cgit v1.2.1 From d1bcc1f49f8080e9549d29e74b0676875b9ca96b Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Wed, 7 Sep 2016 12:40:10 +0530 Subject: Bug #24488141 ACTIVE UNDO TABLESPACE NOT UPDATED WHEN INNODB_UNDO_LOGS IS INCREASED Problem: ======== If we increase innodb_undo_logs value during startup. New rollback segment might get allocated during trx_sys_create_rseg(). Essentially, it would make these tablesapces active with transaction undo data and purge. But it doesn't come in active undo tablespaces. so these tablespace would never get truncated. Fix: === Increase the number of active undo tablespace when we are assigning the undo tablespace to the newly assigned rollback segment. Reviewed-by: Kevin Lewis Reviewed-by: Debarun Banerjee RB: 13746 --- storage/innobase/trx/trx0sys.cc | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'storage') diff --git a/storage/innobase/trx/trx0sys.cc b/storage/innobase/trx/trx0sys.cc index 1ce9f49ba30..09719b40a46 100644 --- a/storage/innobase/trx/trx0sys.cc +++ b/storage/innobase/trx/trx0sys.cc @@ -921,6 +921,15 @@ trx_sys_create_rsegs() " requested innodb_undo_logs"; return(false); } + + /* Increase the number of active undo + tablespace in case new rollback segment + assigned to new undo tablespace. */ + if (space > srv_undo_tablespaces_active) { + srv_undo_tablespaces_active++; + + ut_ad(srv_undo_tablespaces_active == space); + } } } -- cgit v1.2.1 From ce3ffefc454f1e6eea987e47c52796af48585380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 25 Apr 2017 09:26:01 +0300 Subject: Adapt the innodb_undo tests from MySQL 5.7 Simplify the tests that are present in MySQL 5.7. Make the table smaller while generating enough undo log. Do not unnecessarily drop tables. trx_purge_initiate_truncate(): Remove two crash injection points (before and after normal redo log checkpoint), because they are not adding any value. Clarify some messages. trx_sys_create_rsegs(): Display the number of active undo tablespaces. srv_undo_tablespaces_init(): When initializing the data files, do not leave srv_undo_tablespaces_active at 0. Do not display that number; let trx_sys_create_rsegs() display it once the final number is known. innodb_params_adjust(): Adjust parameters after startup. innobase_init(): Do not allow innodb_max_undo_size to be less than SRV_UNDO_TABLESPACE_SIZE_IN_PAGES. This avoids unnecessary repeated truncation of undo tablespaces when using innodb_page_size=32k or innodb_page_size=64k. --- storage/innobase/handler/ha_innodb.cc | 52 +++++++++++++++----------------- storage/innobase/srv/srv0start.cc | 7 ++--- storage/innobase/trx/trx0purge.cc | 57 +++++++++++------------------------ storage/innobase/trx/trx0sys.cc | 10 ++++-- 4 files changed, 53 insertions(+), 73 deletions(-) (limited to 'storage') diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index e2be027b8e9..c1c7330943e 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1359,15 +1359,11 @@ void innobase_commit_concurrency_init_default(); /*=======================================*/ -/** @brief Initialize the default and max value of innodb_undo_logs. - -Once InnoDB is running, the default value and the max value of -innodb_undo_logs must be equal to the available undo logs, -given by srv_available_undo_logs. */ +/** @brief Adjust some InnoDB startup parameters based on file contents +or innodb_page_size. */ static void -innobase_undo_logs_init_default_max(); -/*==================================*/ +innodb_params_adjust(); /************************************************************//** Validate the file format name and return its corresponding id. @@ -4291,6 +4287,11 @@ innobase_change_buffering_inited_ok: if (UNIV_PAGE_SIZE_DEF != srv_page_size) { ib::info() << "innodb_page_size=" << srv_page_size; + + srv_max_undo_log_size = std::max( + srv_max_undo_log_size, + ulonglong(SRV_UNDO_TABLESPACE_SIZE_IN_PAGES) + * srv_page_size); } if (srv_log_write_ahead_size > srv_page_size) { @@ -4457,12 +4458,6 @@ innobase_change_buffering_inited_ok: } */ - /* Since we in this module access directly the fields of a trx - struct, and due to different headers and flags it might happen that - ib_mutex_t has a different size in this module and in InnoDB - modules, we check at run time that the size is the same in - these compilation modules. */ - err = innobase_start_or_create_for_mysql(); if (srv_buf_pool_size_org != 0) { @@ -4488,8 +4483,7 @@ innobase_change_buffering_inited_ok: } srv_was_started = true; - /* Adjust the innodb_undo_logs config object */ - innobase_undo_logs_init_default_max(); + innodb_params_adjust(); innobase_old_blocks_pct = static_cast( buf_LRU_old_ratio_update(innobase_old_blocks_pct, TRUE)); @@ -21493,12 +21487,10 @@ static MYSQL_SYSVAR_ULONG(undo_logs, srv_undo_logs, static MYSQL_SYSVAR_ULONGLONG(max_undo_log_size, srv_max_undo_log_size, PLUGIN_VAR_OPCMDARG, - "Maximum size of UNDO tablespace in MB (If UNDO tablespace grows" - " beyond this size it will be truncated in due course). ", + "Desired maximum UNDO tablespace size in bytes", NULL, NULL, - 1024 * 1024 * 1024L, - 10 * 1024 * 1024L, - ~0ULL, 0); + 10 << 20, 10 << 20, + 1ULL << (32 + UNIV_PAGE_SIZE_SHIFT_MAX), 0); static MYSQL_SYSVAR_ULONG(purge_rseg_truncate_frequency, srv_purge_rseg_truncate_frequency, @@ -22222,19 +22214,25 @@ innobase_commit_concurrency_init_default() = innobase_commit_concurrency; } -/** @brief Initialize the default and max value of innodb_undo_logs. - -Once InnoDB is running, the default value and the max value of -innodb_undo_logs must be equal to the available undo logs, -given by srv_available_undo_logs. */ +/** @brief Adjust some InnoDB startup parameters based on file contents +or innodb_page_size. */ static void -innobase_undo_logs_init_default_max() -/*=================================*/ +innodb_params_adjust() { + /* The default value and the max value of + innodb_undo_logs must be equal to the available undo logs. */ MYSQL_SYSVAR_NAME(undo_logs).max_val = MYSQL_SYSVAR_NAME(undo_logs).def_val = srv_available_undo_logs; + MYSQL_SYSVAR_NAME(max_undo_log_size).max_val + = 1ULL << (32 + UNIV_PAGE_SIZE_SHIFT); + MYSQL_SYSVAR_NAME(max_undo_log_size).min_val + = MYSQL_SYSVAR_NAME(max_undo_log_size).def_val + = ulonglong(SRV_UNDO_TABLESPACE_SIZE_IN_PAGES) + * srv_page_size; + MYSQL_SYSVAR_NAME(max_undo_log_size).max_val + = 1ULL << (32 + UNIV_PAGE_SIZE_SHIFT); } /**************************************************************************** diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index ef0d88b4316..3ef937c303d 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -894,6 +894,7 @@ srv_undo_tablespaces_init(bool create_new_db) } } } else { + srv_undo_tablespaces_active = srv_undo_tablespaces; n_undo_tablespaces = srv_undo_tablespaces; for (i = 1; i <= n_undo_tablespaces; ++i) { @@ -975,12 +976,10 @@ srv_undo_tablespaces_init(bool create_new_db) return(err != DB_SUCCESS ? err : DB_ERROR); - } else if (n_undo_tablespaces > 0) { + } else if (n_undo_tablespaces > 0) { ib::info() << "Opened " << n_undo_tablespaces - << " undo tablespaces (" - << srv_undo_tablespaces_active - << " active)"; + << " undo tablespaces"; if (srv_undo_tablespaces == 0) { ib::warn() << "innodb_undo_tablespaces=0 disables" diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index f83d9377852..42ae4890cab 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -987,27 +987,20 @@ trx_purge_initiate_truncate( initiate truncate. d. Execute actual truncate e. Remove the DDL log. */ - DBUG_EXECUTE_IF("ib_undo_trunc_before_checkpoint", - ib::info() << "ib_undo_trunc_before_checkpoint"; - DBUG_SUICIDE();); /* After truncate if server crashes then redo logging done for this undo tablespace might not stand valid as tablespace has been truncated. */ log_make_checkpoint_at(LSN_MAX, TRUE); - ib::info() << "Truncating UNDO tablespace with space identifier " - << undo_trunc->get_marked_space_id(); + const ulint space_id = undo_trunc->get_marked_space_id(); - DBUG_EXECUTE_IF("ib_undo_trunc_before_ddl_log_start", - ib::info() << "ib_undo_trunc_before_ddl_log_start"; - DBUG_SUICIDE();); + ib::info() << "Truncating UNDO tablespace " << space_id; #ifdef UNIV_DEBUG dberr_t err = #endif /* UNIV_DEBUG */ - undo_trunc->start_logging( - undo_trunc->get_marked_space_id()); + undo_trunc->start_logging(space_id); ut_ad(err == DB_SUCCESS); DBUG_EXECUTE_IF("ib_undo_trunc_before_truncate", @@ -1016,14 +1009,12 @@ trx_purge_initiate_truncate( trx_purge_cleanse_purge_queue(undo_trunc); - bool success = trx_undo_truncate_tablespace(undo_trunc); - if (!success) { + if (!trx_undo_truncate_tablespace(undo_trunc)) { /* Note: In case of error we don't enable the rsegs and neither unmark the tablespace so the tablespace continue to remain inactive. */ - ib::error() << "Failed to truncate UNDO tablespace with" - " space identifier " - << undo_trunc->get_marked_space_id(); + ib::error() << "Failed to truncate UNDO tablespace " + << space_id; return; } @@ -1046,7 +1037,7 @@ trx_purge_initiate_truncate( log_make_checkpoint_at(LSN_MAX, TRUE); - undo_trunc->done_logging(undo_trunc->get_marked_space_id()); + undo_trunc->done_logging(space_id); /* Completed truncate. Now it is safe to re-use the tablespace. */ for (ulint i = 0; i < undo_trunc->rsegs_size(); ++i) { @@ -1054,8 +1045,7 @@ trx_purge_initiate_truncate( rseg->skip_allocation = false; } - ib::info() << "Completed truncate of UNDO tablespace with space" - " identifier " << undo_trunc->get_marked_space_id(); + ib::info() << "Truncated UNDO tablespace " << space_id; undo_trunc->reset(); undo::Truncate::clear_trunc_list(); @@ -1075,7 +1065,7 @@ trx_purge_truncate_history( purge_iter_t* limit, /*!< in: truncate limit */ const ReadView* view) /*!< in: purge view */ { - ulint i; + ut_ad(trx_purge_check_limit()); /* We play safe and set the truncate limit at most to the purge view low_limit number, though this is not necessary */ @@ -1088,7 +1078,7 @@ trx_purge_truncate_history( ut_ad(limit->trx_no <= purge_sys->view.low_limit_no()); - for (i = 0; i < TRX_SYS_N_RSEGS; ++i) { + for (ulint i = 0; i < TRX_SYS_N_RSEGS; ++i) { trx_rseg_t* rseg = trx_sys->rseg_array[i]; if (rseg != NULL) { @@ -1100,8 +1090,7 @@ trx_purge_truncate_history( /* UNDO tablespace truncate. We will try to truncate as much as we can (greedy approach). This will ensure when the server is idle we try and truncate all the UNDO tablespaces. */ - ulint nchances = srv_undo_tablespaces_active; - for (i = 0; i < nchances; i++) { + for (ulint i = srv_undo_tablespaces_active; i--; ) { trx_purge_mark_undo_for_truncate(&purge_sys->undo_trunc); trx_purge_initiate_truncate(limit, &purge_sys->undo_trunc); } @@ -1638,22 +1627,6 @@ trx_purge_wait_for_workers_to_complete( ut_a(srv_get_task_queue_length() == 0); } -/******************************************************************//** -Remove old historical changes from the rollback segments. */ -static -void -trx_purge_truncate(void) -/*====================*/ -{ - ut_ad(trx_purge_check_limit()); - - if (purge_sys->limit.trx_no == 0) { - trx_purge_truncate_history(&purge_sys->iter, &purge_sys->view); - } else { - trx_purge_truncate_history(&purge_sys->limit, &purge_sys->view); - } -} - /*******************************************************************//** This function runs a purge batch. @return number of undo log pages handled in the batch */ @@ -1742,7 +1715,11 @@ run_synchronously: #endif /* UNIV_DEBUG */ if (truncate) { - trx_purge_truncate(); + trx_purge_truncate_history( + purge_sys->limit.trx_no + ? &purge_sys->limit + : &purge_sys->iter, + &purge_sys->view); } MONITOR_INC_VALUE(MONITOR_PURGE_INVOKED, 1); diff --git a/storage/innobase/trx/trx0sys.cc b/storage/innobase/trx/trx0sys.cc index 09719b40a46..47f30138ceb 100644 --- a/storage/innobase/trx/trx0sys.cc +++ b/storage/innobase/trx/trx0sys.cc @@ -935,8 +935,14 @@ trx_sys_create_rsegs() ut_ad(srv_undo_logs <= srv_available_undo_logs); - ib::info() << srv_undo_logs << " out of " << srv_available_undo_logs - << " rollback segments are active."; + ib::info info; + info << srv_undo_logs << " out of " << srv_available_undo_logs; + if (srv_undo_tablespaces_active) { + info << " rollback segments in " << srv_undo_tablespaces_active + << " undo tablespaces are active."; + } else { + info << " rollback segments are active."; + } return(true); } -- cgit v1.2.1 From da76c1bd3e1a63a5e9c6549551dbbd1abac325e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 25 Apr 2017 15:39:06 +0300 Subject: Minor cleanup --- storage/innobase/buf/buf0dblwr.cc | 16 ++++++---------- storage/innobase/include/buf0dblwr.h | 15 +++++++-------- storage/innobase/log/log0recv.cc | 2 +- 3 files changed, 14 insertions(+), 19 deletions(-) (limited to 'storage') diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 6a0179f36ad..f99fc6434de 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -168,8 +168,7 @@ doublewrite buffer is placed on the trx system header page. @return true if successful, false if not. */ MY_ATTRIBUTE((warn_unused_result)) bool -buf_dblwr_create(void) -/*==================*/ +buf_dblwr_create() { buf_block_t* block2; buf_block_t* new_block; @@ -510,7 +509,7 @@ buf_dblwr_init_or_load_pages( /** Process and remove the double write buffer pages for all tablespaces. */ void -buf_dblwr_process(void) +buf_dblwr_process() { ulint page_no_dblwr = 0; byte* read_buf; @@ -527,9 +526,7 @@ buf_dblwr_process(void) i != recv_dblwr.pages.end(); ++i, ++page_no_dblwr) { byte* page = *i; - ulint page_no = page_get_page_no(page); ulint space_id = page_get_space_id(page); - fil_space_t* space = fil_space_get(space_id); if (space == NULL) { @@ -540,6 +537,7 @@ buf_dblwr_process(void) fil_space_open_if_needed(space); + const ulint page_no = page_get_page_no(page); const page_id_t page_id(space_id, page_no); if (page_no >= space->size) { @@ -676,8 +674,7 @@ buf_dblwr_process(void) /****************************************************************//** Frees doublewrite buffer. */ void -buf_dblwr_free(void) -/*================*/ +buf_dblwr_free() { /* Free the double write data structures. */ ut_a(buf_dblwr != NULL); @@ -932,8 +929,7 @@ important to call this function after a batch of writes has been posted, and also when we may have to wait for a page latch! Otherwise a deadlock of threads can occur. */ void -buf_dblwr_flush_buffered_writes(void) -/*=================================*/ +buf_dblwr_flush_buffered_writes() { byte* write_buf; ulint first_free; diff --git a/storage/innobase/include/buf0dblwr.h b/storage/innobase/include/buf0dblwr.h index bf2bf6ab32b..e1ecb6baf56 100644 --- a/storage/innobase/include/buf0dblwr.h +++ b/storage/innobase/include/buf0dblwr.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -44,8 +44,7 @@ doublewrite buffer is placed on the trx system header page. @return true if successful, false if not. */ MY_ATTRIBUTE((warn_unused_result)) bool -buf_dblwr_create(void); -/*==================*/ +buf_dblwr_create(); /****************************************************************//** At a database startup initializes the doublewrite buffer memory structure if @@ -61,13 +60,13 @@ buf_dblwr_init_or_load_pages( /** Process and remove the double write buffer pages for all tablespaces. */ void -buf_dblwr_process(void); +buf_dblwr_process(); /****************************************************************//** frees doublewrite buffer. */ void -buf_dblwr_free(void); -/*================*/ +buf_dblwr_free(); + /********************************************************************//** Updates the doublewrite buffer when an IO request is completed. */ void @@ -105,8 +104,8 @@ important to call this function after a batch of writes has been posted, and also when we may have to wait for a page latch! Otherwise a deadlock of threads can occur. */ void -buf_dblwr_flush_buffered_writes(void); -/*=================================*/ +buf_dblwr_flush_buffered_writes(); + /********************************************************************//** Writes a page to the doublewrite buffer on disk, sync it, then write the page to the datafile and sync the datafile. This function is used diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index adf2d3aca0a..c035917186a 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2986,7 +2986,7 @@ recv_init_missing_space(dberr_t err, const recv_spaces_t::const_iterator& i) @return error code or DB_SUCCESS */ static MY_ATTRIBUTE((warn_unused_result)) dberr_t -recv_init_crash_recovery_spaces(void) +recv_init_crash_recovery_spaces() { typedef std::set space_set_t; bool flag_deleted = false; -- cgit v1.2.1 From 6c76e5a00b7769661cc9d507a89867c61cf07103 Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Mon, 12 Sep 2016 10:54:45 +0300 Subject: Fix Bug#24605956 SERVER MAY CRASH DUE TO A GLIBC BUG IN HANDLING SHORT-LIVED DETACHED THREADS Avoid detaching and exiting from threads that may finish before the caller has returned from pthread_create(). Only exit from such threads, without detach and join with them later. Patch submitted by: Laurynas Biveinis RB: 13983 Reviewed by: Sunny Bains --- storage/innobase/include/os0thread.h | 14 ++++++++++++-- storage/innobase/os/os0thread.cc | 31 ++++++++++++++++++++++++++++--- storage/innobase/row/row0ftsort.cc | 9 +++++---- storage/innobase/row/row0merge.cc | 7 +++++++ 4 files changed, 52 insertions(+), 9 deletions(-) (limited to 'storage') diff --git a/storage/innobase/include/os0thread.h b/storage/innobase/include/os0thread.h index 7aea976a37e..6f521b5a2ec 100644 --- a/storage/innobase/include/os0thread.h +++ b/storage/innobase/include/os0thread.h @@ -118,9 +118,19 @@ os_thread_create_func( os_thread_id_t* thread_id); /*!< out: id of the created thread, or NULL */ -/** Exits the current thread. */ +/** Waits until the specified thread completes and joins it. +Its return value is ignored. +@param[in,out] thread thread to join */ void -os_thread_exit() +os_thread_join( + os_thread_id_t thread); + +/** Exits the current thread. +@param[in] detach if true, the thread will be detached right before +exiting. If false, another thread is responsible for joining this thread */ +void +os_thread_exit( + bool detach = true) UNIV_COLD MY_ATTRIBUTE((noreturn)); /*****************************************************************//** diff --git a/storage/innobase/os/os0thread.cc b/storage/innobase/os/os0thread.cc index 4012c281f8d..72199b4cf0b 100644 --- a/storage/innobase/os/os0thread.cc +++ b/storage/innobase/os/os0thread.cc @@ -161,9 +161,32 @@ os_thread_create_func( return((os_thread_t)new_thread_id); } -/** Exits the current thread. */ +/** Waits until the specified thread completes and joins it. +Its return value is ignored. +@param[in,out] thread thread to join */ void -os_thread_exit() +os_thread_join( + os_thread_id_t thread) +{ +#ifdef _WIN32 + /* Do nothing. */ +#else +#ifdef UNIV_DEBUG + const int ret = +#endif /* UNIV_DEBUG */ + pthread_join(thread, NULL); + + /* Waiting on already-quit threads is allowed. */ + ut_ad(ret == 0 || ret == ESRCH); +#endif /* _WIN32 */ +} + +/** Exits the current thread. +@param[in] detach if true, the thread will be detached right before +exiting. If false, another thread is responsible for joining this thread */ +void +os_thread_exit( + bool detach) { #ifdef UNIV_DEBUG_THREAD_CREATION ib::info() << "Thread exits, id " @@ -184,7 +207,9 @@ os_thread_exit() ExitThread(0); #else mutex_exit(&thread_mutex); - pthread_detach(pthread_self()); + if (detach) { + pthread_detach(pthread_self()); + } pthread_exit(NULL); #endif } diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index 2adf32b79d9..fca6ae5a1bf 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -1144,7 +1144,7 @@ fts_parallel_merge( os_event_set(psort_info->psort_common->merge_event); psort_info->child_status = FTS_CHILD_EXITING; - os_thread_exit(); + os_thread_exit(false); OS_THREAD_DUMMY_RETURN; } @@ -1157,15 +1157,16 @@ row_fts_start_parallel_merge( fts_psort_t* merge_info) /*!< in: parallel sort info */ { int i = 0; - os_thread_id_t thd_id; /* Kick off merge/insert threads */ for (i = 0; i < FTS_NUM_AUX_INDEX; i++) { merge_info[i].psort_id = i; merge_info[i].child_status = 0; - merge_info[i].thread_hdl = os_thread_create(fts_parallel_merge, - (void*) &merge_info[i], &thd_id); + merge_info[i].thread_hdl = os_thread_create( + fts_parallel_merge, + (void*) &merge_info[i], + &merge_info[i].thread_hdl); } } diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 85f0ce2c9e6..82da61dc03b 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -4875,6 +4875,13 @@ wait_again: " threads exited when creating" " FTS index '" << indexes[i]->name << "'"; + } else { + for (j = 0; j < FTS_NUM_AUX_INDEX; + j++) { + + os_thread_join(merge_info[j] + .thread_hdl); + } } } else { /* This cannot report duplicates; an -- cgit v1.2.1 From 32f99b288b26908eea57fc38ddd656c1a10b2fab Mon Sep 17 00:00:00 2001 From: Daniel Blanchard Date: Fri, 23 Sep 2016 11:20:34 +0100 Subject: Bug #24711351 64 BIT WINDOWS MYSQLD BUILD REPORTS INNODB: OPERATING SYSTEM ERROR NUMBER 995 Description =========== Under heavy load, the aysnchronous Windows file IO API can return a failure code that is handled in MySQL Server by retrying the file IO operation. A cast necessary for the correct operation of the retry path in a 64 bit build is missing, leading to the file IO retry result being misinterpreted and ultimately the report of the OS error number 995 (ERROR_OPERATION_ABORTED) in the MySQL error log. Fix === Supply the missing cast. Reviewed-by: Sunny Bains RB: 14109 --- storage/innobase/os/os0file.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'storage') diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 5213090bb52..110ac6f40f1 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -3496,7 +3496,8 @@ SyncFileIO::execute(Slot* slot) /* Wait for async io to complete */ ret = GetOverlappedResult(slot->file, &slot->control, &slot->n_bytes, TRUE); } - return(ret ? slot->n_bytes : -1); + + return(ret ? static_cast(slot->n_bytes) : -1); } /* Startup/shutdown */ -- cgit v1.2.1 From 8923f6b741498a1bea8bd49a6debec0a9fa490c9 Mon Sep 17 00:00:00 2001 From: Vasil Dimov Date: Tue, 27 Sep 2016 14:09:54 +0300 Subject: Fix Bug#24707869 GCC 5 AND 6 MISCOMPILE MACH_PARSE_COMPRESSED Prevent GCC from moving a mach_read_from_4() before we have checked that we have 4 bytes to read. The pointer may only point to a 1, 2 or 3 bytes in which case the code should not read 4 bytes. This is a workaround to a GCC bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77673 Patch submitted by: Laurynas Biveinis RB: 14135 Reviewed by: Pawel Olchawa --- storage/innobase/mach/mach0data.cc | 65 ++++++++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 13 deletions(-) (limited to 'storage') diff --git a/storage/innobase/mach/mach0data.cc b/storage/innobase/mach/mach0data.cc index b22da00c0f1..6cb33715898 100644 --- a/storage/innobase/mach/mach0data.cc +++ b/storage/innobase/mach/mach0data.cc @@ -49,7 +49,22 @@ mach_parse_compressed( /* 0nnnnnnn (7 bits) */ ++*ptr; return(static_cast(val)); - } else if (val < 0xC0) { + } + + /* Workaround GCC bug + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77673: + the compiler moves mach_read_from_4 right to the beginning of the + function, causing and out-of-bounds read if we are reading a short + integer close to the end of buffer. */ +#if defined(__GNUC__) && (__GNUC__ >= 5) && !defined(__clang__) +#define DEPLOY_FENCE +#endif + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif + + if (val < 0xC0) { /* 10nnnnnn nnnnnnnn (14 bits) */ if (end_ptr >= *ptr + 2) { val = mach_read_from_2(*ptr) & 0x3FFF; @@ -57,7 +72,15 @@ mach_parse_compressed( *ptr += 2; return(static_cast(val)); } - } else if (val < 0xE0) { + *ptr = NULL; + return(0); + } + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif + + if (val < 0xE0) { /* 110nnnnn nnnnnnnn nnnnnnnn (21 bits) */ if (end_ptr >= *ptr + 3) { val = mach_read_from_3(*ptr) & 0x1FFFFF; @@ -65,7 +88,15 @@ mach_parse_compressed( *ptr += 3; return(static_cast(val)); } - } else if (val < 0xF0) { + *ptr = NULL; + return(0); + } + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif + + if (val < 0xF0) { /* 1110nnnn nnnnnnnn nnnnnnnn nnnnnnnn (28 bits) */ if (end_ptr >= *ptr + 4) { val = mach_read_from_4(*ptr) & 0xFFFFFFF; @@ -73,16 +104,24 @@ mach_parse_compressed( *ptr += 4; return(static_cast(val)); } - } else { - ut_ad(val == 0xF0); - - /* 11110000 nnnnnnnn nnnnnnnn nnnnnnnn nnnnnnnn (32 bits) */ - if (end_ptr >= *ptr + 5) { - val = mach_read_from_4(*ptr + 1); - ut_ad(val > 0xFFFFFFF); - *ptr += 5; - return(static_cast(val)); - } + *ptr = NULL; + return(0); + } + +#ifdef DEPLOY_FENCE + __atomic_thread_fence(__ATOMIC_ACQUIRE); +#endif + +#undef DEPLOY_FENCE + + ut_ad(val == 0xF0); + + /* 11110000 nnnnnnnn nnnnnnnn nnnnnnnn nnnnnnnn (32 bits) */ + if (end_ptr >= *ptr + 5) { + val = mach_read_from_4(*ptr + 1); + ut_ad(val > 0xFFFFFFF); + *ptr += 5; + return(static_cast(val)); } *ptr = NULL; -- cgit v1.2.1 From 9df04261035638cdf55237decf4cacab256fb473 Mon Sep 17 00:00:00 2001 From: Knut Anders Hatlen Date: Fri, 4 Nov 2016 13:44:36 +0100 Subject: Bug#25048573: STD::MAP INSTANTIATIONS CAUSE STATIC ASSERT FAILURES ON FREEBSD 11 Problem: Some instantiations of std::map have discrepancies between the value_type of the map and the value_type of the map's allocator. On FreeBSD 11 this is detected by Clang, and an error is raised at compilation time. Fix: Specify the correct value_type for the allocators. Also fix an unused variable warning in storage/innobase/os/os0file.cc. --- storage/innobase/buf/buf0buf.cc | 2 +- storage/innobase/dict/dict0stats.cc | 2 +- storage/innobase/sync/sync0debug.cc | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'storage') diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 0a227f6b68c..3761d81b2d1 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -347,7 +347,7 @@ typedef std::map< const byte*, buf_chunk_t*, std::less, - ut_allocator > > + ut_allocator > > buf_pool_chunk_map_t; static buf_pool_chunk_map_t* buf_chunk_map_reg; diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index 577e989c1ca..e7fc56aaa8f 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -139,7 +139,7 @@ then we would store 5,7,10,11,12 in the array. */ typedef std::vector > boundaries_t; /** Allocator type used for index_map_t. */ -typedef ut_allocator > +typedef ut_allocator > index_map_t_allocator; /** Auxiliary map used for sorting indexes by name in dict_stats_save(). */ diff --git a/storage/innobase/sync/sync0debug.cc b/storage/innobase/sync/sync0debug.cc index 135c84ad0d8..c787a32e58a 100644 --- a/storage/innobase/sync/sync0debug.cc +++ b/storage/innobase/sync/sync0debug.cc @@ -130,7 +130,7 @@ struct LatchDebug { os_thread_id_t, Latches*, os_thread_id_less, - ut_allocator > > + ut_allocator > > ThreadMap; /** Constructor */ @@ -425,7 +425,7 @@ private: latch_level_t, std::string, latch_level_less, - ut_allocator > > + ut_allocator > > Levels; /** Mutex protecting the deadlock detector data structures. */ @@ -1717,7 +1717,7 @@ private: const void*, File, std::less, - ut_allocator > > + ut_allocator > > Files; typedef OSMutex Mutex; -- cgit v1.2.1 From bf5be323765b7a7e31e8813bbb995ca08e74f940 Mon Sep 17 00:00:00 2001 From: Debarun Banerjee Date: Sat, 12 Nov 2016 20:53:15 +0530 Subject: BUG#25032066 PREPARED TRANSACTION SHOULD NOT BE ROLLED BACK BY HIGH PRIORITY TRANSACTION Problem : --------- 1. delete_all_rows() and rnd_init() are not returning error after async rollback in 5.7. This results in assert in innodb in next call. 2. High priority transaction is rolling back prepared transaction. This is because TRX_FORCE_ROLLBACK_DISABLE is getting set only for first entry [TrxInInnoDB]. Solution : ---------- 1. return DB_FORCED_ABORT error after rollback. 2. check and disable rollback in TrxInInnodb::enter always. Reviewed-by: Sunny Bains RB: 13777 --- storage/innobase/handler/ha_innodb.cc | 8 +------ storage/innobase/include/trx0trx.h | 44 ++++++++++++++++++----------------- 2 files changed, 24 insertions(+), 28 deletions(-) (limited to 'storage') diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index c1c7330943e..e039722deba 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -10471,13 +10471,7 @@ ha_innobase::rnd_init( bool scan) /*!< in: true if table/index scan FALSE otherwise */ { TrxInInnoDB trx_in_innodb(m_prebuilt->trx); - - if (trx_in_innodb.is_aborted()) { - - return(innobase_rollback(ht, m_user_thd, false)); - } - - int err; + int err; /* Store the active index value so that we can restore the original value after a scan */ diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index b08148578dc..9e5248e7b86 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -1468,10 +1468,30 @@ private: return; } - /* Avoid excessive mutex acquire/release */ - ut_ad(!is_async_rollback(trx)); + /* If it hasn't already been marked for async rollback. + and it will be committed/rolled back. */ + if (disable) { + + trx_mutex_enter(trx); + if (!is_forced_rollback(trx) + && is_started(trx) + && !trx_is_autocommit_non_locking(trx)) { + + ut_ad(trx->killed_by == 0); + + /* This transaction has crossed the point of + no return and cannot be rolled back + asynchronously now. It must commit or rollback + synhronously. */ + + trx->in_innodb |= TRX_FORCE_ROLLBACK_DISABLE; + } + trx_mutex_exit(trx); + } + + /* Avoid excessive mutex acquire/release */ ++trx->in_depth; /* If trx->in_depth is greater than 1 then @@ -1489,25 +1509,7 @@ private: wait(trx); - ut_ad((trx->in_innodb & TRX_FORCE_ROLLBACK_MASK) - < (TRX_FORCE_ROLLBACK_MASK - 1)); - - /* If it hasn't already been marked for async rollback. - and it will be committed/rolled back. */ - - if (!is_forced_rollback(trx) - && disable - && is_started(trx) - && !trx_is_autocommit_non_locking(trx)) { - - ut_ad(trx->killed_by == 0); - - /* This transaction has crossed the point of no - return and cannot be rolled back asynchronously - now. It must commit or rollback synhronously. */ - - trx->in_innodb |= TRX_FORCE_ROLLBACK_DISABLE; - } + ut_ad((trx->in_innodb & TRX_FORCE_ROLLBACK_MASK) == 0); ++trx->in_innodb; -- cgit v1.2.1 From 4e41ac26f5a1c2d5a601754c3593fbc031ceaae6 Mon Sep 17 00:00:00 2001 From: Debarun Banerjee Date: Sun, 13 Nov 2016 10:31:35 +0530 Subject: BUG#25082593 FOREIGN KEY VALIDATION DOESN'T NEED TO ACQUIRE GAP LOCK IN READ COMMITTED Problem : --------- This bug is filed from the base replication bug#25040331 where the slave thread times out while INSERT operation waits on GAP lock taken during Foreign Key validation. The primary reason for the lock wait is because the statements are getting replayed in different order. However, we also observed two things ... 1. The slave thread could always use "Read Committed" isolation for row level replication. 2. It is not necessary to have GAP locks in "READ Committed" isolation level in innodb. This bug is filed to address point(2) to avoid taking GAP locks during Foreign Key validation. Solution : ---------- Innodb is primarily designed for "Repeatable Read" and the GAP lock behaviour is default. For "Read Committed" isolation, we have special handling in row_search_mvcc to avoid taking the GAP lock while scanning records. While looking for Foreign Key, the code is following the default behaviour taking GAP locks. The suggested fix is to avoid GAP locking during FK validation similar to normal search operation (row_search_mvcc) for "Read Committed" isolation level. Reviewed-by: Sunny Bains RB: 14526 --- storage/innobase/row/row0ins.cc | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) (limited to 'storage') diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index 8a180844ce4..5803bc226cd 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -1596,6 +1596,10 @@ row_ins_check_foreign_constraint( ulint offsets_[REC_OFFS_NORMAL_SIZE]; ulint* offsets = offsets_; + bool skip_gap_lock; + + skip_gap_lock = (trx->isolation_level <= TRX_ISO_READ_COMMITTED); + DBUG_ENTER("row_ins_check_foreign_constraint"); rec_offs_init(offsets_); @@ -1725,6 +1729,11 @@ row_ins_check_foreign_constraint( if (page_rec_is_supremum(rec)) { + if (skip_gap_lock) { + + continue; + } + err = row_ins_set_shared_rec_lock(LOCK_ORDINARY, block, rec, check_index, offsets, thr); @@ -1740,10 +1749,17 @@ row_ins_check_foreign_constraint( cmp = cmp_dtuple_rec(entry, rec, offsets); if (cmp == 0) { + + ulint lock_type; + + lock_type = skip_gap_lock + ? LOCK_REC_NOT_GAP + : LOCK_ORDINARY; + if (rec_get_deleted_flag(rec, rec_offs_comp(offsets))) { err = row_ins_set_shared_rec_lock( - LOCK_ORDINARY, block, + lock_type, block, rec, check_index, offsets, thr); switch (err) { case DB_SUCCESS_LOCKED_REC: @@ -1824,9 +1840,13 @@ row_ins_check_foreign_constraint( } else { ut_a(cmp < 0); - err = row_ins_set_shared_rec_lock( - LOCK_GAP, block, - rec, check_index, offsets, thr); + err = DB_SUCCESS; + + if (!skip_gap_lock) { + err = row_ins_set_shared_rec_lock( + LOCK_GAP, block, + rec, check_index, offsets, thr); + } switch (err) { case DB_SUCCESS_LOCKED_REC: -- cgit v1.2.1 From 07e88be5b7c6720fc95598c604274f2bdd2b8a6e Mon Sep 17 00:00:00 2001 From: Allen Lai Date: Tue, 22 Nov 2016 14:35:16 +0800 Subject: Bug#23044098 INSERT OF GIS DATA INTO RTREE HITS ASSERT IN RTR_CUR_RESTORE_POSITION_FUNC() This bug is caused by missing page number field when store btree cursor. Reviewed-by: Jimmy Yang RB: 14617 --- storage/innobase/rem/rem0rec.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'storage') diff --git a/storage/innobase/rem/rem0rec.cc b/storage/innobase/rem/rem0rec.cc index 6d974e7accb..26bb12e8a03 100644 --- a/storage/innobase/rem/rem0rec.cc +++ b/storage/innobase/rem/rem0rec.cc @@ -1652,6 +1652,7 @@ rec_copy_prefix_to_buf( ulint prefix_len; ulint null_mask; ulint status; + bool is_rtr_node_ptr = false; UNIV_PREFETCH_RW(*buf); @@ -1673,6 +1674,7 @@ rec_copy_prefix_to_buf( /* For R-tree, we need to copy the child page number field. */ if (dict_index_is_spatial(index)) { ut_ad(n_fields == DICT_INDEX_SPATIAL_NODEPTR_SIZE + 1); + is_rtr_node_ptr = true; } else { /* it doesn't make sense to copy the child page number field */ @@ -1717,7 +1719,11 @@ rec_copy_prefix_to_buf( null_mask <<= 1; } - if (field->fixed_len) { + if (is_rtr_node_ptr && i == 1) { + /* For rtree node ptr rec, we need to + copy the page no field with 4 bytes len. */ + prefix_len += 4; + } else if (field->fixed_len) { prefix_len += field->fixed_len; } else { ulint len = *lens--; -- cgit v1.2.1 From 49edf2d47636865e35d4916f72fe345f7d3d970e Mon Sep 17 00:00:00 2001 From: Debarun Banerjee Date: Fri, 25 Nov 2016 11:04:23 +0530 Subject: BUG#25126722 FOREIGN KEY CONSTRAINT NAME IS NULL IN INFORMATION_SCHEMA AFTER RESTART Problem : --------- Information_Schema.referential_constraints (UNIQUE_CONSTRAINT_NAME) shows NULL for a foreign key constraint after restarting the server. If any dml or query (select/insert/update/delete) is done on referenced table, then the constraint name is correctly shown. Solution : ---------- UNIQUE_CONSTRAINT_NAME column is the key name of the referenced table. In innodb, FK reference is stored as a list of columns in referenced table in INNODB_SYS_FOREIGN and INNODB_SYS_FOREIGN_COLS. The referenced column must have at least one index/key with the referenced column as prefix but the key name itself is not included in FK metadata. For this reason, the UNIQUE_CONSTRAINT_NAME is only filled up when the referenced table is actually loaded in innodb dictionary cache. The information_schema view calls handler::get_foreign_key_list() on foreign key table to read the FK metadata. The UNIQUE_CONSTRAINT_NAME information shows NULL based on whether the referenced table is already loaded or not. One way to fix this issue is to load the referenced table while reading the FK metadata information, if needed. Reviewed-by: Sunny Bains RB: 14654 --- storage/innobase/handler/ha_innodb.cc | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'storage') diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index e039722deba..af1793d1920 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -15863,6 +15863,28 @@ get_foreign_key_info( f_key_info.update_method = FK_OPTION_RESTRICT; } + /* Load referenced table to update FK referenced key name. */ + if (foreign->referenced_table == NULL) { + + dict_table_t* ref_table; + + ut_ad(mutex_own(&dict_sys->mutex)); + ref_table = dict_table_open_on_name( + foreign->referenced_table_name_lookup, + TRUE, FALSE, DICT_ERR_IGNORE_NONE); + + if (ref_table == NULL) { + + ib::info() << "Foreign Key referenced table " + << foreign->referenced_table_name + << " not found for foreign table " + << foreign->foreign_table_name; + } else { + + dict_table_close(ref_table, TRUE, FALSE); + } + } + if (foreign->referenced_index && foreign->referenced_index->name != NULL) { referenced_key_name = thd_make_lex_string( -- cgit v1.2.1 From 35652ed631c442bf901fd9a6a9938edd787bc484 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Wed, 14 Dec 2016 18:42:46 +0530 Subject: Bug#25222337 FIELD NAME IS NULL IN NEWLY ADDED VIRTUAL INDEX FOR NEWLY ADDED VIRTUAL COLUMN Analysis: ======== Field name comparison happens while filling the virtual columns affected by foreign constraint. But field name is NULL in virtual index for the newly added virtual column. Fix: === Ignore the index if it has newly added virtual column. Foreign key affected virtual column information is filled during loading operation. Reviewed-by: Jimmy Yang RB: 14895 --- storage/innobase/dict/dict0mem.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'storage') diff --git a/storage/innobase/dict/dict0mem.cc b/storage/innobase/dict/dict0mem.cc index 13db213259c..7a6f09569a6 100644 --- a/storage/innobase/dict/dict0mem.cc +++ b/storage/innobase/dict/dict0mem.cc @@ -810,7 +810,12 @@ dict_mem_fill_vcol_from_v_indexes( index; index = dict_table_get_next_index(index)) { - if (!dict_index_has_virtual(index)) { + /* Skip if the index have newly added + virtual column because field name is NULL. + Later virtual column set will be + refreshed during loading of table. */ + if (!dict_index_has_virtual(index) + || index->has_new_v_col) { continue; } -- cgit v1.2.1 From c2d9c0ce85d0c8bf6ed49904707a147fb181e182 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Thu, 15 Dec 2016 15:38:06 +0530 Subject: Bug #24585978 INNODB: ASSERTION TOTAL_RECS > 0 FAILURE IN FILE DICT0STATS.CC Analysis: ======== There was missing bracket for IF conditon in dict_stats_analyze_index_level() and it leads to wrong result. Fix: ==== Fix the IF condition in dict_stats_analyze_index_level() so that it satisfied the if condtion only if level is zero. Reviewed-by : Jimmy Yang --- storage/innobase/dict/dict0stats.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'storage') diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index e7fc56aaa8f..537a70c2069 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -1153,10 +1153,10 @@ dict_stats_analyze_index_level( leaf-level delete marks because delete marks on non-leaf level do not make sense. */ - if (level == 0 && srv_stats_include_delete_marked ? 0: + if (level == 0 && (srv_stats_include_delete_marked ? 0: rec_get_deleted_flag( rec, - page_is_comp(btr_pcur_get_page(&pcur)))) { + page_is_comp(btr_pcur_get_page(&pcur))))) { if (rec_is_last_on_page && !prev_rec_is_copied -- cgit v1.2.1 From 16ed1f9c31db72bc156d80a0f2ee5791b0fdd89f Mon Sep 17 00:00:00 2001 From: Darshan M N Date: Thu, 15 Dec 2016 16:50:44 +0530 Subject: BUG#25053705 INVALID I/O ON TABLE AFTER TRUNCATE Issue: ====== The issue is that if a fts index is present in a table the space size is incorrectly calculated in the case of truncate which results in a invalid read. Fix: ==== Have a different space size calculation in truncate if fts indexes are present. RB:14755 Reviewed-by: Shaohua Wang --- storage/innobase/buf/buf0rea.cc | 20 +++++++++++++++++++- storage/innobase/row/row0trunc.cc | 19 ++++++++++++++++--- 2 files changed, 35 insertions(+), 4 deletions(-) (limited to 'storage') diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 4d68ad5ac51..33d82ee5a87 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2015, 2016 MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -290,6 +290,24 @@ buf_read_ahead_random( below: if DISCARD + IMPORT changes the actual .ibd file meanwhile, we do not try to read outside the bounds of the tablespace! */ if (fil_space_t* space = fil_space_acquire(page_id.space())) { + +#ifdef UNIV_DEBUG + if (srv_file_per_table) { + ulint size = 0; + + for (const fil_node_t* node = + UT_LIST_GET_FIRST(space->chain); + node != NULL; + node = UT_LIST_GET_NEXT(chain, node)) { + + size += os_file_get_size(node->handle) + / page_size.logical(); + } + + ut_ad(size==space->size); + } +#endif /* UNIV_DEBUG */ + if (high > space->size) { high = space->size; } diff --git a/storage/innobase/row/row0trunc.cc b/storage/innobase/row/row0trunc.cc index 46cff288059..8529cfec9fd 100644 --- a/storage/innobase/row/row0trunc.cc +++ b/storage/innobase/row/row0trunc.cc @@ -2000,9 +2000,22 @@ row_truncate_table_for_mysql( } if (is_file_per_table && fsp_flags != ULINT_UNDEFINED) { - fil_reinit_space_header( - table->space, - table->indexes.count + FIL_IBD_FILE_INITIAL_SIZE + 1); + /* A single-table tablespace has initially + FIL_IBD_FILE_INITIAL_SIZE number of pages allocated and an + extra page is allocated for each of the indexes present. But in + the case of clust index 2 pages are allocated and as one is + covered in the calculation as part of table->indexes.count we + take care of the other page by adding 1. */ + ulint space_size = table->indexes.count + + FIL_IBD_FILE_INITIAL_SIZE + 1; + + if (has_internal_doc_id) { + /* Since aux tables are created for fts indexes and + they use seperate tablespaces. */ + space_size -= ib_vector_size(table->fts->indexes); + } + + fil_reinit_space_header(table->space, space_size); } DBUG_EXECUTE_IF("ib_trunc_crash_with_intermediate_log_checkpoint", -- cgit v1.2.1 From 698e37d60c6ac3e0c524011ed33b964d84dcfaee Mon Sep 17 00:00:00 2001 From: Darshan M N Date: Fri, 16 Dec 2016 12:14:38 +0530 Subject: BUG#25251082 DISABLING CERTAIN MACROS IN INNODB RESULTS IN COMPILATION ERRORS Issue: ====== Disabling macros such as UNIV_PFS_MUTEX/UNIV_PFS_RWLOCK/UNIV_PFS_THREAD which are defined in InnoDB throws errors during compilation. Fix: ==== Fix all the compilation errors. RB: 14893 Reviewed-by: Jimmy Yang Reviewed-by: Satya Bodapati --- storage/innobase/include/sync0types.h | 26 +++- storage/innobase/sync/sync0debug.cc | 254 +++++++++++++++++++--------------- 2 files changed, 164 insertions(+), 116 deletions(-) (limited to 'storage') diff --git a/storage/innobase/include/sync0types.h b/storage/innobase/include/sync0types.h index 736c5454711..1d11bfa7811 100644 --- a/storage/innobase/include/sync0types.h +++ b/storage/innobase/include/sync0types.h @@ -505,14 +505,32 @@ private: }; #ifdef UNIV_PFS_MUTEX -/** Latch element +/** Latch element. +Used for mutexes which have PFS keys defined under UNIV_PFS_MUTEX. @param[in] id Latch id @param[in] level Latch level @param[in] key PFS key */ -# define LATCH_ADD(id, level, key) latch_meta[LATCH_ID_ ## id] = \ +# define LATCH_ADD_MUTEX(id, level, key) latch_meta[LATCH_ID_ ## id] =\ + UT_NEW_NOKEY(latch_meta_t(LATCH_ID_ ## id, #id, level, #level, key)) + +#ifdef UNIV_PFS_RWLOCK +/** Latch element. +Used for rwlocks which have PFS keys defined under UNIV_PFS_RWLOCK. +@param[in] id Latch id +@param[in] level Latch level +@param[in] key PFS key */ +# define LATCH_ADD_RWLOCK(id, level, key) latch_meta[LATCH_ID_ ## id] =\ UT_NEW_NOKEY(latch_meta_t(LATCH_ID_ ## id, #id, level, #level, key)) #else -# define LATCH_ADD(id, level, key) latch_meta[LATCH_ID_ ## id] = \ +# define LATCH_ADD_RWLOCK(id, level, key) latch_meta[LATCH_ID_ ## id] =\ + UT_NEW_NOKEY(latch_meta_t(LATCH_ID_ ## id, #id, level, #level, \ + PSI_NOT_INSTRUMENTED)) +#endif /* UNIV_PFS_RWLOCK */ + +#else +# define LATCH_ADD_MUTEX(id, level, key) latch_meta[LATCH_ID_ ## id] =\ + UT_NEW_NOKEY(latch_meta_t(LATCH_ID_ ## id, #id, level, #level)) +# define LATCH_ADD_RWLOCK(id, level, key) latch_meta[LATCH_ID_ ## id] =\ UT_NEW_NOKEY(latch_meta_t(LATCH_ID_ ## id, #id, level, #level)) #endif /* UNIV_PFS_MUTEX */ @@ -919,7 +937,7 @@ sync_latch_get_level(latch_id_t id) return(meta.get_level()); } -#ifdef HAVE_PSI_INTERFACE +#ifdef UNIV_PFS_MUTEX /** Get the latch PFS key from the latch ID @param[in] id Latch ID @return the PFS key */ diff --git a/storage/innobase/sync/sync0debug.cc b/storage/innobase/sync/sync0debug.cc index c787a32e58a..4fff24a77f1 100644 --- a/storage/innobase/sync/sync0debug.cc +++ b/storage/innobase/sync/sync0debug.cc @@ -1342,219 +1342,249 @@ sync_latch_meta_init() /* The latches should be ordered on latch_id_t. So that we can index directly into the vector to update and fetch meta-data. */ - LATCH_ADD(AUTOINC, SYNC_DICT_AUTOINC_MUTEX, autoinc_mutex_key); + LATCH_ADD_MUTEX(AUTOINC, SYNC_DICT_AUTOINC_MUTEX, autoinc_mutex_key); #if defined PFS_SKIP_BUFFER_MUTEX_RWLOCK || defined PFS_GROUP_BUFFER_SYNC - LATCH_ADD(BUF_BLOCK_MUTEX, SYNC_BUF_BLOCK, PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(BUF_BLOCK_MUTEX, SYNC_BUF_BLOCK, PFS_NOT_INSTRUMENTED); #else - LATCH_ADD(BUF_BLOCK_MUTEX, SYNC_BUF_BLOCK, buffer_block_mutex_key); + LATCH_ADD_MUTEX(BUF_BLOCK_MUTEX, SYNC_BUF_BLOCK, + buffer_block_mutex_key); #endif /* PFS_SKIP_BUFFER_MUTEX_RWLOCK || PFS_GROUP_BUFFER_SYNC */ - LATCH_ADD(BUF_POOL, SYNC_BUF_POOL, buf_pool_mutex_key); + LATCH_ADD_MUTEX(BUF_POOL, SYNC_BUF_POOL, buf_pool_mutex_key); - LATCH_ADD(BUF_POOL_ZIP, SYNC_BUF_BLOCK, buf_pool_zip_mutex_key); + LATCH_ADD_MUTEX(BUF_POOL_ZIP, SYNC_BUF_BLOCK, buf_pool_zip_mutex_key); - LATCH_ADD(CACHE_LAST_READ, SYNC_TRX_I_S_LAST_READ, - cache_last_read_mutex_key); + LATCH_ADD_MUTEX(CACHE_LAST_READ, SYNC_TRX_I_S_LAST_READ, + cache_last_read_mutex_key); - LATCH_ADD(DICT_FOREIGN_ERR, SYNC_NO_ORDER_CHECK, - dict_foreign_err_mutex_key); + LATCH_ADD_MUTEX(DICT_FOREIGN_ERR, SYNC_NO_ORDER_CHECK, + dict_foreign_err_mutex_key); - LATCH_ADD(DICT_SYS, SYNC_DICT, dict_sys_mutex_key); + LATCH_ADD_MUTEX(DICT_SYS, SYNC_DICT, dict_sys_mutex_key); - LATCH_ADD(FILE_FORMAT_MAX, SYNC_FILE_FORMAT_TAG, - file_format_max_mutex_key); + LATCH_ADD_MUTEX(FILE_FORMAT_MAX, SYNC_FILE_FORMAT_TAG, + file_format_max_mutex_key); - LATCH_ADD(FIL_SYSTEM, SYNC_ANY_LATCH, fil_system_mutex_key); + LATCH_ADD_MUTEX(FIL_SYSTEM, SYNC_ANY_LATCH, fil_system_mutex_key); - LATCH_ADD(FLUSH_LIST, SYNC_BUF_FLUSH_LIST, flush_list_mutex_key); + LATCH_ADD_MUTEX(FLUSH_LIST, SYNC_BUF_FLUSH_LIST, flush_list_mutex_key); - LATCH_ADD(FTS_BG_THREADS, SYNC_FTS_BG_THREADS, - fts_bg_threads_mutex_key); + LATCH_ADD_MUTEX(FTS_BG_THREADS, SYNC_FTS_BG_THREADS, + fts_bg_threads_mutex_key); - LATCH_ADD(FTS_DELETE, SYNC_FTS_OPTIMIZE, fts_delete_mutex_key); + LATCH_ADD_MUTEX(FTS_DELETE, SYNC_FTS_OPTIMIZE, fts_delete_mutex_key); - LATCH_ADD(FTS_OPTIMIZE, SYNC_FTS_OPTIMIZE, fts_optimize_mutex_key); + LATCH_ADD_MUTEX(FTS_OPTIMIZE, SYNC_FTS_OPTIMIZE, + fts_optimize_mutex_key); - LATCH_ADD(FTS_DOC_ID, SYNC_FTS_OPTIMIZE, fts_doc_id_mutex_key); + LATCH_ADD_MUTEX(FTS_DOC_ID, SYNC_FTS_OPTIMIZE, fts_doc_id_mutex_key); - LATCH_ADD(FTS_PLL_TOKENIZE, SYNC_FTS_TOKENIZE, - fts_pll_tokenize_mutex_key); + LATCH_ADD_MUTEX(FTS_PLL_TOKENIZE, SYNC_FTS_TOKENIZE, + fts_pll_tokenize_mutex_key); - LATCH_ADD(HASH_TABLE_MUTEX, SYNC_BUF_PAGE_HASH, hash_table_mutex_key); + LATCH_ADD_MUTEX(HASH_TABLE_MUTEX, SYNC_BUF_PAGE_HASH, + hash_table_mutex_key); - LATCH_ADD(IBUF_BITMAP, SYNC_IBUF_BITMAP_MUTEX, ibuf_bitmap_mutex_key); + LATCH_ADD_MUTEX(IBUF_BITMAP, SYNC_IBUF_BITMAP_MUTEX, + ibuf_bitmap_mutex_key); - LATCH_ADD(IBUF, SYNC_IBUF_MUTEX, ibuf_mutex_key); + LATCH_ADD_MUTEX(IBUF, SYNC_IBUF_MUTEX, ibuf_mutex_key); - LATCH_ADD(IBUF_PESSIMISTIC_INSERT, SYNC_IBUF_PESS_INSERT_MUTEX, - ibuf_pessimistic_insert_mutex_key); + LATCH_ADD_MUTEX(IBUF_PESSIMISTIC_INSERT, SYNC_IBUF_PESS_INSERT_MUTEX, + ibuf_pessimistic_insert_mutex_key); - LATCH_ADD(LOG_SYS, SYNC_LOG, log_sys_mutex_key); + LATCH_ADD_MUTEX(LOG_SYS, SYNC_LOG, log_sys_mutex_key); - LATCH_ADD(LOG_WRITE, SYNC_LOG_WRITE, log_sys_write_mutex_key); + LATCH_ADD_MUTEX(LOG_WRITE, SYNC_LOG_WRITE, log_sys_write_mutex_key); - LATCH_ADD(LOG_FLUSH_ORDER, SYNC_LOG_FLUSH_ORDER, - log_flush_order_mutex_key); + LATCH_ADD_MUTEX(LOG_FLUSH_ORDER, SYNC_LOG_FLUSH_ORDER, + log_flush_order_mutex_key); - LATCH_ADD(MUTEX_LIST, SYNC_NO_ORDER_CHECK, mutex_list_mutex_key); + LATCH_ADD_MUTEX(MUTEX_LIST, SYNC_NO_ORDER_CHECK, mutex_list_mutex_key); - LATCH_ADD(PAGE_CLEANER, SYNC_PAGE_CLEANER, page_cleaner_mutex_key); + LATCH_ADD_MUTEX(PAGE_CLEANER, SYNC_PAGE_CLEANER, + page_cleaner_mutex_key); - LATCH_ADD(PURGE_SYS_PQ, SYNC_PURGE_QUEUE, purge_sys_pq_mutex_key); + LATCH_ADD_MUTEX(PURGE_SYS_PQ, SYNC_PURGE_QUEUE, + purge_sys_pq_mutex_key); - LATCH_ADD(RECALC_POOL, SYNC_STATS_AUTO_RECALC, - recalc_pool_mutex_key); + LATCH_ADD_MUTEX(RECALC_POOL, SYNC_STATS_AUTO_RECALC, + recalc_pool_mutex_key); - LATCH_ADD(RECV_SYS, SYNC_RECV, recv_sys_mutex_key); + LATCH_ADD_MUTEX(RECV_SYS, SYNC_RECV, recv_sys_mutex_key); - LATCH_ADD(RECV_WRITER, SYNC_RECV_WRITER, recv_writer_mutex_key); + LATCH_ADD_MUTEX(RECV_WRITER, SYNC_RECV_WRITER, recv_writer_mutex_key); - LATCH_ADD(REDO_RSEG, SYNC_REDO_RSEG, redo_rseg_mutex_key); + LATCH_ADD_MUTEX(REDO_RSEG, SYNC_REDO_RSEG, redo_rseg_mutex_key); - LATCH_ADD(NOREDO_RSEG, SYNC_NOREDO_RSEG, noredo_rseg_mutex_key); + LATCH_ADD_MUTEX(NOREDO_RSEG, SYNC_NOREDO_RSEG, noredo_rseg_mutex_key); #ifdef UNIV_DEBUG /* Mutex names starting with '.' are not tracked. They are assumed to be diagnostic mutexes used in debugging. */ latch_meta[LATCH_ID_RW_LOCK_DEBUG] = - LATCH_ADD(RW_LOCK_DEBUG, + LATCH_ADD_MUTEX(RW_LOCK_DEBUG, SYNC_NO_ORDER_CHECK, rw_lock_debug_mutex_key); #endif /* UNIV_DEBUG */ - LATCH_ADD(RTR_SSN_MUTEX, SYNC_ANY_LATCH, rtr_ssn_mutex_key); + LATCH_ADD_MUTEX(RTR_SSN_MUTEX, SYNC_ANY_LATCH, rtr_ssn_mutex_key); - LATCH_ADD(RTR_ACTIVE_MUTEX, SYNC_ANY_LATCH, rtr_active_mutex_key); + LATCH_ADD_MUTEX(RTR_ACTIVE_MUTEX, SYNC_ANY_LATCH, + rtr_active_mutex_key); - LATCH_ADD(RTR_MATCH_MUTEX, SYNC_ANY_LATCH, rtr_match_mutex_key); + LATCH_ADD_MUTEX(RTR_MATCH_MUTEX, SYNC_ANY_LATCH, rtr_match_mutex_key); - LATCH_ADD(RTR_PATH_MUTEX, SYNC_ANY_LATCH, rtr_path_mutex_key); + LATCH_ADD_MUTEX(RTR_PATH_MUTEX, SYNC_ANY_LATCH, rtr_path_mutex_key); - LATCH_ADD(RW_LOCK_LIST, SYNC_NO_ORDER_CHECK, rw_lock_list_mutex_key); + LATCH_ADD_MUTEX(RW_LOCK_LIST, SYNC_NO_ORDER_CHECK, + rw_lock_list_mutex_key); - LATCH_ADD(RW_LOCK_MUTEX, SYNC_NO_ORDER_CHECK, rw_lock_mutex_key); + LATCH_ADD_MUTEX(RW_LOCK_MUTEX, SYNC_NO_ORDER_CHECK, rw_lock_mutex_key); - LATCH_ADD(SRV_DICT_TMPFILE, SYNC_DICT_OPERATION, - srv_dict_tmpfile_mutex_key); + LATCH_ADD_MUTEX(SRV_DICT_TMPFILE, SYNC_DICT_OPERATION, + srv_dict_tmpfile_mutex_key); - LATCH_ADD(SRV_INNODB_MONITOR, SYNC_NO_ORDER_CHECK, - srv_innodb_monitor_mutex_key); + LATCH_ADD_MUTEX(SRV_INNODB_MONITOR, SYNC_NO_ORDER_CHECK, + srv_innodb_monitor_mutex_key); - LATCH_ADD(SRV_MISC_TMPFILE, SYNC_ANY_LATCH, - srv_misc_tmpfile_mutex_key); + LATCH_ADD_MUTEX(SRV_MISC_TMPFILE, SYNC_ANY_LATCH, + srv_misc_tmpfile_mutex_key); - LATCH_ADD(SRV_MONITOR_FILE, SYNC_NO_ORDER_CHECK, - srv_monitor_file_mutex_key); + LATCH_ADD_MUTEX(SRV_MONITOR_FILE, SYNC_NO_ORDER_CHECK, + srv_monitor_file_mutex_key); #ifdef UNIV_DEBUG - LATCH_ADD(SYNC_THREAD, SYNC_NO_ORDER_CHECK, sync_thread_mutex_key); + LATCH_ADD_MUTEX(SYNC_THREAD, SYNC_NO_ORDER_CHECK, + sync_thread_mutex_key); #endif /* UNIV_DEBUG */ - LATCH_ADD(BUF_DBLWR, SYNC_DOUBLEWRITE, buf_dblwr_mutex_key); + LATCH_ADD_MUTEX(BUF_DBLWR, SYNC_DOUBLEWRITE, buf_dblwr_mutex_key); - LATCH_ADD(TRX_UNDO, SYNC_TRX_UNDO, trx_undo_mutex_key); + LATCH_ADD_MUTEX(TRX_UNDO, SYNC_TRX_UNDO, trx_undo_mutex_key); - LATCH_ADD(TRX_POOL, SYNC_POOL, trx_pool_mutex_key); + LATCH_ADD_MUTEX(TRX_POOL, SYNC_POOL, trx_pool_mutex_key); - LATCH_ADD(TRX_POOL_MANAGER, SYNC_POOL_MANAGER, - trx_pool_manager_mutex_key); + LATCH_ADD_MUTEX(TRX_POOL_MANAGER, SYNC_POOL_MANAGER, + trx_pool_manager_mutex_key); - LATCH_ADD(TRX, SYNC_TRX, trx_mutex_key); + LATCH_ADD_MUTEX(TRX, SYNC_TRX, trx_mutex_key); - LATCH_ADD(LOCK_SYS, SYNC_LOCK_SYS, lock_mutex_key); + LATCH_ADD_MUTEX(LOCK_SYS, SYNC_LOCK_SYS, lock_mutex_key); - LATCH_ADD(LOCK_SYS_WAIT, SYNC_LOCK_WAIT_SYS, lock_wait_mutex_key); + LATCH_ADD_MUTEX(LOCK_SYS_WAIT, SYNC_LOCK_WAIT_SYS, + lock_wait_mutex_key); - LATCH_ADD(TRX_SYS, SYNC_TRX_SYS, trx_sys_mutex_key); + LATCH_ADD_MUTEX(TRX_SYS, SYNC_TRX_SYS, trx_sys_mutex_key); - LATCH_ADD(SRV_SYS, SYNC_THREADS, srv_sys_mutex_key); + LATCH_ADD_MUTEX(SRV_SYS, SYNC_THREADS, srv_sys_mutex_key); - LATCH_ADD(SRV_SYS_TASKS, SYNC_ANY_LATCH, srv_threads_mutex_key); + LATCH_ADD_MUTEX(SRV_SYS_TASKS, SYNC_ANY_LATCH, srv_threads_mutex_key); - LATCH_ADD(PAGE_ZIP_STAT_PER_INDEX, SYNC_ANY_LATCH, - page_zip_stat_per_index_mutex_key); + LATCH_ADD_MUTEX(PAGE_ZIP_STAT_PER_INDEX, SYNC_ANY_LATCH, + page_zip_stat_per_index_mutex_key); #ifndef PFS_SKIP_EVENT_MUTEX - LATCH_ADD(EVENT_MANAGER, SYNC_NO_ORDER_CHECK, event_manager_mutex_key); + LATCH_ADD_MUTEX(EVENT_MANAGER, SYNC_NO_ORDER_CHECK, + event_manager_mutex_key); #else - LATCH_ADD(EVENT_MANAGER, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(EVENT_MANAGER, SYNC_NO_ORDER_CHECK, + PFS_NOT_INSTRUMENTED); #endif /* !PFS_SKIP_EVENT_MUTEX */ - LATCH_ADD(EVENT_MUTEX, SYNC_NO_ORDER_CHECK, event_mutex_key); + LATCH_ADD_MUTEX(EVENT_MUTEX, SYNC_NO_ORDER_CHECK, event_mutex_key); - LATCH_ADD(SYNC_ARRAY_MUTEX, SYNC_NO_ORDER_CHECK, - sync_array_mutex_key); + LATCH_ADD_MUTEX(SYNC_ARRAY_MUTEX, SYNC_NO_ORDER_CHECK, + sync_array_mutex_key); - LATCH_ADD(THREAD_MUTEX, SYNC_NO_ORDER_CHECK, thread_mutex_key); + LATCH_ADD_MUTEX(THREAD_MUTEX, SYNC_NO_ORDER_CHECK, thread_mutex_key); - LATCH_ADD(ZIP_PAD_MUTEX, SYNC_NO_ORDER_CHECK, zip_pad_mutex_key); + LATCH_ADD_MUTEX(ZIP_PAD_MUTEX, SYNC_NO_ORDER_CHECK, zip_pad_mutex_key); - LATCH_ADD(OS_AIO_READ_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(OS_AIO_READ_MUTEX, SYNC_NO_ORDER_CHECK, + PFS_NOT_INSTRUMENTED); - LATCH_ADD(OS_AIO_WRITE_MUTEX, SYNC_NO_ORDER_CHECK, - PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(OS_AIO_WRITE_MUTEX, SYNC_NO_ORDER_CHECK, + PFS_NOT_INSTRUMENTED); - LATCH_ADD(OS_AIO_LOG_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(OS_AIO_LOG_MUTEX, SYNC_NO_ORDER_CHECK, + PFS_NOT_INSTRUMENTED); - LATCH_ADD(OS_AIO_IBUF_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(OS_AIO_IBUF_MUTEX, SYNC_NO_ORDER_CHECK, + PFS_NOT_INSTRUMENTED); - LATCH_ADD(OS_AIO_SYNC_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(OS_AIO_SYNC_MUTEX, SYNC_NO_ORDER_CHECK, + PFS_NOT_INSTRUMENTED); - LATCH_ADD(ROW_DROP_LIST, SYNC_NO_ORDER_CHECK, row_drop_list_mutex_key); + LATCH_ADD_MUTEX(ROW_DROP_LIST, SYNC_NO_ORDER_CHECK, + row_drop_list_mutex_key); - LATCH_ADD(INDEX_ONLINE_LOG, SYNC_INDEX_ONLINE_LOG, - index_online_log_key); + LATCH_ADD_MUTEX(INDEX_ONLINE_LOG, SYNC_INDEX_ONLINE_LOG, + index_online_log_key); - LATCH_ADD(WORK_QUEUE, SYNC_WORK_QUEUE, PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(WORK_QUEUE, SYNC_WORK_QUEUE, PFS_NOT_INSTRUMENTED); // Add the RW locks - LATCH_ADD(BTR_SEARCH, SYNC_SEARCH_SYS, btr_search_latch_key); + LATCH_ADD_RWLOCK(BTR_SEARCH, SYNC_SEARCH_SYS, btr_search_latch_key); - LATCH_ADD(BUF_BLOCK_LOCK, SYNC_LEVEL_VARYING, buf_block_lock_key); + LATCH_ADD_RWLOCK(BUF_BLOCK_LOCK, SYNC_LEVEL_VARYING, + buf_block_lock_key); #ifdef UNIV_DEBUG - LATCH_ADD(BUF_BLOCK_DEBUG, SYNC_NO_ORDER_CHECK, - buf_block_debug_latch_key); + LATCH_ADD_RWLOCK(BUF_BLOCK_DEBUG, SYNC_NO_ORDER_CHECK, + buf_block_debug_latch_key); #endif /* UNIV_DEBUG */ - LATCH_ADD(DICT_OPERATION, SYNC_DICT, dict_operation_lock_key); + LATCH_ADD_RWLOCK(DICT_OPERATION, SYNC_DICT, dict_operation_lock_key); - LATCH_ADD(CHECKPOINT, SYNC_NO_ORDER_CHECK, checkpoint_lock_key); + LATCH_ADD_RWLOCK(CHECKPOINT, SYNC_NO_ORDER_CHECK, checkpoint_lock_key); - LATCH_ADD(FIL_SPACE, SYNC_FSP, fil_space_latch_key); + LATCH_ADD_RWLOCK(FIL_SPACE, SYNC_FSP, fil_space_latch_key); - LATCH_ADD(FTS_CACHE, SYNC_FTS_CACHE, fts_cache_rw_lock_key); + LATCH_ADD_RWLOCK(FTS_CACHE, SYNC_FTS_CACHE, fts_cache_rw_lock_key); - LATCH_ADD(FTS_CACHE_INIT, SYNC_FTS_CACHE_INIT, - fts_cache_init_rw_lock_key); + LATCH_ADD_RWLOCK(FTS_CACHE_INIT, SYNC_FTS_CACHE_INIT, + fts_cache_init_rw_lock_key); - LATCH_ADD(TRX_I_S_CACHE, SYNC_TRX_I_S_RWLOCK, trx_i_s_cache_lock_key); + LATCH_ADD_RWLOCK(TRX_I_S_CACHE, SYNC_TRX_I_S_RWLOCK, + trx_i_s_cache_lock_key); - LATCH_ADD(TRX_PURGE, SYNC_PURGE_LATCH, trx_purge_latch_key); + LATCH_ADD_RWLOCK(TRX_PURGE, SYNC_PURGE_LATCH, trx_purge_latch_key); - LATCH_ADD(IBUF_INDEX_TREE, SYNC_IBUF_INDEX_TREE, - index_tree_rw_lock_key); + LATCH_ADD_RWLOCK(IBUF_INDEX_TREE, SYNC_IBUF_INDEX_TREE, + index_tree_rw_lock_key); - LATCH_ADD(INDEX_TREE, SYNC_INDEX_TREE, index_tree_rw_lock_key); + LATCH_ADD_RWLOCK(INDEX_TREE, SYNC_INDEX_TREE, index_tree_rw_lock_key); - LATCH_ADD(DICT_TABLE_STATS, SYNC_INDEX_TREE, dict_table_stats_key); + LATCH_ADD_RWLOCK(DICT_TABLE_STATS, SYNC_INDEX_TREE, + dict_table_stats_key); - LATCH_ADD(HASH_TABLE_RW_LOCK, SYNC_BUF_PAGE_HASH, + LATCH_ADD_RWLOCK(HASH_TABLE_RW_LOCK, SYNC_BUF_PAGE_HASH, hash_table_locks_key); - LATCH_ADD(SYNC_DEBUG_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(SYNC_DEBUG_MUTEX, SYNC_NO_ORDER_CHECK, + PFS_NOT_INSTRUMENTED); /* JAN: TODO: Add PFS instrumentation */ - LATCH_ADD(SCRUB_STAT_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); - LATCH_ADD(DEFRAGMENT_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); - LATCH_ADD(BTR_DEFRAGMENT_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); - LATCH_ADD(MTFLUSH_THREAD_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); - LATCH_ADD(MTFLUSH_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); - LATCH_ADD(FIL_CRYPT_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); - LATCH_ADD(FIL_CRYPT_STAT_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); - LATCH_ADD(FIL_CRYPT_DATA_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); - LATCH_ADD(FIL_CRYPT_THREADS_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(SCRUB_STAT_MUTEX, SYNC_NO_ORDER_CHECK, + PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(DEFRAGMENT_MUTEX, SYNC_NO_ORDER_CHECK, + PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(BTR_DEFRAGMENT_MUTEX, SYNC_NO_ORDER_CHECK, + PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(MTFLUSH_THREAD_MUTEX, SYNC_NO_ORDER_CHECK, + PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(MTFLUSH_MUTEX, SYNC_NO_ORDER_CHECK, + PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(FIL_CRYPT_MUTEX, SYNC_NO_ORDER_CHECK, + PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(FIL_CRYPT_STAT_MUTEX, SYNC_NO_ORDER_CHECK, + PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(FIL_CRYPT_DATA_MUTEX, SYNC_NO_ORDER_CHECK, + PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(FIL_CRYPT_THREADS_MUTEX, SYNC_NO_ORDER_CHECK, + PFS_NOT_INSTRUMENTED); latch_id_t id = LATCH_ID_NONE; -- cgit v1.2.1 From 6b63e4ad4a4dbea4b5e9060c437096d082c73cf3 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Wed, 28 Dec 2016 17:44:16 +0530 Subject: Bug #23219499 CONCURRENT UPDATE DOESN'T APPLY IN VIRTUAL INDEX DURING TABLE REBUILD Problem: ======= Concurrent update dml statement doesn't reflect in virtual index during inplace table rebuild. It results mismatch value in virutal index and clustered index. Deleting the table content tries to search the mismatch value in virtual index but it can't find the value. During log update apply phase, virtual information is being ignored while constructing the new entry. Solution: ========= In row_log_update_apply phase, build the entry with virtual column information. So that it can reflect in newly constructed virtual index. Reviewed-by: Jimmy Yang RB: 14974 --- storage/innobase/handler/handler0alter.cc | 38 +++++++++++++++++++++++-------- storage/innobase/row/row0log.cc | 5 ++-- 2 files changed, 32 insertions(+), 11 deletions(-) (limited to 'storage') diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 90a866b4714..6f931d8d5a6 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -6455,15 +6455,6 @@ ok_exit: ctx->add_autoinc, ctx->sequence, ctx->skip_pk_sort, ctx->m_stage, add_v, eval_table); - if (s_templ) { - ut_ad(ctx->need_rebuild() || ctx->num_to_add_vcol > 0 - || rebuild_templ); - dict_free_vc_templ(s_templ); - UT_DELETE(s_templ); - - ctx->new_table->vc_templ = old_templ; - } - #ifndef DBUG_OFF oom: #endif /* !DBUG_OFF */ @@ -6479,6 +6470,15 @@ oom: onlineddl_rowlog_pct_used = 0; onlineddl_pct_progress = 0; + if (s_templ) { + ut_ad(ctx->need_rebuild() || ctx->num_to_add_vcol > 0 + || rebuild_templ); + dict_free_vc_templ(s_templ); + UT_DELETE(s_templ); + + ctx->new_table->vc_templ = old_templ; + } + DEBUG_SYNC_C("inplace_after_index_build"); DBUG_EXECUTE_IF("create_index_fail", @@ -7732,11 +7732,31 @@ commit_try_rebuild( if (ctx->online) { DEBUG_SYNC_C("row_log_table_apply2_before"); + dict_vcol_templ_t* s_templ = NULL; + + if (ctx->new_table->n_v_cols > 0) { + s_templ = UT_NEW_NOKEY( + dict_vcol_templ_t()); + s_templ->vtempl = NULL; + + innobase_build_v_templ( + altered_table, ctx->new_table, s_templ, + NULL, true); + ctx->new_table->vc_templ = s_templ; + } + error = row_log_table_apply( ctx->thr, user_table, altered_table, static_cast( ha_alter_info->handler_ctx)->m_stage); + if (s_templ) { + ut_ad(ctx->need_rebuild()); + dict_free_vc_templ(s_templ); + UT_DELETE(s_templ); + ctx->new_table->vc_templ = NULL; + } + ulint err_key = thr_get_trx(ctx->thr)->error_key_num; switch (error) { diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc index 776d1891e6b..36ad6cdd3d9 100644 --- a/storage/innobase/row/row0log.cc +++ b/storage/innobase/row/row0log.cc @@ -2195,8 +2195,9 @@ func_exit_committed: goto func_exit_committed; } - dtuple_t* entry = row_build_index_entry( - row, NULL, index, heap); + /** It allows to create tuple with virtual column information. */ + dtuple_t* entry = row_build_index_entry_low( + row, NULL, index, heap, ROW_BUILD_FOR_INSERT); upd_t* update = row_upd_build_difference_binary( index, entry, btr_pcur_get_rec(&pcur), cur_offsets, false, NULL, heap, dup->table); -- cgit v1.2.1 From 797db28d53eb4b568cc87e1db6b762709c75e9f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 26 Apr 2017 07:31:54 +0300 Subject: Remove the unused declaration fsp_is_temporary() --- storage/innobase/include/fil0fil.h | 9 --------- 1 file changed, 9 deletions(-) (limited to 'storage') diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 4583b81c983..666c42a9628 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -552,15 +552,6 @@ void fil_space_set_imported( ulint id); -#ifdef UNIV_DEBUG -/** Determine if a tablespace is temporary. -@param[in] id tablespace identifier -@return whether it is a temporary tablespace */ -bool -fsp_is_temporary(ulint id) -MY_ATTRIBUTE((warn_unused_result, pure)); -#endif /* UNIV_DEBUG */ - /** Append a file to the chain of files of a space. @param[in] name file name of a file that is not open @param[in] size file size in entire database blocks -- cgit v1.2.1 From 6264e892cf3e81f4cdcb13eb0e13955dbf3fdfc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 26 Apr 2017 10:11:46 +0300 Subject: Remove redundant initialization of global InnoDB variables Also, remove the unused global variable srv_priority_boost. --- storage/innobase/include/srv0srv.h | 1 - storage/innobase/srv/srv0srv.cc | 283 ++++++++++++++++++++----------------- 2 files changed, 154 insertions(+), 130 deletions(-) (limited to 'storage') diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 8e3659a50e1..6803047ed91 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -516,7 +516,6 @@ extern ulong srv_n_spin_wait_rounds; extern ulong srv_n_free_tickets_to_enter; extern ulong srv_thread_sleep_delay; extern uint srv_spin_wait_delay; -extern ibool srv_priority_boost; extern ulint srv_truncated_status_writes; /** Number of initialized rollback segments for persistent undo log */ diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index f63c930865f..4e1157fdc7c 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -87,7 +87,7 @@ UNIV_INTERN ulong srv_fatal_semaphore_wait_threshold = DEFAULT_SRV_FATAL_SEMAPH /* How much data manipulation language (DML) statements need to be delayed, in microseconds, in order to reduce the lagging of the purge thread. */ -ulint srv_dml_needed_delay = 0; +ulint srv_dml_needed_delay; bool srv_monitor_active; bool srv_error_monitor_active; @@ -151,7 +151,7 @@ my_bool srv_read_only_mode; dictionary tables are in the system tablespace 0 */ my_bool srv_file_per_table; /** The file format to use on new *.ibd files. */ -ulint srv_file_format = 0; +ulint srv_file_format; /** Whether to check file format during startup. A value of UNIV_FORMAT_MAX + 1 means no checking ie. FALSE. The default is to set it to the highest format we support. */ @@ -178,17 +178,17 @@ use simulated aio we build below with threads. Currently we support native aio on windows and linux */ my_bool srv_use_native_aio; my_bool srv_numa_interleave; -/* If this flag is TRUE, then we will use fallocate(PUCH_HOLE) -to the pages */ -UNIV_INTERN my_bool srv_use_trim; -/* If this flag is TRUE, then we disable doublewrite buffer */ -UNIV_INTERN my_bool srv_use_atomic_writes; -/* If this flag IS TRUE, then we use this algorithm for page compressing the pages */ -UNIV_INTERN ulong innodb_compression_algorithm = PAGE_ZLIB_ALGORITHM; -/* Number of threads used for multi-threaded flush */ -UNIV_INTERN long srv_mtflush_threads = MTFLUSH_DEFAULT_WORKER; -/* If this flag is TRUE, then we will use multi threaded flush. */ -UNIV_INTERN my_bool srv_use_mtflush = FALSE; +/** innodb_use_trim; whether to use fallocate(PUNCH_HOLE) with +page_compression */ +my_bool srv_use_trim; +/** copy of innodb_use_atomic_writes; @see innobase_init() */ +my_bool srv_use_atomic_writes; +/** innodb_compression_algorithm; used with page compression */ +ulong innodb_compression_algorithm; +/** innodb_mtflush_threads; number of threads used for multi-threaded flush */ +long srv_mtflush_threads; +/** innodb_use_mtflush; whether to use multi threaded flush. */ +my_bool srv_use_mtflush; #ifdef UNIV_DEBUG /** Used by SET GLOBAL innodb_master_thread_disabled_debug = X. */ @@ -215,22 +215,27 @@ During startup, this is in bytes, and later converted to pages. */ ib_uint64_t srv_log_file_size; /** The value of the startup parameter innodb_log_file_size */ ib_uint64_t srv_log_file_size_requested; -/* size in database pages */ -ulint srv_log_buffer_size = ULINT_MAX; -ulong srv_flush_log_at_trx_commit = 1; -uint srv_flush_log_at_timeout = 1; -ulong srv_page_size = UNIV_PAGE_SIZE_DEF; -ulong srv_page_size_shift = UNIV_PAGE_SIZE_SHIFT_DEF; -ulong srv_log_write_ahead_size = 0; +/** copy of innodb_log_buffer_size, but in database pages */ +ulint srv_log_buffer_size; +/** innodb_flush_log_at_trx_commit */ +ulong srv_flush_log_at_trx_commit; +/** innodb_flush_log_at_timeout */ +uint srv_flush_log_at_timeout; +/** innodb_page_size */ +ulong srv_page_size; +/** log2 of innodb_page_size; @see innobase_init() */ +ulong srv_page_size_shift; +/** innodb_log_write_ahead_size */ +ulong srv_log_write_ahead_size; page_size_t univ_page_size(0, 0, false); -/* Try to flush dirty pages so as to avoid IO bursts at -the checkpoints. */ -char srv_adaptive_flushing = TRUE; +/** innodb_adaptive_flushing; try to flush dirty pages so as to avoid +IO bursts at the checkpoints. */ +my_bool srv_adaptive_flushing; -/* Allow IO bursts at the checkpoints ignoring io_capacity setting. */ -my_bool srv_flush_sync = TRUE; +/** innodb_flush_sync; whether to ignore io_capacity at log checkpoints */ +my_bool srv_flush_sync; /** Maximum number of times allowed to conditionally acquire mutex before switching to blocking wait on the mutex */ @@ -246,93 +251,100 @@ with mutex_enter(), which will wait until it gets the mutex. */ UNIV_INTERN os_event_t srv_allow_writes_event; #endif /* WITH_INNODB_DISALLOW_WRITES */ -/** Requested size in bytes */ -ulint srv_buf_pool_size = ULINT_MAX; +/** copy of innodb_buffer_pool_size */ +ulint srv_buf_pool_size; const ulint srv_buf_pool_min_size = 5 * 1024 * 1024; /** Default pool size in bytes */ const ulint srv_buf_pool_def_size = 128 * 1024 * 1024; /** Requested buffer pool chunk size. Each buffer pool instance consists of one or more chunks. */ ulong srv_buf_pool_chunk_unit; -/** Requested number of buffer pool instances */ +/** innodb_buffer_pool_instances (0 is interpreted as 1) */ ulong srv_buf_pool_instances; -/** Default number of buffer pool instances */ +/** Default value of innodb_buffer_pool_instances */ const ulong srv_buf_pool_instances_default = 0; -/** Number of locks to protect buf_pool->page_hash */ +/** innodb_page_hash_locks (a debug-only parameter); +number of locks to protect buf_pool->page_hash */ ulong srv_n_page_hash_locks = 16; -/** Scan depth for LRU flush batch i.e.: number of blocks scanned*/ -ulong srv_LRU_scan_depth = 1024; -/** Whether or not to flush neighbors of a block */ -ulong srv_flush_neighbors = 1; +/** innodb_lru_scan_depth; number of blocks scanned in LRU flush batch */ +ulong srv_LRU_scan_depth; +/** innodb_flush_neighbors; whether or not to flush neighbors of a block */ +ulong srv_flush_neighbors; /** Previously requested size */ -ulint srv_buf_pool_old_size = 0; +ulint srv_buf_pool_old_size; /** Current size as scaling factor for the other components */ -ulint srv_buf_pool_base_size = 0; +ulint srv_buf_pool_base_size; /** Current size in bytes */ -ulint srv_buf_pool_curr_size = 0; +ulint srv_buf_pool_curr_size; /** Dump this % of each buffer pool during BP dump */ ulong srv_buf_pool_dump_pct; /** Lock table size in bytes */ ulint srv_lock_table_size = ULINT_MAX; -UNIV_INTERN ulong srv_idle_flush_pct = 100; +/** innodb_idle_flush_pct */ +ulong srv_idle_flush_pct; -/* This parameter is deprecated. Use srv_n_io_[read|write]_threads -instead. */ -ulint srv_n_read_io_threads = ULINT_MAX; -ulint srv_n_write_io_threads = ULINT_MAX; +/** copy of innodb_read_io_threads */ +ulint srv_n_read_io_threads; +/** copy of innodb_write_io_threads */ +ulint srv_n_write_io_threads; -/* Switch to enable random read ahead. */ -my_bool srv_random_read_ahead = FALSE; -/* User settable value of the number of pages that must be present +/** innodb_random_read_ahead */ +my_bool srv_random_read_ahead; +/** innodb_read_ahead_threshold; the number of pages that must be present in the buffer cache and accessed sequentially for InnoDB to trigger a readahead request. */ -ulong srv_read_ahead_threshold = 56; +ulong srv_read_ahead_threshold; -/** Maximum on-disk size of change buffer in terms of percentage -of the buffer pool. */ -uint srv_change_buffer_max_size = CHANGE_BUFFER_DEFAULT_SIZE; +/** innodb_change_buffer_max_size; maximum on-disk size of change +buffer in terms of percentage of the buffer pool. */ +uint srv_change_buffer_max_size; -char* srv_file_flush_method_str = NULL; +char* srv_file_flush_method_str; enum srv_flush_t srv_file_flush_method = IF_WIN(SRV_ALL_O_DIRECT_FSYNC,SRV_FSYNC); -ulint srv_max_n_open_files = 300; +/** copy of innodb_open_files, initialized by innobase_init() */ +ulint srv_max_n_open_files; -/* Number of IO operations per second the server can do */ -ulong srv_io_capacity = 200; -ulong srv_max_io_capacity = 400; +/** innodb_io_capacity */ +ulong srv_io_capacity; +/** innodb_io_capacity_max */ +ulong srv_max_io_capacity; -/* The number of page cleaner threads to use.*/ -ulong srv_n_page_cleaners = 4; +/** innodb_page_cleaners; the number of page cleaner threads */ +ulong srv_n_page_cleaners; /* The InnoDB main thread tries to keep the ratio of modified pages in the buffer pool to all database pages in the buffer pool smaller than the following number. But it is not guaranteed that the value stays below that during a time of heavy update/insert activity. */ -double srv_max_buf_pool_modified_pct = 75.0; -double srv_max_dirty_pages_pct_lwm = 0.0; +/** innodb_max_dirty_pages_pct */ +double srv_max_buf_pool_modified_pct; +/** innodb_max_dirty_pages_pct_lwm */ +double srv_max_dirty_pages_pct_lwm; -/* This is the percentage of log capacity at which adaptive flushing, -if enabled, will kick in. */ -UNIV_INTERN double srv_adaptive_flushing_lwm = 10.0; +/** innodb_adaptive_flushing_lwm; the percentage of log capacity at +which adaptive flushing, if enabled, will kick in. */ +double srv_adaptive_flushing_lwm; -/* Number of iterations over which adaptive flushing is averaged. */ -ulong srv_flushing_avg_loops = 30; +/** innodb_flushing_avg_loops; number of iterations over which +adaptive flushing is averaged */ +ulong srv_flushing_avg_loops; -/* The number of purge threads to use.*/ -ulong srv_n_purge_threads = 4; +/** innodb_purge_threads; the number of purge threads to use */ +ulong srv_n_purge_threads; -/* the number of pages to purge in one batch */ -ulong srv_purge_batch_size = 20; +/** innodb_purge_batch_size, in pages */ +ulong srv_purge_batch_size; -/* Internal setting for "innodb_stats_method". Decides how InnoDB treats +/** innodb_stats_method decides how InnoDB treats NULL value when collecting statistics. By default, it is set to SRV_STATS_NULLS_EQUAL(0), ie. all NULL value are treated equal */ -ulong srv_innodb_stats_method = SRV_STATS_NULLS_EQUAL; +ulong srv_innodb_stats_method; srv_stats_t srv_stats; @@ -345,60 +357,67 @@ by SELECT or mysqldump. When this is nonzero, we do not allow any user modifications to the data. */ ulong srv_force_recovery; -/** Print all user-level transactions deadlocks to mysqld stderr */ +/** innodb_print_all_deadlocks; whether to print all user-level +transactions deadlocks to the error log */ +my_bool srv_print_all_deadlocks; -my_bool srv_print_all_deadlocks = FALSE; +/** innodb_cmp_per_index_enabled; enable +INFORMATION_SCHEMA.innodb_cmp_per_index */ +my_bool srv_cmp_per_index_enabled; -/** Enable INFORMATION_SCHEMA.innodb_cmp_per_index */ -my_bool srv_cmp_per_index_enabled = FALSE; - -/* If the following is set to 1 then we do not run purge and insert buffer +/** innodb_fast_shutdown; if 1 then we do not run purge and insert buffer merge to completion before shutdown. If it is set to 2, do not even flush the buffer pool to data files at the shutdown: we effectively 'crash' InnoDB (but lose no committed transactions). */ uint srv_fast_shutdown; -/* Generate a innodb_status. file */ -ibool srv_innodb_status = FALSE; +/** copy of innodb_status_file; generate a innodb_status. file */ +ibool srv_innodb_status; -/* Optimize prefix index queries to skip cluster index lookup when possible */ -/* Enables or disables this prefix optimization. Disabled by default. */ -UNIV_INTERN my_bool srv_prefix_index_cluster_optimization = 0; +/** innodb_prefix_index_cluster_optimization; whether to optimize +prefix index queries to skip cluster index lookup when possible */ +my_bool srv_prefix_index_cluster_optimization; -/* When estimating number of different key values in an index, sample +/** innodb_stats_transient_sample_pages; +When estimating number of different key values in an index, sample this many index pages, there are 2 ways to calculate statistics: * persistent stats that are calculated by ANALYZE TABLE and saved in the innodb database. * quick transient stats, that are used if persistent stats for the given table/index are not found in the innodb database */ -unsigned long long srv_stats_transient_sample_pages = 8; -my_bool srv_stats_persistent = TRUE; -my_bool srv_stats_include_delete_marked = FALSE; -unsigned long long srv_stats_persistent_sample_pages = 20; -my_bool srv_stats_auto_recalc = TRUE; - -/* The number of rows modified before we calculate new statistics (default 0 -= current limits) */ -UNIV_INTERN unsigned long long srv_stats_modified_counter = 0; - -/* Enable traditional statistic calculation based on number of configured -pages default true. */ -UNIV_INTERN my_bool srv_stats_sample_traditional = TRUE; - -ibool srv_use_doublewrite_buf = TRUE; - -/** doublewrite buffer is 1MB is size i.e.: it can hold 128 16K pages. -The following parameter is the size of the buffer that is used for -batch flushing i.e.: LRU flushing and flush_list flushing. The rest -of the pages are used for single page flushing. */ -ulong srv_doublewrite_batch_size = 120; - -ulong srv_replication_delay = 0; - -/*-------------------------------------------*/ -UNIV_INTERN ulong srv_n_spin_wait_rounds = 15; +unsigned long long srv_stats_transient_sample_pages; +/** innodb_stats_persistent */ +my_bool srv_stats_persistent; +/** innodb_stats_include_delete_marked */ +my_bool srv_stats_include_delete_marked; +/** innodb_stats_persistent_sample_pages */ +unsigned long long srv_stats_persistent_sample_pages; +/** innodb_stats_auto_recalc */ +my_bool srv_stats_auto_recalc; + +/** innodb_stats_modified_counter; The number of rows modified before +we calculate new statistics (default 0 = current limits) */ +unsigned long long srv_stats_modified_counter; + +/** innodb_stats_traditional; enable traditional statistic calculation +based on number of configured pages */ +my_bool srv_stats_sample_traditional; + +/** copy of innodb_doublewrite */ +ibool srv_use_doublewrite_buf; + +/** innodb_doublewrite_batch_size (a debug parameter) specifies the +number of pages to use in LRU and flush_list batch flushing. +The rest of the doublewrite buffer is used for single-page flushing. */ +ulong srv_doublewrite_batch_size = 120; + +/** innodb_replication_delay */ +ulong srv_replication_delay; + +/** innodb_sync_spin_loops */ +ulong srv_n_spin_wait_rounds; +/** innodb_spin_wait_delay */ uint srv_spin_wait_delay; -ibool srv_priority_boost = TRUE; static ulint srv_n_rows_inserted_old; static ulint srv_n_rows_updated_old; @@ -425,23 +444,29 @@ UNIV_INTERN ib_uint64_t srv_index_page_decompressed; /* Defragmentation */ UNIV_INTERN my_bool srv_defragment; -UNIV_INTERN uint srv_defragment_n_pages = 7; +/** innodb_defragment_n_pages */ +UNIV_INTERN uint srv_defragment_n_pages; UNIV_INTERN uint srv_defragment_stats_accuracy; -UNIV_INTERN uint srv_defragment_fill_factor_n_recs = 20; -UNIV_INTERN double srv_defragment_fill_factor = 0.9; -UNIV_INTERN uint srv_defragment_frequency = - SRV_DEFRAGMENT_FREQUENCY_DEFAULT; +/** innodb_defragment_fill_factor_n_recs */ +UNIV_INTERN uint srv_defragment_fill_factor_n_recs; +/** innodb_defragment_fill_factor */ +UNIV_INTERN double srv_defragment_fill_factor; +/** innodb_defragment_frequency */ +UNIV_INTERN uint srv_defragment_frequency; +/** derived from innodb_defragment_frequency; +@see innodb_defragment_frequency_update() */ UNIV_INTERN ulonglong srv_defragment_interval; /* Set the following to 0 if you want InnoDB to write messages on -stderr on startup/shutdown. */ -ibool srv_print_verbose_log = TRUE; -my_bool srv_print_innodb_monitor = FALSE; -my_bool srv_print_innodb_lock_monitor = FALSE; -my_bool srv_print_innodb_tablespace_monitor = FALSE; -my_bool srv_print_innodb_table_monitor = FALSE; -/** If this flag is set tables without primary key are not allowed */ -my_bool srv_force_primary_key = FALSE; +stderr on startup/shutdown. Not enabled on the embedded server. */ +ibool srv_print_verbose_log; +my_bool srv_print_innodb_monitor; +my_bool srv_print_innodb_lock_monitor; +my_bool srv_print_innodb_tablespace_monitor; +my_bool srv_print_innodb_table_monitor; +/** innodb_force_primary_key; whether to disallow CREATE TABLE without +PRIMARY KEY */ +my_bool srv_force_primary_key; /* Array of English strings describing the current state of an i/o handler thread */ @@ -474,19 +499,19 @@ ib_mutex_t srv_misc_tmpfile_mutex; /** Temporary file for miscellanous diagnostic output */ FILE* srv_misc_tmpfile; -static ulint srv_main_thread_process_no = 0; -static ulint srv_main_thread_id = 0; +static ulint srv_main_thread_process_no; +static ulint srv_main_thread_id; /* The following counts are used by the srv_master_thread. */ /** Iterations of the loop bounded by 'srv_active' label. */ -static ulint srv_main_active_loops = 0; +static ulint srv_main_active_loops; /** Iterations of the loop bounded by the 'srv_idle' label. */ -static ulint srv_main_idle_loops = 0; +static ulint srv_main_idle_loops; /** Iterations of the loop bounded by the 'srv_shutdown' label. */ -static ulint srv_main_shutdown_loops = 0; +static ulint srv_main_shutdown_loops; /** Log writes involving flush. */ -static ulint srv_log_writes_and_flush = 0; +static ulint srv_log_writes_and_flush; /* This is only ever touched by the master thread. It records the time when the last flush of log file has happened. The master @@ -507,10 +532,10 @@ current_time % 5 != 0. */ # define SRV_MASTER_DICT_LRU_INTERVAL (47) /** Simulate compression failures. */ -UNIV_INTERN uint srv_simulate_comp_failures = 0; +UNIV_INTERN uint srv_simulate_comp_failures; /** Buffer pool dump status frequence in percentages */ -UNIV_INTERN ulong srv_buf_dump_status_frequency = 0; +UNIV_INTERN ulong srv_buf_dump_status_frequency; /** Acquire the system_mutex. */ #define srv_sys_mutex_enter() do { \ -- cgit v1.2.1 From 731435af9a3e0fae7555439f65f58c79e92da857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 26 Apr 2017 10:13:22 +0300 Subject: srv_purge_coordinator_thread(): Remove redundant variables --- storage/innobase/srv/srv0srv.cc | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) (limited to 'storage') diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 4e1157fdc7c..aa795e972a2 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -2991,14 +2991,9 @@ DECLARE_THREAD(srv_purge_coordinator_thread)( ut_a(srv_purge_should_exit(thd, n_total_purged)); - ulint n_pages_purged = ULINT_MAX; - - /* Ensure that all records are purged if it is not a fast shutdown. - This covers the case where a record can be added after we exit the - loop above. */ - while (srv_fast_shutdown == 0 && n_pages_purged > 0) { - n_pages_purged = trx_purge(1, srv_purge_batch_size, false); - } + /* Ensure that all records are purged on slow shutdown. */ + while (srv_fast_shutdown == 0 + && trx_purge(1, srv_purge_batch_size, false)); #ifdef UNIV_DEBUG if (srv_fast_shutdown == 0) { @@ -3012,12 +3007,10 @@ DECLARE_THREAD(srv_purge_coordinator_thread)( delay in shutdown ,so reducing the batch size to magic number 20 (which was default in 5.5), which we hope will be sufficient to remove all the undo records */ - const uint temp_batch_size = 20; - n_pages_purged = trx_purge(1, srv_purge_batch_size <= temp_batch_size - ? srv_purge_batch_size : temp_batch_size, - true); - ut_a(n_pages_purged == 0 || srv_fast_shutdown != 0); + if (trx_purge(1, std::min(srv_purge_batch_size, 20UL), true)) { + ut_a(srv_fast_shutdown); + } /* The task queue should always be empty, independent of fast shutdown state. */ -- cgit v1.2.1 From 13dcdb0903f30130e69e2d6c7cf7770a3d3b9427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 26 Apr 2017 10:30:13 +0300 Subject: MDEV-11802 InnoDB purge does not always run when there is work to do srv_sys_t::n_threads_active[]: Protect writes by both the mutex and by atomic memory access. srv_active_wake_master_thread_low(): Reliably wake up the master thread if there is work to do. The trick is to atomically read srv_sys->n_threads_active[]. srv_wake_purge_thread_if_not_active(): Atomically read srv_sys->n_threads_active[] (and trx_sys->rseg_history_len), so that the purge should always be triggered when there is work to do. trx_commit_in_memory(): Invoke srv_wake_purge_thread_if_not_active() whenever a transaction is committed. Purge could have been prevented by the read view of the currently committing transaction, even if it is a read-only transaction. trx_purge_add_update_undo_to_history(): Do not wake up the purge. This is only called by trx_undo_update_cleanup(), as part of trx_write_serialisation_history(), which in turn is only called by trx_commit_low() which will always call trx_commit_in_memory(). Thus, the added call in trx_commit_in_memory() will cover also this use case where a committing read-write transaction added some update_undo log to the purge queue. trx_rseg_mem_restore(): Atomically modify trx_sys->rseg_history_len. --- storage/innobase/include/srv0srv.h | 30 ++++++-------------- storage/innobase/srv/srv0srv.cc | 57 +++++++++++++------------------------- storage/innobase/trx/trx0purge.cc | 1 - storage/innobase/trx/trx0rseg.cc | 2 +- storage/innobase/trx/trx0trx.cc | 1 + 5 files changed, 30 insertions(+), 61 deletions(-) (limited to 'storage') diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 6803047ed91..230d7b7effe 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -749,36 +749,24 @@ srv_set_io_thread_op_info( Resets the info describing an i/o thread current state. */ void srv_reset_io_thread_op_info(); -/*=========================*/ -/*******************************************************************//** -Tells the purge thread that there has been activity in the database -and wakes up the purge thread if it is suspended (not sleeping). Note -that there is a small chance that the purge thread stays suspended -(we do not protect our operation with the srv_sys_t:mutex, for -performance reasons). */ + +/** Wake up the purge threads if there is work to do. */ void -srv_wake_purge_thread_if_not_active(void); -/*=====================================*/ -/*******************************************************************//** -Tells the Innobase server that there has been activity in the database -and wakes up the master thread if it is suspended (not sleeping). Used -in the MySQL interface. Note that there is a small chance that the master -thread stays suspended (we do not protect our operation with the kernel -mutex, for performace reasons). */ +srv_wake_purge_thread_if_not_active(); +/** Wake up the InnoDB master thread if it was suspended (not sleeping). */ void -srv_active_wake_master_thread_low(void); -/*===================================*/ +srv_active_wake_master_thread_low(); + #define srv_active_wake_master_thread() \ do { \ if (!srv_read_only_mode) { \ srv_active_wake_master_thread_low(); \ } \ } while (0) -/*******************************************************************//** -Wakes up the master thread if it is suspended or being suspended. */ +/** Wake up the master thread if it is suspended or being suspended. */ void -srv_wake_master_thread(void); -/*========================*/ +srv_wake_master_thread(); + /******************************************************************//** Outputs to a file the output of the InnoDB Monitor. @return FALSE if not all information printed diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index aa795e972a2..8c62cc976b1 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -648,14 +648,16 @@ struct srv_sys_t{ ulint n_threads_active[SRV_MASTER + 1]; /*!< number of threads active - in a thread class */ + in a thread class; protected + by both my_atomic_addlint() + and mutex */ srv_stats_t::ulint_ctr_1_t activity_count; /*!< For tracking server activity */ }; -static srv_sys_t* srv_sys = NULL; +static srv_sys_t* srv_sys; /** Event to signal srv_monitor_thread. Not protected by a mutex. Set after setting srv_print_innodb_monitor. */ @@ -853,7 +855,7 @@ srv_reserve_slot( ut_ad(srv_slot_get_type(slot) == type); - ++srv_sys->n_threads_active[type]; + my_atomic_addlint(&srv_sys->n_threads_active[type], 1); srv_sys_mutex_exit(); @@ -894,16 +896,15 @@ srv_suspend_thread_low( case SRV_WORKER: ut_a(srv_n_purge_threads > 1); - ut_a(srv_sys->n_threads_active[type] > 0); break; } ut_a(!slot->suspended); slot->suspended = TRUE; - ut_a(srv_sys->n_threads_active[type] > 0); - - srv_sys->n_threads_active[type]--; + if (my_atomic_addlint(&srv_sys->n_threads_active[type], -1) < 0) { + ut_error; + } return(os_event_reset(slot->event)); } @@ -958,7 +959,7 @@ srv_resume_thread(srv_slot_t* slot, int64_t sig_count = 0, bool wait = true, ut_ad(slot->suspended); slot->suspended = FALSE; - ++srv_sys->n_threads_active[slot->type]; + my_atomic_addlint(&srv_sys->n_threads_active[slot->type], 1); srv_sys_mutex_exit(); return(timeout); } @@ -2066,22 +2067,16 @@ srv_get_active_thread_type(void) return(ret); } -/*******************************************************************//** -Tells the InnoDB server that there has been activity in the database -and wakes up the master thread if it is suspended (not sleeping). Used -in the MySQL interface. Note that there is a small chance that the master -thread stays suspended (we do not protect our operation with the -srv_sys_t->mutex, for performance reasons). */ +/** Wake up the InnoDB master thread if it was suspended (not sleeping). */ void srv_active_wake_master_thread_low() -/*===============================*/ { ut_ad(!srv_read_only_mode); ut_ad(!srv_sys_mutex_own()); srv_inc_activity_count(); - if (srv_sys->n_threads_active[SRV_MASTER] == 0) { + if (my_atomic_loadlint(&srv_sys->n_threads_active[SRV_MASTER]) == 0) { srv_slot_t* slot; srv_sys_mutex_enter(); @@ -2099,35 +2094,25 @@ srv_active_wake_master_thread_low() } } -/*******************************************************************//** -Tells the purge thread that there has been activity in the database -and wakes up the purge thread if it is suspended (not sleeping). Note -that there is a small chance that the purge thread stays suspended -(we do not protect our check with the srv_sys_t:mutex and the -purge_sys->latch, for performance reasons). */ +/** Wake up the purge threads if there is work to do. */ void -srv_wake_purge_thread_if_not_active(void) -/*=====================================*/ +srv_wake_purge_thread_if_not_active() { ut_ad(!srv_sys_mutex_own()); if (purge_sys->state == PURGE_STATE_RUN - && srv_sys->n_threads_active[SRV_PURGE] == 0) { + && !my_atomic_loadlint(&srv_sys->n_threads_active[SRV_PURGE]) + && my_atomic_loadlint(&trx_sys->rseg_history_len)) { srv_release_threads(SRV_PURGE, 1); } } -/*******************************************************************//** -Wakes up the master thread if it is suspended or being suspended. */ +/** Wake up the master thread if it is suspended or being suspended. */ void -srv_wake_master_thread(void) -/*========================*/ +srv_wake_master_thread() { - ut_ad(!srv_sys_mutex_own()); - srv_inc_activity_count(); - srv_release_threads(SRV_MASTER, 1); } @@ -2713,12 +2698,8 @@ DECLARE_THREAD(srv_worker_thread)( slot = srv_reserve_slot(SRV_WORKER); ut_a(srv_n_purge_threads > 1); - - srv_sys_mutex_enter(); - - ut_a(srv_sys->n_threads_active[SRV_WORKER] < srv_n_purge_threads); - - srv_sys_mutex_exit(); + ut_a(my_atomic_loadlint(&srv_sys->n_threads_active[SRV_WORKER]) + < static_cast(srv_n_purge_threads)); /* We need to ensure that the worker threads exit after the purge coordinator thread. Otherwise the purge coordinator can diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 42ae4890cab..b21ec75c3a6 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -291,7 +291,6 @@ trx_purge_add_update_undo_to_history( undo_header + TRX_UNDO_HISTORY_NODE, mtr); my_atomic_addlint(&trx_sys->rseg_history_len, 1); - srv_wake_purge_thread_if_not_active(); /* Write the trx number to the undo log header */ mlog_write_ull(undo_header + TRX_UNDO_TRX_NO, trx->no, mtr); diff --git a/storage/innobase/trx/trx0rseg.cc b/storage/innobase/trx/trx0rseg.cc index 3393a0464a8..663566cf26f 100644 --- a/storage/innobase/trx/trx0rseg.cc +++ b/storage/innobase/trx/trx0rseg.cc @@ -204,7 +204,7 @@ trx_rseg_mem_restore(trx_rseg_t* rseg, mtr_t* mtr) len = flst_get_len(rseg_header + TRX_RSEG_HISTORY); if (len > 0) { - trx_sys->rseg_history_len += len; + my_atomic_addlint(&trx_sys->rseg_history_len, len); node_addr = trx_purge_get_log_from_hist( flst_get_last(rseg_header + TRX_RSEG_HISTORY, mtr)); diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 36324c43970..d6755fca7c0 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -1908,6 +1908,7 @@ trx_commit_in_memory( trx_mutex_exit(trx); ut_a(trx->error_state == DB_SUCCESS); + srv_wake_purge_thread_if_not_active(); } /****************************************************************//** -- cgit v1.2.1 From 62dca454e725b47eb8d7b4ee383a2ff9ef28255f Mon Sep 17 00:00:00 2001 From: Aditya A Date: Wed, 4 Jan 2017 14:34:38 +0530 Subject: Bug #23070734 CONCURRENT TRUNCATE TABLES CAUSE STALLS PROBLEM When truncating single tablespace tables, we need to scan the entire buffer pool to remove the pages of the table from the buffer pool. During this scan and removal dict_sys->mutex is being held ,causing stalls in other DDL operations. FIX Release the dict_sys->mutex during the scan and reacquire it after the scan. Make sure that purge thread doesn't purge the records of the table being truncated and background stats collection thread skips the updation of stats for the table being truncated. [#rb 14564 Approved by Jimmy and satya ] --- storage/innobase/dict/dict0stats_bg.cc | 11 +++++++++-- storage/innobase/fil/fil0fil.cc | 34 +++++++++++++++++++++++----------- storage/innobase/include/fil0fil.h | 18 +++++++++++------- storage/innobase/row/row0purge.cc | 17 ++++++++++++++++- storage/innobase/row/row0trunc.cc | 4 ++-- storage/innobase/row/row0uins.cc | 7 ++++++- storage/innobase/row/row0umod.cc | 8 +++----- 7 files changed, 70 insertions(+), 29 deletions(-) (limited to 'storage') diff --git a/storage/innobase/dict/dict0stats_bg.cc b/storage/innobase/dict/dict0stats_bg.cc index 4ffee160c9f..986b6a2613f 100644 --- a/storage/innobase/dict/dict0stats_bg.cc +++ b/storage/innobase/dict/dict0stats_bg.cc @@ -1,7 +1,7 @@ /***************************************************************************** -Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2012, 2017, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -31,6 +31,7 @@ Created Apr 25, 2012 Vasil Dimov #include "row0mysql.h" #include "srv0start.h" #include "ut0new.h" +#include "fil0fil.h" #include @@ -318,6 +319,12 @@ dict_stats_process_entry_from_recalc_pool() return; } + if (fil_space_is_being_truncated(table->space)) { + dict_table_close(table, TRUE, FALSE); + mutex_exit(&dict_sys->mutex); + return; + } + /* Check whether table is corrupted */ if (table->corrupted) { dict_table_close(table, TRUE, FALSE); diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index d91ab3530a1..7fd41a8fcde 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -30,6 +30,7 @@ Created 10/25/1995 Heikki Tuuri #include "fil0crypt.h" #include "btr0btr.h" +#include "btr0sea.h" #include "buf0buf.h" #include "dict0boot.h" #include "dict0dict.h" @@ -55,9 +56,6 @@ Created 10/25/1995 Heikki Tuuri #include "os0event.h" #include "sync0sync.h" #include "buf0flu.h" -#include "srv0start.h" -#include "trx0purge.h" -#include "ut0new.h" #include "os0api.h" /** Tries to close a file in the LRU list. The caller must hold the fil_sys @@ -3266,20 +3264,34 @@ fil_prepare_for_truncate( return(err); } -/**********************************************************************//** -Reinitialize the original tablespace header with the same space id -for single tablespace */ +/** Reinitialize the original tablespace header with the same space id +for single tablespace +@param[in] id space id of the tablespace +@param[in] size size in blocks +@param[in] trx Transaction covering truncate */ void fil_reinit_space_header( -/*====================*/ - ulint id, /*!< in: space id */ - ulint size) /*!< in: size in blocks */ + ulint id, + ulint size, + trx_t* trx) { ut_a(!is_system_tablespace(id)); /* Invalidate in the buffer pool all pages belonging - to the tablespace */ + to the tablespace. The buffer pool scan may take long + time to complete, therefore we release dict_sys->mutex + and the dict operation lock during the scan and aquire + it again after the buffer pool scan.*/ + + row_mysql_unlock_data_dictionary(trx); + + /* Lock the search latch in shared mode to prevent user + from disabling AHI during the scan */ + btr_search_s_lock_all(); + DEBUG_SYNC_C("simulate_buffer_pool_scan"); buf_LRU_flush_or_remove_pages(id, BUF_REMOVE_ALL_NO_WRITE, 0); + btr_search_s_unlock_all(); + row_mysql_lock_data_dictionary(trx); /* Remove all insert buffer entries for the tablespace */ ibuf_delete_for_discarded_space(id); diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 666c42a9628..9c6d4f920ff 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -914,14 +914,18 @@ dberr_t fil_prepare_for_truncate( /*=====================*/ ulint id); /*!< in: space id */ -/**********************************************************************//** -Reinitialize the original tablespace header with the same space id -for single tablespace */ + +/** Reinitialize the original tablespace header with the same space id +for single tablespace +@param[in] id space id of the tablespace +@param[in] size size in blocks +@param[in] trx Transaction covering truncate */ void fil_reinit_space_header( -/*====================*/ - ulint id, /*!< in: space id */ - ulint size); /*!< in: size in blocks */ + ulint id, + ulint size, + trx_t* trx); + /*******************************************************************//** Closes a single-table tablespace. The tablespace must be cached in the memory cache. Free all pages used by the tablespace. diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc index 2a13203b747..5202fb15af8 100644 --- a/storage/innobase/row/row0purge.cc +++ b/storage/innobase/row/row0purge.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -43,6 +43,7 @@ Created 3/14/1997 Heikki Tuuri #include "srv0start.h" #include "handler.h" #include "ha_innodb.h" +#include "fil0fil.h" /************************************************************************* IMPORTANT NOTE: Any operation that generates redo MUST check that there @@ -854,6 +855,20 @@ try_again: } ut_ad(!dict_table_is_temporary(node->table)); + if (fil_space_is_being_truncated(node->table->space)) { + +#if UNIV_DEBUG + ib::info() << "Record with space id " + << node->table->space + << " belongs to table which is being truncated" + << " therefore skipping this undo record."; +#endif + ut_ad(dict_table_is_file_per_table(node->table)); + dict_table_close(node->table, FALSE, FALSE); + node->table = NULL; + goto err_exit; + } + if (node->table->n_v_cols && !node->table->vc_templ && dict_table_has_indexed_v_cols(node->table)) { /* Need server fully up for virtual column computation */ diff --git a/storage/innobase/row/row0trunc.cc b/storage/innobase/row/row0trunc.cc index 8529cfec9fd..1fc30e714f4 100644 --- a/storage/innobase/row/row0trunc.cc +++ b/storage/innobase/row/row0trunc.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2013, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2013, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -2015,7 +2015,7 @@ row_truncate_table_for_mysql( space_size -= ib_vector_size(table->fts->indexes); } - fil_reinit_space_header(table->space, space_size); + fil_reinit_space_header(table->space, space_size, trx); } DBUG_EXECUTE_IF("ib_trunc_crash_with_intermediate_log_checkpoint", diff --git a/storage/innobase/row/row0uins.cc b/storage/innobase/row/row0uins.cc index 25504e32087..e0b9508caed 100644 --- a/storage/innobase/row/row0uins.cc +++ b/storage/innobase/row/row0uins.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -41,6 +41,7 @@ Created 2/25/1997 Heikki Tuuri #include "que0que.h" #include "ibuf0ibuf.h" #include "log0log.h" +#include "fil0fil.h" /************************************************************************* IMPORTANT NOTE: Any operation that generates redo MUST check that there @@ -347,6 +348,10 @@ row_undo_ins_parse_undo_rec( if (UNIV_UNLIKELY(node->table == NULL)) { } else if (UNIV_UNLIKELY(node->table->ibd_file_missing)) { close_table: + dict_table_close(node->table, dict_locked, FALSE); + node->table = NULL; + } else if (fil_space_is_being_truncated(node->table->space)) { + dict_table_close(node->table, dict_locked, FALSE); node->table = NULL; } else { diff --git a/storage/innobase/row/row0umod.cc b/storage/innobase/row/row0umod.cc index 378cad00b93..c583a438988 100644 --- a/storage/innobase/row/row0umod.cc +++ b/storage/innobase/row/row0umod.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1139,12 +1139,10 @@ row_undo_mod_parse_undo_rec( return; } - if (node->table->ibd_file_missing) { + if (node->table->ibd_file_missing || + fil_space_is_being_truncated(node->table->space) ) { dict_table_close(node->table, dict_locked, FALSE); - - /* We skip undo operations to missing .ibd files */ node->table = NULL; - return; } -- cgit v1.2.1 From 849af74a48dd7c556ecc7c91ed7504550da8eac1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 25 Apr 2017 18:25:57 +0300 Subject: MariaDB adjustments for Oracle Bug#23070734 fix Split the test case so that a server restart is not needed. Reduce the test cases and use a simpler mechanism for triggering and waiting for purge. fil_table_accessible(): Check if a table can be accessed without enjoying MDL protection. --- storage/innobase/dict/dict0stats_bg.cc | 9 ++------- storage/innobase/fil/fil0fil.cc | 28 +++++++++++++++++++++++++++- storage/innobase/include/btr0sea.h | 2 ++ storage/innobase/include/fil0fil.h | 11 +++++++++++ storage/innobase/row/row0purge.cc | 21 ++------------------- storage/innobase/row/row0uins.cc | 21 ++++++++++++++++----- storage/innobase/row/row0umod.cc | 31 +++++++++++++++++++++++-------- 7 files changed, 83 insertions(+), 40 deletions(-) (limited to 'storage') diff --git a/storage/innobase/dict/dict0stats_bg.cc b/storage/innobase/dict/dict0stats_bg.cc index 986b6a2613f..876d1bcb342 100644 --- a/storage/innobase/dict/dict0stats_bg.cc +++ b/storage/innobase/dict/dict0stats_bg.cc @@ -319,14 +319,9 @@ dict_stats_process_entry_from_recalc_pool() return; } - if (fil_space_is_being_truncated(table->space)) { - dict_table_close(table, TRUE, FALSE); - mutex_exit(&dict_sys->mutex); - return; - } + ut_ad(!dict_table_is_temporary(table)); - /* Check whether table is corrupted */ - if (table->corrupted) { + if (!fil_table_accessible(table)) { dict_table_close(table, TRUE, FALSE); mutex_exit(&dict_sys->mutex); return; diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 7fd41a8fcde..47c1c480a3f 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -3055,6 +3055,32 @@ fil_close_tablespace( return(err); } +/** Determine whether a table can be accessed in operations that are +not (necessarily) protected by meta-data locks. +(Rollback would generally be protected, but rollback of +FOREIGN KEY CASCADE/SET NULL is not protected by meta-data locks +but only by InnoDB table locks, which may be broken by TRUNCATE TABLE.) +@param[in] table persistent table +checked @return whether the table is accessible */ +bool +fil_table_accessible(const dict_table_t* table) +{ + if (UNIV_UNLIKELY(table->ibd_file_missing || table->corrupted)) { + return(false); + } + + if (fil_space_t* space = fil_space_acquire(table->space)) { + bool accessible = !space->is_stopping(); + fil_space_release(space); + ut_ad(accessible || dict_table_is_file_per_table(table)); + return(accessible); + } else { + /* The tablespace may momentarily be missing during + TRUNCATE TABLE. */ + return(false); + } +} + /** Deletes an IBD tablespace, either general or single-table. The tablespace must be cached in the memory cache. This will delete the datafile, fil_space_t & fil_node_t entries from the file_system_t cache. @@ -3288,7 +3314,7 @@ fil_reinit_space_header( /* Lock the search latch in shared mode to prevent user from disabling AHI during the scan */ btr_search_s_lock_all(); - DEBUG_SYNC_C("simulate_buffer_pool_scan"); + DEBUG_SYNC_C("buffer_pool_scan"); buf_LRU_flush_or_remove_pages(id, BUF_REMOVE_ALL_NO_WRITE, 0); btr_search_s_unlock_all(); row_mysql_lock_data_dictionary(trx); diff --git a/storage/innobase/include/btr0sea.h b/storage/innobase/include/btr0sea.h index 8887a3f81ad..fad0dac93c4 100644 --- a/storage/innobase/include/btr0sea.h +++ b/storage/innobase/include/btr0sea.h @@ -247,6 +247,8 @@ btr_get_search_table(const dict_index_t* index); # define btr_search_drop_page_hash_index(block) # define btr_search_s_lock(index) # define btr_search_s_unlock(index) +# define btr_search_s_lock_all(index) +# define btr_search_s_unlock_all(index) # define btr_search_x_lock(index) # define btr_search_x_unlock(index) # define btr_search_info_update(index, cursor) diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 9c6d4f920ff..081362fb6f4 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -883,6 +883,17 @@ fil_op_replay_rename( const char* new_name) MY_ATTRIBUTE((warn_unused_result)); +/** Determine whether a table can be accessed in operations that are +not (necessarily) protected by meta-data locks. +(Rollback would generally be protected, but rollback of +FOREIGN KEY CASCADE/SET NULL is not protected by meta-data locks +but only by InnoDB table locks, which may be broken by TRUNCATE TABLE.) +@param[in] table persistent table +checked @return whether the table is accessible */ +bool +fil_table_accessible(const dict_table_t* table) + MY_ATTRIBUTE((warn_unused_result, nonnull)); + /** Deletes an IBD tablespace, either general or single-table. The tablespace must be cached in the memory cache. This will delete the datafile, fil_space_t & fil_node_t entries from the file_system_t cache. diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc index 5202fb15af8..492d864ec96 100644 --- a/storage/innobase/row/row0purge.cc +++ b/storage/innobase/row/row0purge.cc @@ -853,17 +853,10 @@ try_again: /* The table has been dropped: no need to do purge */ goto err_exit; } - ut_ad(!dict_table_is_temporary(node->table)); - if (fil_space_is_being_truncated(node->table->space)) { + ut_ad(!dict_table_is_temporary(node->table)); -#if UNIV_DEBUG - ib::info() << "Record with space id " - << node->table->space - << " belongs to table which is being truncated" - << " therefore skipping this undo record."; -#endif - ut_ad(dict_table_is_file_per_table(node->table)); + if (!fil_table_accessible(node->table)) { dict_table_close(node->table, FALSE, FALSE); node->table = NULL; goto err_exit; @@ -887,16 +880,6 @@ try_again: innobase_init_vc_templ(node->table); } - if (node->table->ibd_file_missing) { - /* We skip purge of missing .ibd files */ - - dict_table_close(node->table, FALSE, FALSE); - - node->table = NULL; - - goto err_exit; - } - clust_index = dict_table_get_first_index(node->table); if (clust_index == NULL diff --git a/storage/innobase/row/row0uins.cc b/storage/innobase/row/row0uins.cc index e0b9508caed..9288adb21a4 100644 --- a/storage/innobase/row/row0uins.cc +++ b/storage/innobase/row/row0uins.cc @@ -346,12 +346,20 @@ row_undo_ins_parse_undo_rec( /* Skip the UNDO if we can't find the table or the .ibd file. */ if (UNIV_UNLIKELY(node->table == NULL)) { - } else if (UNIV_UNLIKELY(node->table->ibd_file_missing)) { -close_table: - dict_table_close(node->table, dict_locked, FALSE); - node->table = NULL; - } else if (fil_space_is_being_truncated(node->table->space)) { + return; + } + if (UNIV_UNLIKELY(!fil_table_accessible(node->table))) { +close_table: + /* Normally, tables should not disappear or become + unaccessible during ROLLBACK, because they should be + protected by InnoDB table locks. TRUNCATE TABLE + or table corruption could be valid exceptions. + + FIXME: When running out of temporary tablespace, it + would probably be better to just drop all temporary + tables (and temporary undo log records) of the current + connection, instead of doing this rollback. */ dict_table_close(node->table, dict_locked, FALSE); node->table = NULL; } else { @@ -362,6 +370,9 @@ close_table: ptr, clust_index, &node->ref, node->heap); if (!row_undo_search_clust_to_pcur(node)) { + /* An error probably occurred during + an insert into the clustered index, + after we wrote the undo log record. */ goto close_table; } if (node->table->n_v_cols) { diff --git a/storage/innobase/row/row0umod.cc b/storage/innobase/row/row0umod.cc index c583a438988..ba072a72aa1 100644 --- a/storage/innobase/row/row0umod.cc +++ b/storage/innobase/row/row0umod.cc @@ -1139,8 +1139,17 @@ row_undo_mod_parse_undo_rec( return; } - if (node->table->ibd_file_missing || - fil_space_is_being_truncated(node->table->space) ) { + if (UNIV_UNLIKELY(!fil_table_accessible(node->table))) { +close_table: + /* Normally, tables should not disappear or become + unaccessible during ROLLBACK, because they should be + protected by InnoDB table locks. TRUNCATE TABLE + or table corruption could be valid exceptions. + + FIXME: When running out of temporary tablespace, it + would probably be better to just drop all temporary + tables (and temporary undo log records) of the current + connection, instead of doing this rollback. */ dict_table_close(node->table, dict_locked, FALSE); node->table = NULL; return; @@ -1160,15 +1169,21 @@ row_undo_mod_parse_undo_rec( node->new_trx_id = trx_id; node->cmpl_info = cmpl_info; - if (!row_undo_search_clust_to_pcur(node)) { - - dict_table_close(node->table, dict_locked, FALSE); - - node->table = NULL; + if (UNIV_UNLIKELY(!row_undo_search_clust_to_pcur(node))) { + /* This should never occur. As long as this + rolling-back transaction exists, the PRIMARY KEY value + pointed to by the undo log record must exist. + btr_cur_upd_lock_and_undo() only writes the undo log + record after successfully acquiring an exclusive lock + on the the clustered index record. That lock will not + be released before the transaction is committed or + fully rolled back. */ + ut_ad(0); + goto close_table; } /* Extract indexed virtual columns from undo log */ - if (node->table && node->table->n_v_cols) { + if (node->table->n_v_cols) { row_upd_replace_vcol(node->row, node->table, node->update, false, node->undo_row, (node->cmpl_info & UPD_NODE_NO_ORD_CHANGE) -- cgit v1.2.1 From dbe4c4e354ee8825dd186b1df624089683250b50 Mon Sep 17 00:00:00 2001 From: rahul malik Date: Fri, 6 Jan 2017 10:37:07 +0100 Subject: BUG#25330449 ASSERT SIZE==SPACE->SIZE DURING BUF_READ_AHEAD_RANDOM Problem: During read head, wrong page size is used to calcuate the tablespace size. Fix: Use physical page size to calculate tablespace size Reveiwed-By: Satya Bodapati RB: 14993 --- storage/innobase/buf/buf0rea.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'storage') diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 33d82ee5a87..368c1fcb9ae 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2015, 2016 MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -301,10 +301,10 @@ buf_read_ahead_random( node = UT_LIST_GET_NEXT(chain, node)) { size += os_file_get_size(node->handle) - / page_size.logical(); + / page_size.physical(); } - ut_ad(size==space->size); + ut_ad(size == space->size); } #endif /* UNIV_DEBUG */ -- cgit v1.2.1 From 88a84f49b3c65a4e6b50910b51b4573d578ec057 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Fri, 6 Jan 2017 19:48:54 +0530 Subject: Bug #25167032 CRASH WHEN ASSIGNING MY_ERRNO - MISSING MY_THREAD_INIT IN BACKGROUND THREAD Description: =========== Add my_thread_init() and my_thread_exit() for background threads which initializes and frees the st_my_thread_var structure. Reviewed-by: Jimmy Yang RB: 15003 --- storage/innobase/buf/buf0dump.cc | 6 ++++-- storage/innobase/log/log0recv.cc | 4 +++- storage/innobase/trx/trx0roll.cc | 5 ++++- 3 files changed, 11 insertions(+), 4 deletions(-) (limited to 'storage') diff --git a/storage/innobase/buf/buf0dump.cc b/storage/innobase/buf/buf0dump.cc index 9b86b1c16da..ce7488e3d1f 100644 --- a/storage/innobase/buf/buf0dump.cc +++ b/storage/innobase/buf/buf0dump.cc @@ -1,7 +1,7 @@ /***************************************************************************** -Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2011, 2017, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -796,6 +796,7 @@ extern "C" os_thread_ret_t DECLARE_THREAD(buf_dump_thread)(void*) { + my_thread_init(); ut_ad(!srv_read_only_mode); /* JAN: TODO: MySQL 5.7 PSI #ifdef UNIV_PFS_THREAD @@ -836,6 +837,7 @@ DECLARE_THREAD(buf_dump_thread)(void*) srv_buf_dump_thread_active = false; + my_thread_end(); /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ os_thread_exit(); diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index c035917186a..e23341201d6 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. Copyright (c) 2013, 2017, MariaDB Corporation. @@ -532,6 +532,7 @@ DECLARE_THREAD(recv_writer_thread)( /*!< in: a dummy parameter required by os_thread_create */ { + my_thread_init(); ut_ad(!srv_read_only_mode); #ifdef UNIV_PFS_THREAD @@ -568,6 +569,7 @@ DECLARE_THREAD(recv_writer_thread)( recv_writer_thread_active = false; + my_thread_end(); /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ diff --git a/storage/innobase/trx/trx0roll.cc b/storage/innobase/trx/trx0roll.cc index ec723375fe9..69f01e64b59 100644 --- a/storage/innobase/trx/trx0roll.cc +++ b/storage/innobase/trx/trx0roll.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2016, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -868,6 +868,8 @@ DECLARE_THREAD(trx_rollback_or_clean_all_recovered)( /*!< in: a dummy parameter required by os_thread_create */ { + my_thread_init(); + ut_ad(!srv_read_only_mode); #ifdef UNIV_PFS_THREAD @@ -878,6 +880,7 @@ DECLARE_THREAD(trx_rollback_or_clean_all_recovered)( trx_rollback_or_clean_is_active = false; + my_thread_end(); /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ -- cgit v1.2.1 From 2ef1baa75f35d8281af31fb1a22ff5c5c18763d3 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Tue, 10 Jan 2017 10:48:56 +0530 Subject: Bug #24793413 LOG PARSING BUFFER OVERFLOW Problem: ======== During checkpoint, we are writing all MLOG_FILE_NAME records in one mtr and parse buffer can't be processed till MLOG_MULTI_REC_END. Eventually parse buffer exceeds the RECV_PARSING_BUF_SIZE and eventually it overflows. Fix: === 1) Break the large mtr if it exceeds LOG_CHECKPOINT_FREE_PER_THREAD into multiple mtr during checkpoint. 2) Move the parsing buffer if we are encountering only MLOG_FILE_NAME records. So that it will never exceed the RECV_PARSING_BUF_SIZE. Reviewed-by: Debarun Bannerjee Reviewed-by: Rahul M Malik RB: 14743 --- storage/innobase/fil/fil0fil.cc | 22 +++++++++++++++++++++- storage/innobase/include/log0log.h | 8 +++++++- storage/innobase/include/mtr0mtr.h | 10 +++++++--- storage/innobase/log/log0log.cc | 8 +------- storage/innobase/log/log0recv.cc | 30 ++++++++++++++++++++++++++++-- storage/innobase/mtr/mtr0mtr.cc | 27 ++++++++++++++++++--------- 6 files changed, 82 insertions(+), 23 deletions(-) (limited to 'storage') diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 47c1c480a3f..84c2f07b361 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -40,6 +40,7 @@ Created 10/25/1995 Heikki Tuuri #include "fsp0space.h" #include "fsp0sysspace.h" #include "hash0hash.h" +#include "log0log.h" #include "log0recv.h" #include "mach0data.h" #include "mem0mem.h" @@ -6536,6 +6537,12 @@ fil_names_clear( bool do_write) { mtr_t mtr; + ulint mtr_checkpoint_size = LOG_CHECKPOINT_FREE_PER_THREAD; + + DBUG_EXECUTE_IF( + "increase_mtr_checkpoint_size", + mtr_checkpoint_size = 75 * 1024; + ); ut_ad(log_mutex_own()); @@ -6569,11 +6576,24 @@ fil_names_clear( fil_names_write(space, &mtr); do_write = true; + const mtr_buf_t* mtr_log = mtr_get_log(&mtr); + + /** If the mtr buffer size exceeds the size of + LOG_CHECKPOINT_FREE_PER_THREAD then commit the multi record + mini-transaction, start the new mini-transaction to + avoid the parsing buffer overflow error during recovery. */ + + if (mtr_log->size() > mtr_checkpoint_size) { + ut_ad(mtr_log->size() < (RECV_PARSING_BUF_SIZE / 2)); + mtr.commit_checkpoint(lsn, false); + mtr.start(); + } + space = next; } if (do_write) { - mtr.commit_checkpoint(lsn); + mtr.commit_checkpoint(lsn, true); } else { ut_ad(!mtr.has_modifications()); } diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 35a94513632..d1aae64227e 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2016, Oracle and/or its affiliates. All rights reserved. +Copyright (c) 1995, 2017, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2009, Google Inc. Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. @@ -46,6 +46,12 @@ struct log_group_t; /** Magic value to use instead of log checksums when they are disabled */ #define LOG_NO_CHECKSUM_MAGIC 0xDEADBEEFUL +/* Margin for the free space in the smallest log group, before a new query +step which modifies the database, is started */ + +#define LOG_CHECKPOINT_FREE_PER_THREAD (4 * UNIV_PAGE_SIZE) +#define LOG_CHECKPOINT_EXTRA_FREE (8 * UNIV_PAGE_SIZE) + typedef ulint (*log_checksum_func_t)(const byte* log_block); /** Pointer to the log checksum calculation function. Protected with diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index a02333109cc..045a14221a3 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. Copyright (c) 2013, 2017, MariaDB Corporation @@ -269,8 +269,12 @@ struct mtr_t { MLOG_FILE_NAME records and a MLOG_CHECKPOINT marker. The caller must invoke log_mutex_enter() and log_mutex_exit(). This is to be used at log_checkpoint(). - @param[in] checkpoint_lsn the LSN of the log checkpoint */ - void commit_checkpoint(lsn_t checkpoint_lsn); + @param[in] checkpoint_lsn the LSN of the log checkpoint + @param[in] write_mlog_checkpoint Write MLOG_CHECKPOINT marker + if it is enabled. */ + void commit_checkpoint( + lsn_t checkpoint_lsn, + bool write_mlog_checkpoint); /** Return current size of the buffer. @return savepoint */ diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index c96260ca1ac..bf228c077a5 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Google Inc. Copyright (c) 2014, 2017, MariaDB Corporation. @@ -106,12 +106,6 @@ static time_t log_last_margine_warning_time; #define LOG_BUF_FLUSH_RATIO 2 #define LOG_BUF_FLUSH_MARGIN (LOG_BUF_WRITE_MARGIN + 4 * UNIV_PAGE_SIZE) -/* Margin for the free space in the smallest log group, before a new query -step which modifies the database, is started */ - -#define LOG_CHECKPOINT_FREE_PER_THREAD (4 * UNIV_PAGE_SIZE) -#define LOG_CHECKPOINT_EXTRA_FREE (8 * UNIV_PAGE_SIZE) - /* This parameter controls asynchronous making of a new checkpoint; the value should be bigger than LOG_POOL_PREFLUSH_RATIO_SYNC */ diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index e23341201d6..0b46c92f00a 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2482,6 +2482,8 @@ loop: ulint total_len = 0; ulint n_recs = 0; + bool only_mlog_file = true; + ulint mlog_rec_len = 0; for (;;) { len = recv_parse_log_rec( @@ -2510,6 +2512,22 @@ loop: = recv_sys->recovered_offset + total_len; recv_previous_parsed_rec_is_multi = 1; + /* MLOG_FILE_NAME redo log records doesn't make changes + to persistent data. If only MLOG_FILE_NAME redo + log record exists then reset the parsing buffer pointer + by changing recovered_lsn and recovered_offset. */ + if (type != MLOG_FILE_NAME && only_mlog_file == true) { + only_mlog_file = false; + } + + if (only_mlog_file) { + new_recovered_lsn = recv_calc_lsn_on_data_add( + recv_sys->recovered_lsn, len); + mlog_rec_len += len; + recv_sys->recovered_offset += len; + recv_sys->recovered_lsn = new_recovered_lsn; + } + total_len += len; n_recs++; @@ -2523,6 +2541,7 @@ loop: " n=" ULINTPF, recv_sys->recovered_lsn, total_len, n_recs)); + total_len -= mlog_rec_len; break; } @@ -2742,6 +2761,7 @@ recv_scan_log_recs( ulint data_len; bool more_data = false; bool apply = recv_sys->mlog_checkpoint_lsn != 0; + ulint recv_parsing_buf_size = RECV_PARSING_BUF_SIZE; ut_ad(start_lsn % OS_FILE_LOG_BLOCK_SIZE == 0); ut_ad(end_lsn % OS_FILE_LOG_BLOCK_SIZE == 0); @@ -2814,8 +2834,14 @@ recv_scan_log_recs( parsing buffer if parse_start_lsn is already non-zero */ + DBUG_EXECUTE_IF( + "reduce_recv_parsing_buf", + recv_parsing_buf_size + = (70 * 1024); + ); + if (recv_sys->len + 4 * OS_FILE_LOG_BLOCK_SIZE - >= RECV_PARSING_BUF_SIZE) { + >= recv_parsing_buf_size) { ib::error() << "Log parsing buffer overflow." " Recovery may have failed!"; @@ -2865,7 +2891,7 @@ recv_scan_log_recs( *store_to_hash = STORE_NO; } - if (recv_sys->recovered_offset > RECV_PARSING_BUF_SIZE / 4) { + if (recv_sys->recovered_offset > recv_parsing_buf_size / 4) { /* Move parsing buffer data to the buffer start */ recv_sys_justify_left_parsing_buf(); diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index c626937b0d8..d8c7d59fe27 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -580,9 +580,13 @@ but generated some redo log on a higher level, such as MLOG_FILE_NAME records and a MLOG_CHECKPOINT marker. The caller must invoke log_mutex_enter() and log_mutex_exit(). This is to be used at log_checkpoint(). -@param[in] checkpoint_lsn the LSN of the log checkpoint */ +@param[in] checkpoint_lsn the LSN of the log checkpoint +@param[in] write_mlog_checkpoint Write MLOG_CHECKPOINT marker + if it is enabled. */ void -mtr_t::commit_checkpoint(lsn_t checkpoint_lsn) +mtr_t::commit_checkpoint( + lsn_t checkpoint_lsn, + bool write_mlog_checkpoint) { ut_ad(log_mutex_own()); ut_ad(is_active()); @@ -593,6 +597,7 @@ mtr_t::commit_checkpoint(lsn_t checkpoint_lsn) ut_ad(m_impl.m_memo.size() == 0); ut_ad(!srv_read_only_mode); ut_d(m_impl.m_state = MTR_STATE_COMMITTING); + ut_ad(write_mlog_checkpoint || m_impl.m_n_log_recs > 1); /* This is a dirty read, for debugging. */ ut_ad(!recv_no_log_write); @@ -608,20 +613,24 @@ mtr_t::commit_checkpoint(lsn_t checkpoint_lsn) &m_impl.m_log, MLOG_MULTI_REC_END, MLOG_1BYTE); } - byte* ptr = m_impl.m_log.push(SIZE_OF_MLOG_CHECKPOINT); + if (write_mlog_checkpoint) { + byte* ptr = m_impl.m_log.push(SIZE_OF_MLOG_CHECKPOINT); #if SIZE_OF_MLOG_CHECKPOINT != 9 # error SIZE_OF_MLOG_CHECKPOINT != 9 #endif - *ptr = MLOG_CHECKPOINT; - mach_write_to_8(ptr + 1, checkpoint_lsn); + *ptr = MLOG_CHECKPOINT; + mach_write_to_8(ptr + 1, checkpoint_lsn); + } Command cmd(this); cmd.finish_write(m_impl.m_log.size()); cmd.release_resources(); - DBUG_PRINT("ib_log", - ("MLOG_CHECKPOINT(" LSN_PF ") written at " LSN_PF, - checkpoint_lsn, log_sys->lsn)); + if (write_mlog_checkpoint) { + DBUG_PRINT("ib_log", + ("MLOG_CHECKPOINT(" LSN_PF ") written at " LSN_PF, + checkpoint_lsn, log_sys->lsn)); + } } #ifdef UNIV_DEBUG -- cgit v1.2.1 From 7223ec4ca79a436509a1734c8cfb082927159143 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Wed, 1 Feb 2017 16:03:57 +0530 Subject: Bug #25385590 DROP TABLE CRASHES IF INNODB_FORCE_RECOVERY > 4 Problem: ======== - Drop table assert if innodb_force_recovery is set to 5 or 6. For innodb_force_recovery 5 and 6, InnoDB doesn't scan the undo log and it makes the redo rollback segment as NULL. There is no way for transaction to write any undo log. - If innodb_force_recovery is set to 6 then InnoDB does not do the redo log roll-forward in connection with recovery. In this case, log_sys will be initalized only and it will not have latest checkpoint information. Checkpoint is done during shutdown even innodb_force_recovery is set to 6. So it leads to incorrect information update in checkpoint header. Solution: ======== 1) Allow drop table only if innodb_force_recovery < 5. 2) Make innodb as read-only if innodb_force_recovery is set to 6. 3) During shutdown, remove the checkpoint if innodb_force_recovery is set to 6. Reviewed-by: Jimmy Yang RB: 15075 --- storage/innobase/handler/ha_innodb.cc | 3 ++- storage/innobase/srv/srv0start.cc | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'storage') diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index af1793d1920..60528e2e3ec 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -13797,7 +13797,8 @@ ha_innobase::delete_table( extension, in contrast to ::create */ normalize_table_name(norm_name, name); - if (srv_read_only_mode) { + if (srv_read_only_mode + || srv_force_recovery >= SRV_FORCE_NO_UNDO_LOG_SCAN) { DBUG_RETURN(HA_ERR_TABLE_READONLY); } diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 3ef937c303d..6d153ff1bc6 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -1442,6 +1442,10 @@ innobase_start_or_create_for_mysql(void) size_t dirnamelen; unsigned i = 0; + if (srv_force_recovery == SRV_FORCE_NO_LOG_REDO) { + srv_read_only_mode = true; + } + high_level_read_only = srv_read_only_mode || srv_force_recovery > SRV_FORCE_NO_TRX_UNDO; -- cgit v1.2.1 From 67e9c4cf6c70480dfbd80e1565faf25c17f398a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 26 Apr 2017 13:53:13 +0300 Subject: Adapt the test case for Oracle Bug#25385590 buf_chunk_not_freed(), logs_empty_and_mark_files_at_shutdown(): Relax debug assertions when innodb_force_recovery=6 implies innodb_read_only. --- storage/innobase/buf/buf0buf.cc | 4 +++- storage/innobase/log/log0log.cc | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) (limited to 'storage') diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 3761d81b2d1..a1efa446f76 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1762,7 +1762,9 @@ buf_chunk_not_freed( == block->page.newest_modification); ut_ad(block->page.oldest_modification == 0 || block->page.oldest_modification - == recv_sys->recovered_lsn); + == recv_sys->recovered_lsn + || srv_force_recovery + == SRV_FORCE_NO_LOG_REDO); ut_ad(block->page.buf_fix_count == 0); ut_ad(block->page.io_fix == BUF_IO_NONE); break; diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index bf228c077a5..7d4e0459610 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -2172,7 +2172,8 @@ wait_suspend_loop: bool freed = buf_all_freed(); ut_a(freed); - ut_a(lsn == log_sys->lsn); + ut_a(lsn == log_sys->lsn + || srv_force_recovery == SRV_FORCE_NO_LOG_REDO); if (lsn < srv_start_lsn) { ib::error() << "Shutdown LSN=" << lsn @@ -2198,7 +2199,8 @@ wait_suspend_loop: freed = buf_all_freed(); ut_a(freed); - ut_a(lsn == log_sys->lsn); + ut_a(lsn == log_sys->lsn + || srv_force_recovery == SRV_FORCE_NO_LOG_REDO); } /******************************************************//** -- cgit v1.2.1 From fca0698fd8d04fd695f45fb4e28eefffc596513f Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Tue, 14 Feb 2017 14:34:44 +0530 Subject: Bug #25385590 DROP TABLE CRASHES IF INNODB_FORCE_RECOVERY > 4 - Allow the server to start if innodb force recovery is set to 6 even though change buffer is not empty Reviewed-by: Jimmy Yang --- storage/innobase/dict/dict0boot.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'storage') diff --git a/storage/innobase/dict/dict0boot.cc b/storage/innobase/dict/dict0boot.cc index 8d5bde4e0e2..3ae6639cd73 100644 --- a/storage/innobase/dict/dict0boot.cc +++ b/storage/innobase/dict/dict0boot.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2016, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -488,7 +488,9 @@ dict_boot(void) err = ibuf_init_at_db_start(); if (err == DB_SUCCESS) { - if (srv_read_only_mode && !ibuf_is_empty()) { + if (srv_read_only_mode + && srv_force_recovery != SRV_FORCE_NO_LOG_REDO + && !ibuf_is_empty()) { if (srv_force_recovery < SRV_FORCE_NO_IBUF_MERGE) { ib::error() << "Change buffer must be empty when" -- cgit v1.2.1 From 14fe6dd23947636caca6743425ad0501f9b73f51 Mon Sep 17 00:00:00 2001 From: Aditya A Date: Wed, 15 Feb 2017 12:00:07 +0530 Subject: Bug #23046302 COUNT(*) MUCH SLOWER ON 5.7 THAN 5.6 ANALYSIS This is regression caused due to worklog 6742 which implemented ha_innobase::records() which always uses clustered index to get the row count. Previously optimizer chose secondary index which was smaller in size of clustered index to scan for rows and resulted in a quicker scan. FIX After discussion it was decided to remove this feature in 5.7. [#rb14040 Approved by Kevin and Oystein ] --- storage/innobase/handler/ha_innodb.cc | 112 +------------------------------ storage/innobase/handler/ha_innodb.h | 5 +- storage/innobase/handler/ha_innopart.cc | 37 +--------- storage/innobase/handler/ha_innopart.h | 4 -- storage/innobase/include/ha_prototypes.h | 1 - storage/innobase/include/row0mysql.h | 5 +- storage/innobase/row/row0mysql.cc | 11 +-- 7 files changed, 6 insertions(+), 169 deletions(-) (limited to 'storage') diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 60528e2e3ec..c486b0d4c4d 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -14230,116 +14230,6 @@ ha_innobase::rename_table( DBUG_RETURN(convert_error_code_to_mysql(error, 0, NULL)); } -/*********************************************************************//** -Returns the exact number of records that this client can see using this -handler object. -@return Error code in case something goes wrong. -These errors will abort the current query: - case HA_ERR_LOCK_DEADLOCK: - case HA_ERR_LOCK_TABLE_FULL: - case HA_ERR_LOCK_WAIT_TIMEOUT: - case HA_ERR_QUERY_INTERRUPTED: -For other error codes, the server will fall back to counting records. */ - -#ifdef MYSQL_57_SELECT_COUNT_OPTIMIZATION -int -ha_innobase::records( -/*==================*/ - ha_rows* num_rows) /*!< out: number of rows */ -{ - DBUG_ENTER("ha_innobase::records()"); - - dberr_t ret; - ulint n_rows = 0; /* Record count in this view */ - - update_thd(); - - if (dict_table_is_discarded(m_prebuilt->table)) { - ib_senderrf( - m_user_thd, - IB_LOG_LEVEL_ERROR, - ER_TABLESPACE_DISCARDED, - table->s->table_name.str); - - *num_rows = HA_POS_ERROR; - DBUG_RETURN(HA_ERR_NO_SUCH_TABLE); - - } else if (m_prebuilt->table->ibd_file_missing) { - ib_senderrf( - m_user_thd, IB_LOG_LEVEL_ERROR, - ER_TABLESPACE_MISSING, - table->s->table_name.str); - - *num_rows = HA_POS_ERROR; - DBUG_RETURN(HA_ERR_TABLESPACE_MISSING); - - } else if (m_prebuilt->table->corrupted) { - ib_errf(m_user_thd, IB_LOG_LEVEL_WARN, - ER_INNODB_INDEX_CORRUPT, - "Table '%s' is corrupt.", - table->s->table_name.str); - - *num_rows = HA_POS_ERROR; - DBUG_RETURN(HA_ERR_INDEX_CORRUPT); - } - - TrxInInnoDB trx_in_innodb(m_prebuilt->trx); - - m_prebuilt->trx->op_info = "counting records"; - - dict_index_t* index = dict_table_get_first_index(m_prebuilt->table); - - ut_ad(dict_index_is_clust(index)); - - m_prebuilt->index_usable = row_merge_is_index_usable( - m_prebuilt->trx, index); - - if (!m_prebuilt->index_usable) { - *num_rows = HA_POS_ERROR; - DBUG_RETURN(HA_ERR_TABLE_DEF_CHANGED); - } - - /* (Re)Build the m_prebuilt->mysql_template if it is null to use - the clustered index and just the key, no off-record data. */ - m_prebuilt->index = index; - dtuple_set_n_fields(m_prebuilt->search_tuple, 0); - m_prebuilt->read_just_key = 1; - build_template(false); - - /* Count the records in the clustered index */ - ret = row_scan_index_for_mysql(m_prebuilt, index, false, &n_rows); - reset_template(); - switch (ret) { - case DB_SUCCESS: - break; - case DB_DEADLOCK: - case DB_LOCK_TABLE_FULL: - case DB_LOCK_WAIT_TIMEOUT: - *num_rows = HA_POS_ERROR; - DBUG_RETURN(convert_error_code_to_mysql(ret, 0, m_user_thd)); - case DB_INTERRUPTED: - *num_rows = HA_POS_ERROR; - DBUG_RETURN(HA_ERR_QUERY_INTERRUPTED); - default: - /* No other error besides the three below is returned from - row_scan_index_for_mysql(). Make a debug catch. */ - *num_rows = HA_POS_ERROR; - ut_ad(0); - DBUG_RETURN(-1); - } - - m_prebuilt->trx->op_info = ""; - - if (thd_killed(m_user_thd)) { - *num_rows = HA_POS_ERROR; - DBUG_RETURN(HA_ERR_QUERY_INTERRUPTED); - } - - *num_rows= n_rows; - DBUG_RETURN(0); -} -#endif /* MYSQL_57_SELECT_COUNT_OPTIMIZATION */ - /*********************************************************************//** Estimates the number of index records in a range. @return estimated number of rows */ @@ -15586,7 +15476,7 @@ ha_innobase::check( ret = row_count_rtree_recs(m_prebuilt, &n_rows); } else { ret = row_scan_index_for_mysql( - m_prebuilt, index, true, &n_rows); + m_prebuilt, index, &n_rows); } DBUG_EXECUTE_IF( diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index 463717ee6b2..a5e9c533fb0 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -219,9 +219,6 @@ public: ha_rows estimate_rows_upper_bound(); - // JAN: TODO: MySQL 5.7 - // int records(ha_rows* num_rows); - void update_create_info(HA_CREATE_INFO* create_info); int create( diff --git a/storage/innobase/handler/ha_innopart.cc b/storage/innobase/handler/ha_innopart.cc index 7abc1f29b60..1041871805e 100644 --- a/storage/innobase/handler/ha_innopart.cc +++ b/storage/innobase/handler/ha_innopart.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2014, 2016, Oracle and/or its affiliates. All rights reserved. +Copyright (c) 2014, 2017, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2016, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -3004,41 +3004,6 @@ ha_innopart::truncate() DBUG_RETURN(error); } -/** Total number of rows in all used partitions. -Returns the exact number of records that this client can see using this -handler object. -@param[out] num_rows Number of rows. -@return 0 or error number. */ -int -ha_innopart::records( - ha_rows* num_rows) -{ - ha_rows n_rows; - int err; - DBUG_ENTER("ha_innopart::records()"); - - *num_rows = 0; - - /* The index scan is probably so expensive, so the overhead - of the rest of the function is neglectable for each partition. - So no current reason for optimizing this further. */ - - for (uint i = m_part_info->get_first_used_partition(); - i < m_tot_parts; - i = m_part_info->get_next_used_partition(i)) { - - set_partition(i); - err = ha_innobase::records(&n_rows); - update_partition(i); - if (err != 0) { - *num_rows = HA_POS_ERROR; - DBUG_RETURN(err); - } - *num_rows += n_rows; - } - DBUG_RETURN(0); -} - /** Estimates the number of index records in a range. @param[in] keynr Index number. @param[in] min_key Start key value (or NULL). diff --git a/storage/innobase/handler/ha_innopart.h b/storage/innobase/handler/ha_innopart.h index 5f3f8eaa318..67db9e07150 100644 --- a/storage/innobase/handler/ha_innopart.h +++ b/storage/innobase/handler/ha_innopart.h @@ -1221,10 +1221,6 @@ protected: uchar* record, uchar* pos); - int - records( - ha_rows* num_rows); - int index_next( uchar* record) diff --git a/storage/innobase/include/ha_prototypes.h b/storage/innobase/include/ha_prototypes.h index 6992f4a6689..7b3726b1fef 100644 --- a/storage/innobase/include/ha_prototypes.h +++ b/storage/innobase/include/ha_prototypes.h @@ -38,7 +38,6 @@ simple headers. class THD; // JAN: TODO missing features: -#undef MYSQL_57_SELECT_COUNT_OPTIMIZATION #undef MYSQL_FT_INIT_EXT #undef MYSQL_INNODB_PARTITIONING #undef MYSQL_PFS diff --git a/storage/innobase/include/row0mysql.h b/storage/innobase/include/row0mysql.h index 59bd786d25c..0557b585ae1 100644 --- a/storage/innobase/include/row0mysql.h +++ b/storage/innobase/include/row0mysql.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -540,9 +540,6 @@ row_scan_index_for_mysql( row_prebuilt_t* prebuilt, /*!< in: prebuilt struct in MySQL handle */ const dict_index_t* index, /*!< in: index */ - bool check_keys, /*!< in: true=check for mis- - ordered or duplicate records, - false=count the rows only */ ulint* n_rows) /*!< out: number of entries seen in the consistent read */ MY_ATTRIBUTE((warn_unused_result)); diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index d34b42e50ba..20a8777b71e 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under @@ -5046,9 +5046,6 @@ row_scan_index_for_mysql( row_prebuilt_t* prebuilt, /*!< in: prebuilt struct in MySQL handle */ const dict_index_t* index, /*!< in: index */ - bool check_keys, /*!< in: true=check for mis- - ordered or duplicate records, - false=count the rows only */ ulint* n_rows) /*!< out: number of entries seen in the consistent read */ { @@ -5115,7 +5112,7 @@ loop: goto func_exit; default: { - const char* doing = check_keys? "CHECK TABLE" : "COUNT(*)"; + const char* doing = "CHECK TABLE"; ib::warn() << doing << " on index " << index->name << " of" " table " << index->table->name << " returned " << ret; /* fall through (this error is ignored by CHECK TABLE) */ @@ -5131,9 +5128,6 @@ func_exit: *n_rows = *n_rows + 1; - if (!check_keys) { - goto next_rec; - } /* else this code is doing handler::check() for CHECK TABLE */ /* row_search... returns the index record in buf, record origin offset @@ -5215,7 +5209,6 @@ not_ok: } } -next_rec: ret = row_search_for_mysql( buf, PAGE_CUR_G, prebuilt, 0, ROW_SEL_NEXT); -- cgit v1.2.1 From 4b24467ff37d6db82500e736e832d0a53842ac9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 28 Apr 2017 12:23:35 +0300 Subject: MDEV-12602 InnoDB: Failing assertion: space->n_pending_ops == 0 This fixes a regression caused by MDEV-12428. When we introduced a variant of fil_space_acquire() that could increment space->n_pending_ops after space->stop_new_ops was set, the logic of fil_check_pending_operations() was broken. fil_space_t::n_pending_ios: A new field to track read or write access from the buffer pool routines immediately before a block write or after a block read in the file system. fil_space_acquire_for_io(), fil_space_release_for_io(): Similar to fil_space_acquire_silent() and fil_space_release(), but modify fil_space_t::n_pending_ios instead of fil_space_t::n_pending_ops. fil_space_free_low(): Wait for space->n_pending_ios to reach 0, to avoid accessing freed data in a concurrent thread. Future calls to fil_space_acquire_for_io() will not find this tablespace, because it will already have been detached from fil_system. Adjust a number of places accordingly, and remove some redundant tablespace lookups. FIXME: buf_page_check_corrupt() should take a tablespace from fil_space_acquire_for_io() as a parameter. This will be done in the 10.1 version of this patch and merged from there. That depends on MDEV-12253, which has not been merged from 10.1 yet. --- storage/innobase/buf/buf0buf.cc | 52 +++++++++++++++++--------- storage/innobase/buf/buf0flu.cc | 10 ++++- storage/innobase/fil/fil0crypt.cc | 4 +- storage/innobase/fil/fil0fil.cc | 64 +++++++++++++++++++++++++------- storage/innobase/fil/fil0pagecompress.cc | 5 ++- storage/innobase/include/buf0buf.h | 11 ------ storage/innobase/include/fil0fil.h | 49 +++++++++++++++--------- 7 files changed, 130 insertions(+), 65 deletions(-) (limited to 'storage') diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 56f39a78acb..502df6c6160 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -405,10 +405,22 @@ buf_pool_register_chunk( /** Decrypt a page. @param[in,out] bpage Page control block +@param[in,out] space tablespace @return whether the operation was successful */ static bool -buf_page_decrypt_after_read(buf_page_t* bpage); +buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) + MY_ATTRIBUTE((nonnull)); + +/** Check if page is maybe compressed, encrypted or both when we encounter +corrupted page. Note that we can't be 100% sure if page is corrupted +or decrypt/decompress just failed. +@param[in,out] bpage Page +@return true if page corrupted, false if not */ +static +bool +buf_page_check_corrupt(buf_page_t* bpage) + MY_ATTRIBUTE((nonnull, warn_unused_result)); /* prototypes for new functions added to ha_innodb.cc */ trx_t* innobase_get_trx(); @@ -5844,16 +5856,14 @@ buf_mark_space_corrupt( return(ret); } -/********************************************************************//** -Check if page is maybe compressed, encrypted or both when we encounter +/** Check if page is maybe compressed, encrypted or both when we encounter corrupted page. Note that we can't be 100% sure if page is corrupted or decrypt/decompress just failed. @param[in,out] bpage Page @return true if page corrupted, false if not */ -UNIV_INTERN +static bool -buf_page_check_corrupt( - buf_page_t* bpage) +buf_page_check_corrupt(buf_page_t* bpage) { byte* dst_frame = (bpage->zip.data) ? bpage->zip.data : ((buf_block_t*) bpage)->frame; @@ -5956,8 +5966,13 @@ buf_page_io_complete( ulint read_space_id; ut_ad(bpage->zip.data != NULL || ((buf_block_t*)bpage)->frame != NULL); + fil_space_t* space = fil_space_acquire_for_io( + bpage->id.space()); + if (!space) { + return false; + } - buf_page_decrypt_after_read(bpage); + buf_page_decrypt_after_read(bpage, space); if (bpage->size.is_compressed()) { frame = bpage->zip.data; @@ -6031,21 +6046,17 @@ database_corrupted: && buf_mark_space_corrupt(bpage)) { ib::info() << "Simulated IMPORT " "corruption"; + fil_space_release_for_io(space); return(true); } goto page_not_corrupt; ); if (!bpage->encrypted) { - fil_system_enter(); - fil_space_t* space = fil_space_get_by_id(bpage->id.space()); - fil_system_exit(); - ib::error() << "Database page corruption on disk" " or a failed file read of tablespace " - << (space->name ? space->name : "NULL") - << " page " << bpage->id + << space->name << " page " << bpage->id << ". You may have to recover from " << "a backup."; @@ -6074,6 +6085,7 @@ database_corrupted: if (bpage->id.space() > TRX_SYS_SPACE && buf_mark_space_corrupt(bpage)) { + fil_space_release_for_io(space); return(false); } else { if (!bpage->encrypted) { @@ -6099,6 +6111,7 @@ database_corrupted: ut_error; } + fil_space_release_for_io(space); return(false); } } @@ -6142,6 +6155,8 @@ database_corrupted: } } + + fil_space_release_for_io(space); } else { /* io_type == BUF_IO_WRITE */ if (bpage->slot) { @@ -7502,11 +7517,15 @@ buf_page_encrypt_before_write( /** Decrypt a page. @param[in,out] bpage Page control block +@param[in,out] space tablespace @return whether the operation was successful */ static bool -buf_page_decrypt_after_read(buf_page_t* bpage) +buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) { + ut_ad(space->n_pending_ios > 0); + ut_ad(space->id == bpage->id.space()); + bool compressed = bpage->size.is_compressed(); const page_size_t& size = bpage->size; byte* dst_frame = compressed ? bpage->zip.data : @@ -7525,12 +7544,10 @@ buf_page_decrypt_after_read(buf_page_t* bpage) return (true); } - FilSpace space(bpage->id.space(), true); - /* Page is encrypted if encryption information is found from tablespace and page contains used key_version. This is true also for pages first compressed and then encrypted. */ - if (!space() || !space()->crypt_data) { + if (!space->crypt_data) { key_version = 0; } @@ -7602,6 +7619,7 @@ buf_page_decrypt_after_read(buf_page_t* bpage) } } + ut_ad(space->n_pending_ios > 0); return (success); } diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 366e92a7112..d0c0316bf13 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1006,7 +1006,7 @@ buf_flush_write_block_low( buf_flush_t flush_type, /*!< in: type of flush */ bool sync) /*!< in: true if sync IO request */ { - fil_space_t* space = fil_space_acquire(bpage->id.space(), true); + fil_space_t* space = fil_space_acquire_for_io(bpage->id.space()); if (!space) { return; } @@ -1121,10 +1121,16 @@ buf_flush_write_block_low( /* true means we want to evict this page from the LRU list as well. */ + /* The tablespace could already have been dropped, + because fil_io(request, sync) would already have + decremented the node->n_pending. However, + buf_page_io_complete() only needs to look up the + tablespace during read requests, not during writes. */ + ut_ad(buf_page_get_io_fix(bpage) == BUF_IO_WRITE); buf_page_io_complete(bpage, true); } - fil_space_release(space); + fil_space_release_for_io(space); /* Increment the counter of I/O operations used for selecting LRU policy. */ diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 016b7c3d24e..033ddd39f90 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -629,7 +629,7 @@ fil_space_encrypt( fil_space_crypt_t* crypt_data = space->crypt_data; const page_size_t page_size(space->flags); - ut_ad(space->n_pending_ops); + ut_ad(space->n_pending_ios > 0); byte* tmp = fil_encrypt_buf(crypt_data, space->id, offset, lsn, src_frame, page_size, dst_frame); @@ -821,7 +821,7 @@ fil_space_decrypt( *decrypted = false; ut_ad(space->crypt_data != NULL && space->crypt_data->is_encrypted()); - ut_ad(space->n_pending_ops > 0); + ut_ad(space->n_pending_ios > 0); bool encrypted = fil_space_decrypt(space->crypt_data, tmp_frame, page_size, src_frame, &err); diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 84c2f07b361..7a169c0bf2c 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1506,6 +1506,13 @@ fil_space_free_low( ut_ad(srv_fast_shutdown == 2 || !srv_was_started || space->max_lsn == 0); + /* Wait for fil_space_release_for_io(); after + fil_space_detach(), the tablespace cannot be found, so + fil_space_acquire_for_io() would return NULL */ + while (space->n_pending_ios) { + os_thread_sleep(100); + } + for (fil_node_t* node = UT_LIST_GET_FIRST(space->chain); node != NULL; ) { ut_d(space->size -= node->size); @@ -2267,13 +2274,11 @@ Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID @param[in] silent whether to silently ignore missing tablespaces -@param[in] for_io whether to look up the tablespace while performing I/O - (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted or truncated */ inline fil_space_t* -fil_space_acquire_low(ulint id, bool silent, bool for_io = false) +fil_space_acquire_low(ulint id, bool silent) { fil_space_t* space; @@ -2286,7 +2291,7 @@ fil_space_acquire_low(ulint id, bool silent, bool for_io = false) ib::warn() << "Trying to access missing" " tablespace " << id; } - } else if (!for_io && space->is_stopping()) { + } else if (space->is_stopping()) { space = NULL; } else { space->n_pending_ops++; @@ -2301,14 +2306,12 @@ fil_space_acquire_low(ulint id, bool silent, bool for_io = false) Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID -@param[in] for_io whether to look up the tablespace while performing I/O - (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted or truncated */ fil_space_t* -fil_space_acquire(ulint id, bool for_io) +fil_space_acquire(ulint id) { - return(fil_space_acquire_low(id, false, for_io)); + return(fil_space_acquire_low(id, false)); } /** Acquire a tablespace that may not exist. @@ -2335,6 +2338,39 @@ fil_space_release(fil_space_t* space) mutex_exit(&fil_system->mutex); } +/** Acquire a tablespace for reading or writing a block, +when it could be dropped concurrently. +@param[in] id tablespace ID +@return the tablespace +@retval NULL if missing */ +fil_space_t* +fil_space_acquire_for_io(ulint id) +{ + mutex_enter(&fil_system->mutex); + + fil_space_t* space = fil_space_get_by_id(id); + + if (space) { + space->n_pending_ios++; + } + + mutex_exit(&fil_system->mutex); + + return(space); +} + +/** Release a tablespace acquired with fil_space_acquire_for_io(). +@param[in,out] space tablespace to release */ +void +fil_space_release_for_io(fil_space_t* space) +{ + mutex_enter(&fil_system->mutex); + ut_ad(space->magic_n == FIL_SPACE_MAGIC_N); + ut_ad(space->n_pending_ios > 0); + space->n_pending_ios--; + mutex_exit(&fil_system->mutex); +} + /********************************************************//** Creates the database directory for a table if it does not exist yet. */ void @@ -2836,15 +2872,15 @@ enum fil_operation_t { @return 0 if no operations else count + 1. */ static ulint -fil_check_pending_ops( - fil_space_t* space, - ulint count) +fil_check_pending_ops(const fil_space_t* space, ulint count) { ut_ad(mutex_own(&fil_system->mutex)); - const ulint n_pending_ops = space ? space->n_pending_ops : 0; + if (space == NULL) { + return 0; + } - if (n_pending_ops) { + if (ulint n_pending_ops = space->n_pending_ops) { if (count > 5000) { ib::warn() << "Trying to close/delete/truncate" @@ -5531,7 +5567,7 @@ fil_flush( void fil_flush(fil_space_t* space) { - ut_ad(space->n_pending_ops > 0); + ut_ad(space->n_pending_ios > 0); ut_ad(space->purpose == FIL_TYPE_TABLESPACE || space->purpose == FIL_TYPE_IMPORT); diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index 41966cf005a..cb3e50745a7 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -631,11 +631,11 @@ err_exit: /* Note that as we have found the page is corrupted, so all this could be incorrect. */ ulint space_id = mach_read_from_4(buf+FIL_PAGE_SPACE_ID); - const FilSpace space(space_id, true); + fil_space_t* space = fil_space_acquire_for_io(space_id); ib::error() << "Corruption: Page is marked as compressed" << " space: " << space_id << " name: " - << (space() ? space()->name : "NULL") + << (space ? space->name : "NULL") << " but uncompress failed with error: " << err << " size: " << actual_size << " len: " << len @@ -643,4 +643,5 @@ err_exit: << fil_get_compression_alg_name(compression_alg) << "."; buf_page_print(buf, univ_page_size, 0); + fil_space_release_for_io(space); } diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 6832c133d58..746ac6a25bd 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -835,17 +835,6 @@ buf_page_is_checksum_valid_none( ) MY_ATTRIBUTE((nonnull(1), warn_unused_result)); -/********************************************************************//** -Check if page is maybe compressed, encrypted or both when we encounter -corrupted page. Note that we can't be 100% sure if page is corrupted -or decrypt/decompress just failed. -@param[in] bpage Page -@return true if page corrupted, false if not */ -bool -buf_page_check_corrupt( - buf_page_t* bpage) /*!< in/out: buffer page read from disk */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); - /** Checks if a page contains only zeroes. @param[in] read_buf database page @param[in] page_size page size diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 188e0b0bd3b..cef5b578a4a 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -142,14 +142,21 @@ struct fil_space_t { ulint n_pending_flushes; /*!< this is positive when flushing the tablespace to disk; dropping of the tablespace is forbidden if this is positive */ - ulint n_pending_ops;/*!< this is positive when we - have pending operations against this - tablespace. The pending operations can - be ibuf merges or lock validation code - trying to read a block. - Dropping of the tablespace is forbidden - if this is positive. - Protected by fil_system->mutex. */ + /** Number of pending buffer pool operations accessing the tablespace + without holding a table lock or dict_operation_lock S-latch + that would prevent the table (and tablespace) from being + dropped. An example is change buffer merge. + The tablespace cannot be dropped while this is nonzero, + or while fil_node_t::n_pending is nonzero. + Protected by fil_system->mutex. */ + ulint n_pending_ops; + /** Number of pending block read or write operations + (when a write is imminent or a read has recently completed). + The tablespace object cannot be freed while this is nonzero, + but it can be detached from fil_system. + Note that fil_node_t::n_pending tracks actual pending I/O requests. + Protected by fil_system->mutex. */ + ulint n_pending_ios; hash_node_t hash; /*!< hash chain node */ hash_node_t name_hash;/*!< hash chain the name_hash table */ rw_lock_t latch; /*!< latch protecting the file space storage @@ -726,12 +733,10 @@ MY_ATTRIBUTE((warn_unused_result)); Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID -@param[in] for_io whether to look up the tablespace while performing I/O - (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted or truncated */ fil_space_t* -fil_space_acquire(ulint id, bool for_io = false) +fil_space_acquire(ulint id) MY_ATTRIBUTE((warn_unused_result)); /** Acquire a tablespace that may not exist. @@ -749,6 +754,19 @@ fil_space_acquire_silent(ulint id) void fil_space_release(fil_space_t* space); +/** Acquire a tablespace for reading or writing a block, +when it could be dropped concurrently. +@param[in] id tablespace ID +@return the tablespace +@retval NULL if missing */ +fil_space_t* +fil_space_acquire_for_io(ulint id); + +/** Release a tablespace acquired with fil_space_acquire_for_io(). +@param[in,out] space tablespace to release */ +void +fil_space_release_for_io(fil_space_t* space); + /** Return the next fil_space_t. Once started, the caller must keep calling this until it returns NULL. fil_space_acquire() and fil_space_release() are invoked here which @@ -785,12 +803,9 @@ public: /** Constructor: Look up the tablespace and increment the reference count if found. - @param[in] space_id tablespace ID - @param[in] for_io whether to look up the tablespace - while performing I/O - (possibly executing TRUNCATE) */ - explicit FilSpace(ulint space_id, bool for_io = false) - : m_space(fil_space_acquire(space_id, for_io)) {} + @param[in] space_id tablespace ID */ + explicit FilSpace(ulint space_id) + : m_space(fil_space_acquire(space_id)) {} /** Assignment operator: This assumes that fil_space_acquire() has already been done for the fil_space_t. The caller must -- cgit v1.2.1 From 0072d2e9a162e87c50988d9f820ae40fdfdf7251 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 7 Apr 2017 15:01:27 +0200 Subject: InnoDB cleanup: remove a bunch of #ifdef UNIV_INNOCHECKSUM innochecksum uses global variables. great, let's use them all the way down, instead of passing them as arguments to innodb internals, conditionally modifying function prototypes with #ifdefs --- storage/innobase/buf/buf0buf.cc | 199 +++++++++-------------------------- storage/innobase/fil/fil0crypt.cc | 25 +---- storage/innobase/include/buf0buf.h | 46 +------- storage/innobase/include/fil0crypt.h | 4 - storage/innobase/include/page0zip.h | 14 +-- storage/innobase/include/univ.i | 6 ++ storage/innobase/page/page0zip.cc | 30 ++---- 7 files changed, 72 insertions(+), 252 deletions(-) (limited to 'storage') diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 502df6c6160..754b2cd1c25 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -612,10 +612,6 @@ buf_page_is_zeroes( @param[in] read_buf database page @param[in] checksum_field1 new checksum field @param[in] checksum_field2 old checksum field -@param[in] page_no page number of given read_buf -@param[in] is_log_enabled true if log option is enabled -@param[in] log_file file pointer to log_file -@param[in] curr_algo current checksum algorithm @param[in] use_legacy_big_endian use legacy big endian algorithm @return true if the page is in crc32 checksum format. */ bool @@ -623,24 +619,18 @@ buf_page_is_checksum_valid_crc32( const byte* read_buf, ulint checksum_field1, ulint checksum_field2, -#ifdef UNIV_INNOCHECKSUM - uintmax_t page_no, - bool is_log_enabled, - FILE* log_file, - const srv_checksum_algorithm_t curr_algo, -#endif /* UNIV_INNOCHECKSUM */ bool use_legacy_big_endian) { const uint32_t crc32 = buf_calc_page_crc32(read_buf, use_legacy_big_endian); #ifdef UNIV_INNOCHECKSUM - if (is_log_enabled - && curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { + if (log_file + && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { fprintf(log_file, "page::%lu;" " crc32 calculated = %u;" " recorded checksum field1 = %lu recorded" - " checksum field2 =%lu\n", page_no, + " checksum field2 =%lu\n", cur_page_num, crc32, checksum_field1, checksum_field2); } #endif /* UNIV_INNOCHECKSUM */ @@ -671,23 +661,12 @@ invalid: @param[in] read_buf database page @param[in] checksum_field1 new checksum field @param[in] checksum_field2 old checksum field -@param[in] page_no page number of given read_buf -@param[in] is_log_enabled true if log option is enabled -@param[in] log_file file pointer to log_file -@param[in] curr_algo current checksum algorithm @return true if the page is in innodb checksum format. */ bool buf_page_is_checksum_valid_innodb( const byte* read_buf, ulint checksum_field1, - ulint checksum_field2 -#ifdef UNIV_INNOCHECKSUM - ,uintmax_t page_no, - bool is_log_enabled, - FILE* log_file, - const srv_checksum_algorithm_t curr_algo -#endif /* UNIV_INNOCHECKSUM */ - ) + ulint checksum_field2) { /* There are 2 valid formulas for checksum_field2 (old checksum field) which algo=innodb could have @@ -703,31 +682,31 @@ buf_page_is_checksum_valid_innodb( ulint new_checksum = buf_calc_page_new_checksum(read_buf); #ifdef UNIV_INNOCHECKSUM - if (is_log_enabled - && curr_algo == SRV_CHECKSUM_ALGORITHM_INNODB) { + if (log_file + && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB) { fprintf(log_file, "page::%lu;" " old style: calculated =" " %lu; recorded = %lu\n", - page_no, old_checksum, + cur_page_num, old_checksum, checksum_field2); fprintf(log_file, "page::%lu;" " new style: calculated =" " %lu; crc32 = %u; recorded = %lu\n", - page_no, new_checksum, + cur_page_num, new_checksum, buf_calc_page_crc32(read_buf), checksum_field1); } - if (is_log_enabled - && curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { + if (log_file + && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { fprintf(log_file, "page::%lu;" " old style: calculated =" " %lu; recorded checksum = %lu\n", - page_no, old_checksum, + cur_page_num, old_checksum, checksum_field2); fprintf(log_file, "page::%lu;" " new style: calculated =" " %lu; recorded checksum = %lu\n", - page_no, new_checksum, + cur_page_num, new_checksum, checksum_field1); } #endif /* UNIV_INNOCHECKSUM */ @@ -767,23 +746,12 @@ buf_page_is_checksum_valid_innodb( @param[in] read_buf database page @param[in] checksum_field1 new checksum field @param[in] checksum_field2 old checksum field -@param[in] page_no page number of given read_buf -@param[in] is_log_enabled true if log option is enabled -@param[in] log_file file pointer to log_file -@param[in] curr_algo current checksum algorithm @return true if the page is in none checksum format. */ bool buf_page_is_checksum_valid_none( const byte* read_buf, ulint checksum_field1, - ulint checksum_field2 -#ifdef UNIV_INNOCHECKSUM - ,uintmax_t page_no, - bool is_log_enabled, - FILE* log_file, - const srv_checksum_algorithm_t curr_algo -#endif /* UNIV_INNOCHECKSUM */ - ) + ulint checksum_field2) { #ifndef DBUG_OFF if (checksum_field1 != checksum_field2 @@ -799,13 +767,13 @@ buf_page_is_checksum_valid_none( #endif /* DBUG_OFF */ #ifdef UNIV_INNOCHECKSUM - if (is_log_enabled - && curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_NONE) { + if (log_file + && srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_NONE) { fprintf(log_file, "page::%lu; none checksum: calculated" " = %lu; recorded checksum_field1 = %lu" " recorded checksum_field2 = %lu\n", - page_no, BUF_NO_CHECKSUM_MAGIC, + cur_page_num, BUF_NO_CHECKSUM_MAGIC, checksum_field1, checksum_field2); } #endif /* UNIV_INNOCHECKSUM */ @@ -820,10 +788,6 @@ the LSN @param[in] read_buf database page @param[in] page_size page size @param[in] space tablespace -@param[in] page_no page number of given read_buf -@param[in] strict_check true if strict-check option is enabled -@param[in] is_log_enabled true if log option is enabled -@param[in] log_file file pointer to log_file @return TRUE if corrupted */ bool buf_page_is_corrupted( @@ -831,12 +795,6 @@ buf_page_is_corrupted( const byte* read_buf, const page_size_t& page_size, const fil_space_t* space -#ifdef UNIV_INNOCHECKSUM - ,uintmax_t page_no, - bool strict_check, - bool is_log_enabled, - FILE* log_file -#endif /* UNIV_INNOCHECKSUM */ ) { ulint checksum_field1; @@ -926,15 +884,8 @@ buf_page_is_corrupted( } if (page_size.is_compressed()) { -#ifdef UNIV_INNOCHECKSUM - return(!page_zip_verify_checksum(read_buf, - page_size.physical(), - page_no, strict_check, - is_log_enabled, log_file)); -#else return(!page_zip_verify_checksum(read_buf, page_size.physical())); -#endif /* UNIV_INNOCHECKSUM */ } checksum_field1 = mach_read_from_4( @@ -976,10 +927,10 @@ buf_page_is_corrupted( } #ifdef UNIV_INNOCHECKSUM if (i >= page_size.logical()) { - if (is_log_enabled) { + if (log_file) { fprintf(log_file, "Page::%lu" " is empty and uncorrupted\n", - page_no); + cur_page_num); } return(false); } @@ -1005,20 +956,13 @@ buf_page_is_corrupted( case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: if (buf_page_is_checksum_valid_crc32(read_buf, - checksum_field1, checksum_field2, -#ifdef UNIV_INNOCHECKSUM - page_no, is_log_enabled, log_file, curr_algo, -#endif /* UNIV_INNOCHECKSUM */ - false)) { + checksum_field1, checksum_field2, false)) { return(false); } if (buf_page_is_checksum_valid_none(read_buf, - checksum_field1, checksum_field2 -#ifdef UNIV_INNOCHECKSUM - , page_no, is_log_enabled, log_file, curr_algo)) { -#else /* UNIV_INNOCHECKSUM */ - )) { + checksum_field1, checksum_field2)) { +#ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { page_warn_strict_checksum( @@ -1026,20 +970,20 @@ buf_page_is_corrupted( SRV_CHECKSUM_ALGORITHM_NONE, page_id); } -#endif /* UNIV_INNOCHECKSUM */ +#endif /* !UNIV_INNOCHECKSUM */ #ifdef UNIV_INNOCHECKSUM - if (is_log_enabled) { + if (log_file) { fprintf(log_file, "page::%lu;" " old style: calculated = " ULINTPF ";" - " recorded = " ULINTPF "\n", page_no, + " recorded = " ULINTPF "\n", cur_page_num, buf_calc_page_old_checksum(read_buf), checksum_field2); fprintf(log_file, "page::%lu;" " new style: calculated = " ULINTPF ";" " crc32 = %u; recorded = " ULINTPF "\n", - page_no, + cur_page_num, buf_calc_page_new_checksum(read_buf), buf_calc_page_crc32(read_buf), checksum_field1); @@ -1055,11 +999,7 @@ buf_page_is_corrupted( Otherwise we check innodb checksum first. */ if (legacy_big_endian_checksum) { if (buf_page_is_checksum_valid_crc32(read_buf, - checksum_field1, checksum_field2, -#ifdef UNIV_INNOCHECKSUM - page_no, is_log_enabled, log_file, curr_algo, -#endif /* UNIV_INNOCHECKSUM */ - true)) { + checksum_field1, checksum_field2, true)) { return(false); } @@ -1067,11 +1007,8 @@ buf_page_is_corrupted( } if (buf_page_is_checksum_valid_innodb(read_buf, - checksum_field1, checksum_field2 -#ifdef UNIV_INNOCHECKSUM - , page_no, is_log_enabled, log_file, curr_algo)) { -#else /* UNIV_INNOCHECKSUM */ - )) { + checksum_field1, checksum_field2)) { +#ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { page_warn_strict_checksum( @@ -1079,27 +1016,23 @@ buf_page_is_corrupted( SRV_CHECKSUM_ALGORITHM_INNODB, page_id); } -#endif /* UNIV_INNOCHECKSUM */ +#endif /* !UNIV_INNOCHECKSUM */ return(false); } /* If legacy checksum is not checked, do it now. */ if (!legacy_checksum_checked && buf_page_is_checksum_valid_crc32( - read_buf, checksum_field1, checksum_field2, -#ifdef UNIV_INNOCHECKSUM - page_no, is_log_enabled, log_file, curr_algo, -#endif /* UNIV_INNOCHECKSUM */ - true)) { + read_buf, checksum_field1, checksum_field2, true)) { legacy_big_endian_checksum = true; return(false); } #ifdef UNIV_INNOCHECKSUM - if (is_log_enabled) { + if (log_file) { fprintf(log_file, "Fail; page %lu" " invalid (fails crc32 checksum)\n", - page_no); + cur_page_num); } #endif /* UNIV_INNOCHECKSUM */ return(true); @@ -1108,20 +1041,13 @@ buf_page_is_corrupted( case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: if (buf_page_is_checksum_valid_innodb(read_buf, - checksum_field1, checksum_field2 -#ifdef UNIV_INNOCHECKSUM - , page_no, is_log_enabled, log_file, curr_algo -#endif /* UNIV_INNOCHECKSUM */ - )) { + checksum_field1, checksum_field2)) { return(false); } if (buf_page_is_checksum_valid_none(read_buf, - checksum_field1, checksum_field2 -#ifdef UNIV_INNOCHECKSUM - , page_no, is_log_enabled, log_file, curr_algo)) { -#else /* UNIV_INNOCHECKSUM */ - )) { + checksum_field1, checksum_field2)) { +#ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { page_warn_strict_checksum( @@ -1129,19 +1055,19 @@ buf_page_is_corrupted( SRV_CHECKSUM_ALGORITHM_NONE, page_id); } -#endif /* UNIV_INNOCHECKSUM */ +#endif /* !UNIV_INNOCHECKSUM */ #ifdef UNIV_INNOCHECKSUM - if (is_log_enabled) { + if (log_file) { fprintf(log_file, "page::%lu;" " old style: calculated = %lu;" - " recorded = %lu\n", page_no, + " recorded = %lu\n", cur_page_num, buf_calc_page_old_checksum(read_buf), checksum_field2); fprintf(log_file, "page::%lu;" " new style: calculated = %lu;" " crc32 = %u; recorded = %lu\n", - page_no, + cur_page_num, buf_calc_page_new_checksum(read_buf), buf_calc_page_crc32(read_buf), checksum_field1); @@ -1150,19 +1076,11 @@ buf_page_is_corrupted( return(false); } -#ifdef UNIV_INNOCHECKSUM - if (buf_page_is_checksum_valid_crc32(read_buf, - checksum_field1, checksum_field2, - page_no, is_log_enabled, log_file, curr_algo, false) - || buf_page_is_checksum_valid_crc32(read_buf, - checksum_field1, checksum_field2, - page_no, is_log_enabled, log_file, curr_algo, true)) { -#else /* UNIV_INNOCHECKSUM */ if (buf_page_is_checksum_valid_crc32(read_buf, checksum_field1, checksum_field2, false) || buf_page_is_checksum_valid_crc32(read_buf, checksum_field1, checksum_field2, true)) { - +#ifndef UNIV_INNOCHECKSUM if (curr_algo == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { page_warn_strict_checksum( @@ -1170,16 +1088,16 @@ buf_page_is_corrupted( SRV_CHECKSUM_ALGORITHM_CRC32, page_id); } -#endif /* UNIV_INNOCHECKSUM */ +#endif /* !UNIV_INNOCHECKSUM */ return(false); } #ifdef UNIV_INNOCHECKSUM - if (is_log_enabled) { + if (log_file) { fprintf(log_file, "Fail; page %lu" " invalid (fails innodb checksum)\n", - page_no); + cur_page_num); } #endif /* UNIV_INNOCHECKSUM */ return(true); @@ -1187,54 +1105,39 @@ buf_page_is_corrupted( case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: if (buf_page_is_checksum_valid_none(read_buf, - checksum_field1, checksum_field2 -#ifdef UNIV_INNOCHECKSUM - , page_no, is_log_enabled, log_file, curr_algo -#endif /* UNIV_INNOCHECKSUM */ - )) { + checksum_field1, checksum_field2)) { return(false); } -#ifdef UNIV_INNOCHECKSUM - if (buf_page_is_checksum_valid_crc32(read_buf, - checksum_field1, checksum_field2, - page_no, is_log_enabled, log_file, curr_algo, false) - || buf_page_is_checksum_valid_crc32(read_buf, - checksum_field1, checksum_field2, - page_no, is_log_enabled, log_file, curr_algo, true)) { -#else /* UNIV_INNOCHECKSUM */ if (buf_page_is_checksum_valid_crc32(read_buf, checksum_field1, checksum_field2, false) || buf_page_is_checksum_valid_crc32(read_buf, checksum_field1, checksum_field2, true)) { - +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_CRC32, page_id); -#endif /* UNIV_INNOCHECKSUM */ +#endif /* !UNIV_INNOCHECKSUM */ return(false); } if (buf_page_is_checksum_valid_innodb(read_buf, - checksum_field1, checksum_field2 -#ifdef UNIV_INNOCHECKSUM - , page_no, is_log_enabled, log_file, curr_algo)) { -#else /* UNIV_INNOCHECKSUM */ - )) { + checksum_field1, checksum_field2)) { +#ifndef UNIV_INNOCHECKSUM page_warn_strict_checksum( curr_algo, SRV_CHECKSUM_ALGORITHM_INNODB, page_id); -#endif /* UNIV_INNOCHECKSUM */ +#endif /* !UNIV_INNOCHECKSUM */ return(false); } #ifdef UNIV_INNOCHECKSUM - if (is_log_enabled) { + if (log_file) { fprintf(log_file, "Fail; page %lu" " invalid (fails none checksum)\n", - page_no); + cur_page_num); } #endif /* UNIV_INNOCHECKSUM */ return(true); diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 033ddd39f90..b031a77baa8 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -2492,10 +2492,6 @@ bool fil_space_verify_crypt_checksum( byte* page, const page_size_t& page_size, -#ifdef UNIV_INNOCHECKSUM - bool strict_check, /*!< --strict-check */ - FILE* log_file, /*!< --log */ -#endif /* UNIV_INNOCHECKSUM */ ulint space, ulint offset) { @@ -2541,14 +2537,7 @@ fil_space_verify_crypt_checksum( mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); bool valid = page_zip_verify_checksum(page, - page_size.physical() -#ifdef UNIV_INNOCHECKSUM - , offset, - strict_check, - log_file != NULL, - log_file -#endif - ); + page_size.physical()); mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, old); @@ -2594,19 +2583,11 @@ fil_space_verify_crypt_checksum( ulint checksum2 = mach_read_from_4( page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); -#ifdef UNIV_INNOCHECKSUM -# define CKARGS page, checksum1, checksum2, \ - offset, log_file != NULL, log_file, algorithm -#else -# define CKARGS page, checksum1, checksum2 -#endif - bool valid = buf_page_is_checksum_valid_crc32( - CKARGS, false + page, checksum1, checksum2, false /* FIXME: also try the original crc32 that was buggy on big-endian architectures? */) - || buf_page_is_checksum_valid_innodb(CKARGS); -#undef CKARGS + || buf_page_is_checksum_valid_innodb(page, checksum1, checksum2); if (encrypted && valid) { /* If page is encrypted and traditional checksums match, diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 746ac6a25bd..7dee52d8f74 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -769,10 +769,6 @@ buf_block_unfix( @param[in] read_buf database page @param[in] checksum_field1 new checksum field @param[in] checksum_field2 old checksum field -@param[in] page_no page number of given read_buf -@param[in] is_log_enabled true if log option is enabled -@param[in] log_file file pointer to log_file -@param[in] curr_algo current checksum algorithm @param[in] use_legacy_big_endian use legacy big endian algorithm @return true if the page is in crc32 checksum format. */ bool @@ -780,12 +776,6 @@ buf_page_is_checksum_valid_crc32( const byte* read_buf, ulint checksum_field1, ulint checksum_field2, -#ifdef UNIV_INNOCHECKSUM - uintmax_t page_no, - bool is_log_enabled, - FILE* log_file, - const srv_checksum_algorithm_t curr_algo, -#endif /* UNIV_INNOCHECKSUM */ bool use_legacy_big_endian) MY_ATTRIBUTE((nonnull(1), warn_unused_result)); @@ -793,46 +783,24 @@ buf_page_is_checksum_valid_crc32( @param[in] read_buf database page @param[in] checksum_field1 new checksum field @param[in] checksum_field2 old checksum field -@param[in] page_no page number of given read_buf -@param[in] is_log_enabled true if log option is enabled -@param[in] log_file file pointer to log_file -@param[in] curr_algo current checksum algorithm @return true if the page is in innodb checksum format. */ bool buf_page_is_checksum_valid_innodb( const byte* read_buf, ulint checksum_field1, - ulint checksum_field2 -#ifdef UNIV_INNOCHECKSUM - ,uintmax_t page_no, - bool is_log_enabled, - FILE* log_file, - const srv_checksum_algorithm_t curr_algo -#endif /* UNIV_INNOCHECKSUM */ - ) + ulint checksum_field2) MY_ATTRIBUTE((nonnull(1), warn_unused_result)); /** Checks if the page is in none checksum format. @param[in] read_buf database page @param[in] checksum_field1 new checksum field @param[in] checksum_field2 old checksum field -@param[in] page_no page number of given read_buf -@param[in] is_log_enabled true if log option is enabled -@param[in] log_file file pointer to log_file -@param[in] curr_algo current checksum algorithm @return true if the page is in none checksum format. */ bool buf_page_is_checksum_valid_none( const byte* read_buf, ulint checksum_field1, - ulint checksum_field2 -#ifdef UNIV_INNOCHECKSUM - ,uintmax_t page_no, - bool is_log_enabled, - FILE* log_file, - const srv_checksum_algorithm_t curr_algo -#endif /* UNIV_INNOCHECKSUM */ - ) + ulint checksum_field2) MY_ATTRIBUTE((nonnull(1), warn_unused_result)); /** Checks if a page contains only zeroes. @@ -850,10 +818,6 @@ the LSN @param[in] read_buf database page @param[in] page_size page size @param[in] space tablespace -@param[in] page_no page number of given read_buf -@param[in] strict_check true if strict-check option is enabled -@param[in] is_log_enabled true if log option is enabled -@param[in] log_file file pointer to log_file @return whether the page is corrupted */ bool buf_page_is_corrupted( @@ -861,12 +825,6 @@ buf_page_is_corrupted( const byte* read_buf, const page_size_t& page_size, const fil_space_t* space = NULL -#ifdef UNIV_INNOCHECKSUM - ,uintmax_t page_no = 0, - bool strict_check = false, - bool is_log_enabled = false, - FILE* log_file = NULL -#endif /* UNIV_INNOCHECKSUM */ ) MY_ATTRIBUTE((warn_unused_result)); #ifndef UNIV_INNOCHECKSUM /**********************************************************************//** diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index 497595bab7c..ca65df21402 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -492,10 +492,6 @@ bool fil_space_verify_crypt_checksum( byte* page, const page_size_t& page_size, -#ifdef UNIV_INNOCHECKSUM - bool strict_check, /*!< --strict-check */ - FILE* log_file, /*!< --log */ -#endif /* UNIV_INNOCHECKSUM */ ulint space, ulint offset) MY_ATTRIBUTE((warn_unused_result)); diff --git a/storage/innobase/include/page0zip.h b/storage/innobase/include/page0zip.h index e1062c00a6f..c1d5443d9e5 100644 --- a/storage/innobase/include/page0zip.h +++ b/storage/innobase/include/page0zip.h @@ -512,19 +512,7 @@ ibool page_zip_verify_checksum( /*=====================*/ const void* data, /*!< in: compressed page */ - ulint size /*!< in: size of compressed page */ -#ifdef UNIV_INNOCHECKSUM - /* these variables are used only for innochecksum tool. */ - ,uintmax_t page_no, /*!< in: page number of - given read_buf */ - bool strict_check, /*!< in: true if strict-check - option is enable */ - bool is_log_enabled, /*!< in: true if log option is - enable */ - FILE* log_file /*!< in: file pointer to - log_file */ -#endif /* UNIV_INNOCHECKSUM */ -); + ulint size); /*!< in: size of compressed page */ #ifndef UNIV_INNOCHECKSUM /**********************************************************************//** diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index faf5387a5d1..ee759534301 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -108,6 +108,12 @@ support cross-platform development and expose comonly used SQL names. */ #include #endif +#ifdef UNIV_INNOCHECKSUM +extern bool strict_verify; +extern FILE* log_file; +extern uintmax_t cur_page_num; +#endif /* UNIV_INNOCHECKSUM */ + #include "my_pthread.h" /* Following defines are to enable performance schema instrumentation in each of five InnoDB modules if diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index 88c5a425818..e718081ed63 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -4973,19 +4973,7 @@ ibool page_zip_verify_checksum( /*=====================*/ const void* data, /*!< in: compressed page */ - ulint size /*!< in: size of compressed page */ -#ifdef UNIV_INNOCHECKSUM - /* these variables are used only for innochecksum tool. */ - ,uintmax_t page_no, /*!< in: page number of - given read_buf */ - bool strict_check, /*!< in: true if strict-check - option is enable */ - bool is_log_enabled, /*!< in: true if log option is - enabled */ - FILE* log_file /*!< in: file pointer to - log_file */ -#endif /* UNIV_INNOCHECKSUM */ -) + ulint size) /*!< in: size of compressed page */ { const unsigned char* p = static_cast(data) + FIL_PAGE_SPACE_OR_CHKSUM; @@ -5023,9 +5011,9 @@ page_zip_verify_checksum( break; } if (i >= size) { - if (is_log_enabled) { + if (log_file) { fprintf(log_file, "Page::%lu is empty and" - " uncorrupted\n", page_no); + " uncorrupted\n", cur_page_num); } return(TRUE); @@ -5061,28 +5049,28 @@ page_zip_verify_checksum( const uint32_t calc = page_zip_calc_checksum(data, size, curr_algo); #ifdef UNIV_INNOCHECKSUM - if (is_log_enabled) { + if (log_file) { fprintf(log_file, "page::%lu;" " %s checksum: calculated = %u;" - " recorded = %u\n", page_no, + " recorded = %u\n", cur_page_num, buf_checksum_algorithm_name( static_cast( srv_checksum_algorithm)), calc, stored); } - if (!strict_check) { + if (!strict_verify) { const uint32_t crc32 = page_zip_calc_checksum( data, size, SRV_CHECKSUM_ALGORITHM_CRC32); - if (is_log_enabled) { + if (log_file) { fprintf(log_file, "page::%lu: crc32 checksum:" " calculated = %u; recorded = %u\n", - page_no, crc32, stored); + cur_page_num, crc32, stored); fprintf(log_file, "page::%lu: none checksum:" " calculated = %lu; recorded = %u\n", - page_no, BUF_NO_CHECKSUM_MAGIC, stored); + cur_page_num, BUF_NO_CHECKSUM_MAGIC, stored); } } #endif /* UNIV_INNOCHECKSUM */ -- cgit v1.2.1 From 7afcee4cf66c73ae3f9c5f04763b4b7a1d37e419 Mon Sep 17 00:00:00 2001 From: Jacob Mathew Date: Mon, 1 May 2017 16:04:47 -0700 Subject: MDEV-8954 unnecessary fetch of entire table Merged fix into 10.2. --- storage/spider/ha_spider.cc | 80 ++++++++++++++++++++++++++++++++++ storage/spider/ha_spider.h | 2 + storage/spider/spd_db_handlersocket.cc | 9 ++++ storage/spider/spd_db_handlersocket.h | 3 ++ storage/spider/spd_db_include.h | 3 ++ storage/spider/spd_db_mysql.cc | 59 +++++++++++++++++++++++++ storage/spider/spd_db_mysql.h | 3 ++ storage/spider/spd_db_oracle.cc | 59 +++++++++++++++++++++++++ storage/spider/spd_db_oracle.h | 3 ++ 9 files changed, 221 insertions(+) (limited to 'storage') diff --git a/storage/spider/ha_spider.cc b/storage/spider/ha_spider.cc index 6ded5992c79..d6a5a20df8f 100644 --- a/storage/spider/ha_spider.cc +++ b/storage/spider/ha_spider.cc @@ -2109,6 +2109,7 @@ int ha_spider::index_read_map_internal( result_list.desc_flg = FALSE; result_list.sorted = TRUE; result_list.key_info = &table->key_info[active_index]; + check_distinct_key_query(); result_list.limit_num = result_list.internal_limit >= result_list.split_read ? result_list.split_read : result_list.internal_limit; @@ -2624,6 +2625,7 @@ int ha_spider::index_read_last_map_internal( result_list.desc_flg = TRUE; result_list.sorted = TRUE; result_list.key_info = &table->key_info[active_index]; + check_distinct_key_query(); result_list.limit_num = result_list.internal_limit >= result_list.split_read ? result_list.split_read : result_list.internal_limit; @@ -3089,6 +3091,7 @@ int ha_spider::index_first_internal( result_list.sorted = TRUE; result_list.key_info = &table->key_info[active_index]; result_list.key_order = 0; + check_distinct_key_query(); result_list.limit_num = result_list.internal_limit >= result_list.split_read ? result_list.split_read : result_list.internal_limit; @@ -3472,6 +3475,7 @@ int ha_spider::index_last_internal( result_list.sorted = TRUE; result_list.key_info = &table->key_info[active_index]; result_list.key_order = 0; + check_distinct_key_query(); result_list.limit_num = result_list.internal_limit >= result_list.split_read ? result_list.split_read : result_list.internal_limit; @@ -3914,6 +3918,7 @@ int ha_spider::read_range_first_internal( result_list.desc_flg = FALSE; result_list.sorted = sorted; result_list.key_info = &table->key_info[active_index]; + check_distinct_key_query(); result_list.limit_num = result_list.internal_limit >= result_list.split_read ? result_list.split_read : result_list.internal_limit; @@ -12077,6 +12082,81 @@ void ha_spider::check_direct_order_limit() DBUG_VOID_RETURN; } +/******************************************************************** + * Check whether the current query is a SELECT DISTINCT using an + * index in a non-partitioned Spider configuration, with a + * projection list that consists solely of the first key prefix + * column. + * + * For a SELECT DISTINCT query using an index in a non-partitioned + * Spider configuration, with a projection list that consists + * solely of the first key prefix, set the internal row retrieval + * limit to avoid visiting each row multiple times. + ********************************************************************/ +void ha_spider::check_distinct_key_query() +{ + DBUG_ENTER( "ha_spider::check_distinct_key_query" ); + + if ( result_list.direct_distinct && !partition_handler_share->handlers && + result_list.keyread && result_list.check_direct_order_limit ) + { + // SELECT DISTINCT query using an index in a non-partitioned configuration + KEY_PART_INFO* key_part = result_list.key_info->key_part; + Field* key_field = key_part->field; + + if ( is_sole_projection_field( key_field->field_index ) ) + { + // Projection list consists solely of the first key prefix column + + // Set the internal row retrieval limit to avoid visiting each row + // multiple times. This fixes a Spider performance bug that + // caused each row to be visited multiple times. + result_list.internal_limit = 1; + } + } + + DBUG_VOID_RETURN; +} + +/******************************************************************** + * Determine whether the current query's projection list + * consists solely of the specified column. + * + * Params IN - field_index: + * Field index of the column of interest within + * its table. + * + * Returns TRUE - if the query's projection list consists + * solely of the specified column. + * FALSE - otherwise. + ********************************************************************/ +bool ha_spider::is_sole_projection_field( uint16 field_index ) +{ + // NOTE: It is assumed that spider_db_append_select_columns() has already been called + // to build the bitmap of projection fields + bool is_ha_sole_projection_field; + uint loop_index, dbton_id; + spider_db_handler* dbton_hdl; + DBUG_ENTER( "ha_spider::is_sole_projection_field" ); + + for ( loop_index = 0; loop_index < share->use_sql_dbton_count; loop_index++ ) + { + dbton_id = share->use_sql_dbton_ids[ loop_index ]; + dbton_hdl = dbton_handler[ dbton_id ]; + + if ( dbton_hdl->first_link_idx >= 0 ) + { + is_ha_sole_projection_field = dbton_hdl->is_sole_projection_field( field_index ); + if ( !is_ha_sole_projection_field ) + { + DBUG_RETURN( FALSE ); + } + } + } + + DBUG_RETURN( TRUE ); +} + int ha_spider::check_ha_range_eof() { DBUG_ENTER("ha_spider::check_ha_range_eof"); diff --git a/storage/spider/ha_spider.h b/storage/spider/ha_spider.h index 50d890dc854..acc75ed1c05 100644 --- a/storage/spider/ha_spider.h +++ b/storage/spider/ha_spider.h @@ -751,6 +751,8 @@ public: ); uint check_partitioned(); void check_direct_order_limit(); + void check_distinct_key_query(); + bool is_sole_projection_field( uint16 field_index ); int check_ha_range_eof(); int drop_tmp_tables(); bool handler_opened( diff --git a/storage/spider/spd_db_handlersocket.cc b/storage/spider/spd_db_handlersocket.cc index 47f22b04116..32acb8df6fb 100644 --- a/storage/spider/spd_db_handlersocket.cc +++ b/storage/spider/spd_db_handlersocket.cc @@ -4994,6 +4994,15 @@ int spider_handlersocket_handler::append_explain_select_part( DBUG_RETURN(0); } +int spider_handlersocket_handler::is_sole_projection_field( + uint16 field_index +) { + DBUG_ENTER("spider_handlersocket_handler::is_sole_projection_field"); + DBUG_PRINT("info", ("spider this=%p", this)); + DBUG_ASSERT(0); + DBUG_RETURN(0); +} + bool spider_handlersocket_handler::is_bulk_insert_exec_period( bool bulk_end ) { diff --git a/storage/spider/spd_db_handlersocket.h b/storage/spider/spd_db_handlersocket.h index d3fdf5564b7..a3955aea044 100644 --- a/storage/spider/spd_db_handlersocket.h +++ b/storage/spider/spd_db_handlersocket.h @@ -776,6 +776,9 @@ public: ulong sql_type, int link_idx ); + bool is_sole_projection_field( + uint16 field_index + ); bool is_bulk_insert_exec_period( bool bulk_end ); diff --git a/storage/spider/spd_db_include.h b/storage/spider/spd_db_include.h index 936951d3860..56bc2ccad42 100644 --- a/storage/spider/spd_db_include.h +++ b/storage/spider/spd_db_include.h @@ -1279,6 +1279,9 @@ public: ulong sql_type, int link_idx ) = 0; + virtual bool is_sole_projection_field( + uint16 field_index + ) = 0; virtual bool is_bulk_insert_exec_period( bool bulk_end ) = 0; diff --git a/storage/spider/spd_db_mysql.cc b/storage/spider/spd_db_mysql.cc index 1c1c440c2ed..9974b6b0628 100644 --- a/storage/spider/spd_db_mysql.cc +++ b/storage/spider/spd_db_mysql.cc @@ -9516,6 +9516,65 @@ int spider_mysql_handler::append_explain_select( DBUG_RETURN(0); } +/******************************************************************** + * Determine whether the current query's projection list + * consists solely of the specified column. + * + * Params IN - field_index: + * Field index of the column of interest within + * its table. + * + * Returns TRUE - if the query's projection list consists + * solely of the specified column. + * FALSE - otherwise. + ********************************************************************/ +bool spider_mysql_handler::is_sole_projection_field( uint16 field_index ) +{ + // Determine whether the projection list consists solely of the field of interest + bool is_field_in_projection_list = FALSE; + TABLE* table = spider->get_table(); + uint16 projection_field_count = 0; + uint16 projection_field_index; + Field** field; + DBUG_ENTER( "spider_mysql_handler::is_sole_projection_field" ); + + for ( field = table->field; *field ; field++ ) + { + projection_field_index = ( *field )->field_index; + + if ( !( minimum_select_bit_is_set( projection_field_index ) ) ) + { + // Current field is not in the projection list + continue; + } + + projection_field_count++; + + if ( !is_field_in_projection_list ) + { + if ( field_index == projection_field_index ) + { + // Field of interest is in the projection list + is_field_in_projection_list = TRUE; + } + } + + if ( is_field_in_projection_list && ( projection_field_count != 1 ) ) + { + // Field of interest is not the sole column in the projection list + DBUG_RETURN( FALSE ); + } + } + + if ( is_field_in_projection_list && ( projection_field_count == 1 ) ) + { + // Field of interest is the only column in the projection list + DBUG_RETURN( TRUE ); + } + + DBUG_RETURN( FALSE ); +} + bool spider_mysql_handler::is_bulk_insert_exec_period( bool bulk_end ) { diff --git a/storage/spider/spd_db_mysql.h b/storage/spider/spd_db_mysql.h index 9a4f08ade98..482289d1d68 100644 --- a/storage/spider/spd_db_mysql.h +++ b/storage/spider/spd_db_mysql.h @@ -1128,6 +1128,9 @@ public: ulong sql_type, int link_idx ); + bool is_sole_projection_field( + uint16 field_index + ); bool is_bulk_insert_exec_period( bool bulk_end ); diff --git a/storage/spider/spd_db_oracle.cc b/storage/spider/spd_db_oracle.cc index c3dfe8b8cf2..94f354e0a0f 100644 --- a/storage/spider/spd_db_oracle.cc +++ b/storage/spider/spd_db_oracle.cc @@ -9571,6 +9571,65 @@ int spider_oracle_handler::append_explain_select( DBUG_RETURN(0); } +/******************************************************************** + * Determine whether the current query's projection list + * consists solely of the specified column. + * + * Params IN - field_index: + * Field index of the column of interest within + * its table. + * + * Returns TRUE - if the query's projection list consists + * solely of the specified column. + * FALSE - otherwise. + ********************************************************************/ +bool spider_oracle_handler::is_sole_projection_field( uint16 field_index ) +{ + // Determine whether the projection list consists solely of the field of interest + bool is_field_in_projection_list = FALSE; + TABLE* table = spider->get_table(); + uint16 projection_field_count = 0; + uint16 projection_field_index; + Field** field; + DBUG_ENTER( "spider_oracle_handler::is_sole_projection_field" ); + + for ( field = table->field; *field; field++ ) + { + projection_field_index = ( *field )->field_index; + + if ( !( minimum_select_bit_is_set( projection_field_index ) ) ) + { + // Current field is not in the projection list + continue; + } + + projection_field_count++; + + if ( !is_field_in_projection_list ) + { + if (field_index == projection_field_index) + { + // Field of interest is in the projection list + is_field_in_projection_list = TRUE; + } + } + + if ( is_field_in_projection_list && ( projection_field_count != 1 ) ) + { + // Field of interest is not the sole column in the projection list + DBUG_RETURN( FALSE ); + } + } + + if ( is_field_in_projection_list && ( projection_field_count == 1 ) ) + { + // Field of interest is the only column in the projection list + DBUG_RETURN( TRUE ); + } + + DBUG_RETURN( FALSE ); +} + bool spider_oracle_handler::is_bulk_insert_exec_period( bool bulk_end ) { diff --git a/storage/spider/spd_db_oracle.h b/storage/spider/spd_db_oracle.h index 0e84435d9be..7a070f498da 100644 --- a/storage/spider/spd_db_oracle.h +++ b/storage/spider/spd_db_oracle.h @@ -1208,6 +1208,9 @@ public: ulong sql_type, int link_idx ); + bool is_sole_projection_field( + uint16 field_index + ); bool is_bulk_insert_exec_period( bool bulk_end ); -- cgit v1.2.1 From 52463ccff7ee5024f00541c12938ffbaf9b93958 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 28 Apr 2017 12:25:52 +1000 Subject: MDEV-12469: rocksdb having large numberic storage errors on ppc64 (BE) (from: http://buildbot.askmonty.org/buildbot/builders/p8-rhel6-bintar/builds/820/steps/test/logs/stdio) Errors like the following indicate a potential endian storage issue: rocksdb.rocksdb_range w1 [ fail ] Test ended at 2017-04-27 18:56:11 CURRENT_TEST: rocksdb.rocksdb_range --- /home/buildbot/maria-slave/p8-rhel6-bintar/build/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_range.result 2017-04-27 17:41:27.740050347 -0400 +++ /home/buildbot/maria-slave/p8-rhel6-bintar/build/storage/rocksdb/mysql-test/rocksdb/r/rocksdb_range.reject 2017-04-27 18:56:11.230050346 -0400 @@ -25,15 +25,15 @@ select * from t2 force index (a) where a=0; pk a b 0 0 0 -1 0 1 -2 0 2 -3 0 3 -4 0 4 -5 0 5 -6 0 6 -7 0 7 -8 0 8 -9 0 9 +16777216 0 1 +33554432 0 2 +50331648 0 3 +67108864 0 4 +83886080 0 5 +100663296 0 6 +117440512 0 7 +134217728 0 8 +150994944 0 9 # The rest are for code coverage: explain select * from t2 force index (a) where a=2; @@ -41,23 +41,23 @@ 1 SIMPLE t2 ref a a 4 const # select * from t2 force index (a) where a=2; pk a b -20 2 20 -21 2 21 -22 2 22 -23 2 23 -24 2 24 -25 2 25 -26 2 26 -27 2 27 -28 2 28 -29 2 29 +335544320 2 20 +352321536 2 21 +369098752 2 22 +385875968 2 23 +402653184 2 24 +419430400 2 25 +436207616 2 26 +452984832 2 27 +469762048 2 28 +486539264 2 29 explain select * from t2 force index (a) where a=3 and pk=33; id select_type table type possible_keys key key_len ref rows Extra 1 SIMPLE t2 const a a 8 const,const # select * from t2 force index (a) where a=3 and pk=33; pk a b -33 3 33 +553648128 3 33 select * from t2 force index (a) where a=99 and pk=99; pk a b select * from t2 force index (a) where a=0 and pk=0; ... Signed-off-by: Daniel Black --- storage/rocksdb/CMakeLists.txt | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'storage') diff --git a/storage/rocksdb/CMakeLists.txt b/storage/rocksdb/CMakeLists.txt index 3223e084709..d72b4e54e43 100644 --- a/storage/rocksdb/CMakeLists.txt +++ b/storage/rocksdb/CMakeLists.txt @@ -22,6 +22,13 @@ IF(CMAKE_SYSTEM_PROCESSOR MATCHES "i[36]86") SKIP_ROCKSDB_PLUGIN("Intel 32 bit not supported.") ENDIF() +# Due to retrieved data being incorrect endian +include(TestBigEndian) +test_big_endian(BIG_ENDIAN) +if(BIG_ENDIAN) + SKIP_ROCKSDB_PLUGIN("Big Endian not supported.") +endif() + # # Also, disable building on 32-bit Windows # -- cgit v1.2.1