diff options
Diffstat (limited to 'storage')
| -rw-r--r-- | storage/innobase/buf/buf0buf.cc | 6 | ||||
| -rw-r--r-- | storage/innobase/buf/buf0lru.cc | 40 | ||||
| -rw-r--r-- | storage/innobase/dict/dict0dict.cc | 1 | ||||
| -rw-r--r-- | storage/innobase/gis/gis0sea.cc | 7 | ||||
| -rw-r--r-- | storage/innobase/handler/ha_innodb.cc | 15 | ||||
| -rw-r--r-- | storage/innobase/include/log0log.h | 4 | ||||
| -rw-r--r-- | storage/innobase/log/log0recv.cc | 86 | ||||
| -rw-r--r-- | storage/innobase/rem/rem0rec.cc | 8 | ||||
| -rw-r--r-- | storage/innobase/row/row0mysql.cc | 11 | ||||
| -rw-r--r-- | storage/innobase/srv/srv0start.cc | 8 | ||||
| -rw-r--r-- | storage/innobase/trx/trx0purge.cc | 22 | ||||
| -rw-r--r-- | storage/innobase/ut/ut0dbg.cc | 4 | ||||
| -rw-r--r-- | storage/xtradb/buf/buf0lru.cc | 138 | ||||
| -rw-r--r-- | storage/xtradb/dict/dict0dict.cc | 1 | ||||
| -rw-r--r-- | storage/xtradb/include/log0crypt.h | 6 | ||||
| -rw-r--r-- | storage/xtradb/log/log0crypt.cc | 54 | ||||
| -rw-r--r-- | storage/xtradb/log/log0log.cc | 7 | ||||
| -rw-r--r-- | storage/xtradb/trx/trx0purge.cc | 26 |
18 files changed, 191 insertions, 253 deletions
diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index ae081fa39da..07d3f7efe27 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -4360,11 +4360,7 @@ loop: << ". The most probable cause" " of this error may be that the" " table has been corrupted." - " You can try to fix this" - " problem by using" - " innodb_force_recovery." - " Please see " REFMAN " for more" - " details. Aborting..."; + " See https://mariadb.com/kb/en/library/xtradbinnodb-recovery-modes/"; } #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 26814418033..10ee106137d 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -80,6 +80,10 @@ static const ulint BUF_LRU_SEARCH_SCAN_THRESHOLD = 100; frames in the buffer pool, we set this to TRUE */ static bool buf_lru_switched_on_innodb_mon = false; +/** True if diagnostic message about difficult to find free blocks +in the buffer bool has already printed. */ +static bool buf_lru_free_blocks_error_printed; + /******************************************************************//** These statistics are not 'of' LRU but 'for' LRU. We keep count of I/O and page_zip_decompress() operations. Based on the statistics, @@ -1079,8 +1083,6 @@ buf_LRU_get_free_block( bool freed = false; ulint n_iterations = 0; ulint flush_failures = 0; - bool mon_value_was = false; - bool started_monitor = false; MONITOR_INC(MONITOR_LRU_GET_FREE_SEARCH); loop: @@ -1088,6 +1090,11 @@ loop: buf_LRU_check_size_of_non_data_objects(buf_pool); + DBUG_EXECUTE_IF("ib_lru_force_no_free_page", + if (!buf_lru_free_blocks_error_printed) { + n_iterations = 21; + goto not_found;}); + /* If there is a block in the free list, take it */ block = buf_LRU_get_free_only(buf_pool); @@ -1097,11 +1104,6 @@ loop: ut_ad(buf_pool_from_block(block) == buf_pool); memset(&block->page.zip, 0, sizeof block->page.zip); - if (started_monitor) { - srv_print_innodb_monitor = - static_cast<my_bool>(mon_value_was); - } - block->skip_flush_check = false; block->page.flush_observer = NULL; return(block); @@ -1131,24 +1133,24 @@ loop: } } +#ifndef DBUG_OFF +not_found: +#endif + buf_pool_mutex_exit(buf_pool); if (freed) { goto loop; } - if (n_iterations > 20 + if (n_iterations > 20 && !buf_lru_free_blocks_error_printed && srv_buf_pool_old_size == srv_buf_pool_size) { ib::warn() << "Difficult to find free blocks in the buffer pool" " (" << n_iterations << " search iterations)! " << flush_failures << " failed attempts to" - " flush a page! Consider increasing the buffer pool" - " size. It is also possible that in your Unix version" - " fsync is very slow, or completely frozen inside" - " the OS kernel. Then upgrading to a newer version" - " of your operating system may help. Look at the" - " number of fsyncs in diagnostic info below." + " flush a page!" + " Consider increasing innodb_buffer_pool_size." " Pending flushes (fsync) log: " << fil_n_pending_log_flushes << "; buffer pool: " @@ -1156,13 +1158,9 @@ loop: << ". " << os_n_file_reads << " OS file reads, " << os_n_file_writes << " OS file writes, " << os_n_fsyncs - << " OS fsyncs. Starting InnoDB Monitor to print" - " further diagnostics to the standard output."; + << " OS fsyncs."; - mon_value_was = srv_print_innodb_monitor; - started_monitor = true; - srv_print_innodb_monitor = true; - os_event_set(srv_monitor_event); + buf_lru_free_blocks_error_printed = true; } /* If we have scanned the whole LRU and still are unable to diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index f93a71a48c5..16b3e2111f9 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -3442,6 +3442,7 @@ dict_foreign_find_index( && !(index->type & DICT_FTS) && !dict_index_is_spatial(index) && !index->to_be_dropped + && !dict_index_is_online_ddl(index) && dict_foreign_qualify_index( table, col_names, columns, n_cols, index, types_idx, diff --git a/storage/innobase/gis/gis0sea.cc b/storage/innobase/gis/gis0sea.cc index 87ebd9ad34a..dcf8cc6f781 100644 --- a/storage/innobase/gis/gis0sea.cc +++ b/storage/innobase/gis/gis0sea.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -773,8 +773,9 @@ rtr_page_get_father_node_ptr( error << ". You should dump + drop + reimport the table to" " fix the corruption. If the crash happens at" - " database startup, see " REFMAN - "forcing-innodb-recovery.html about forcing" + " database startup, see " + "https://mariadb.com/kb/en/library/xtradbinnodb-recovery-modes/" + " about forcing" " recovery. Then dump + drop + reimport."; } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index e59b4460551..64f7a1d89aa 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. Copyright (c) 2012, Facebook Inc. @@ -3652,7 +3652,7 @@ static uint innobase_partition_flags() #define DEPRECATED_FORMAT_PARAMETER(x) \ "Using " x " is deprecated and the parameter" \ " may be removed in future releases." \ - " See " REFMAN "innodb-file-format.html" + " See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/" /** Deprecation message about innodb_file_format */ static const char* deprecated_file_format @@ -22580,7 +22580,8 @@ const char* BUG_REPORT_MSG = "Submit a detailed bug report to https://jira.mariadb.org/"; const char* FORCE_RECOVERY_MSG = - "Please refer to " REFMAN "forcing-innodb-recovery.html" + "Please refer to " + "https://mariadb.com/kb/en/library/xtradbinnodb-recovery-modes/" " for information about forcing recovery."; const char* ERROR_CREATING_MSG = @@ -22588,17 +22589,17 @@ const char* ERROR_CREATING_MSG = const char* OPERATING_SYSTEM_ERROR_MSG = "Some operating system error numbers are described at" - " " REFMAN "operating-system-error-codes.html"; + " https://mariadb.com/kb/en/library/operating-system-error-codes/"; const char* FOREIGN_KEY_CONSTRAINTS_MSG = - "Please refer to " REFMAN "innodb-foreign-key-constraints.html" + "Please refer to https://mariadb.com/kb/en/library/foreign-keys/" " for correct foreign key definition."; const char* SET_TRANSACTION_MSG = - "Please refer to " REFMAN "set-transaction.html"; + "Please refer to https://mariadb.com/kb/en/library/set-transaction/"; const char* INNODB_PARAMETERS_MSG = - "Please refer to " REFMAN "innodb-parameters.html"; + "Please refer to https://mariadb.com/kb/en/library/xtradbinnodb-server-system-variables/"; /********************************************************************** Converts an identifier from my_charset_filename to UTF-8 charset. diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 543302c52f0..91ca389df83 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2009, Google Inc. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -513,6 +513,8 @@ or the MySQL version that created the redo log file. */ /** The redo log format identifier corresponding to the current format version. Stored in LOG_HEADER_FORMAT. */ #define LOG_HEADER_FORMAT_CURRENT 1 +/** The MariaDB 10.3.2 log format */ +#define LOG_HEADER_FORMAT_10_3 103 /** Encrypted MariaDB redo log */ #define LOG_HEADER_FORMAT_ENCRYPTED (1U<<31) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 93943620ecf..821173d0f33 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2,7 +2,7 @@ Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -842,7 +842,7 @@ recv_find_max_checkpoint_0(log_group_t** max_group, ulint* max_field) " This redo log was created before MariaDB 10.2.2," " and we did not find a valid checkpoint." " Please follow the instructions at" - " " REFMAN "upgrading.html"; + " https://mariadb.com/kb/en/library/upgrading/"; return(DB_ERROR); } @@ -869,7 +869,7 @@ recv_log_format_0_recover(lsn_t lsn) " This redo log was created before MariaDB 10.2.2"; static const char* NO_UPGRADE_RTFM_MSG = ". Please follow the instructions at " - REFMAN "upgrading.html"; + "https://mariadb.com/kb/en/library/upgrading/"; fil_io(IORequestLogRead, true, page_id_t(SRV_LOG_SPACE_FIRST_ID, page_no), @@ -907,6 +907,58 @@ recv_log_format_0_recover(lsn_t lsn) return(DB_SUCCESS); } +/** Determine if a redo log from MariaDB 10.3 is clean. +@return error code +@retval DB_SUCCESS if the redo log is clean +@retval DB_CORRUPTION if the redo log is corrupted +@retval DB_ERROR if the redo log is not empty */ +static +dberr_t +recv_log_recover_10_3() +{ + log_group_t* group = &log_sys->log; + const lsn_t lsn = group->lsn; + const lsn_t source_offset = log_group_calc_lsn_offset(lsn, group); + const ulint page_no + = (ulint) (source_offset / univ_page_size.physical()); + byte* buf = log_sys->buf; + + fil_io(IORequestLogRead, true, + page_id_t(SRV_LOG_SPACE_FIRST_ID, page_no), + univ_page_size, + (ulint) ((source_offset & ~(OS_FILE_LOG_BLOCK_SIZE - 1)) + % univ_page_size.physical()), + OS_FILE_LOG_BLOCK_SIZE, buf, NULL); + + if (log_block_calc_checksum(buf) != log_block_get_checksum(buf)) { + return(DB_CORRUPTION); + } + + if (group->is_encrypted()) { + log_crypt(buf, lsn, OS_FILE_LOG_BLOCK_SIZE, true); + } + + /* On a clean shutdown, the redo log will be logically empty + after the checkpoint lsn. */ + + if (log_block_get_data_len(buf) + != (source_offset & (OS_FILE_LOG_BLOCK_SIZE - 1))) { + return(DB_ERROR); + } + + /* Mark the redo log for downgrading. */ + srv_log_file_size = 0; + recv_sys->parse_start_lsn = recv_sys->recovered_lsn + = recv_sys->scanned_lsn + = recv_sys->mlog_checkpoint_lsn = lsn; + log_sys->last_checkpoint_lsn = log_sys->next_checkpoint_lsn + = log_sys->lsn = log_sys->write_lsn + = log_sys->current_flush_lsn = log_sys->flushed_to_disk_lsn + = lsn; + log_sys->next_checkpoint_no = 0; + return(DB_SUCCESS); +} + /** Find the latest checkpoint in the log header. @param[out] max_field LOG_CHECKPOINT_1 or LOG_CHECKPOINT_2 @return error code or DB_SUCCESS */ @@ -938,18 +990,24 @@ recv_find_max_checkpoint(ulint* max_field) return(DB_CORRUPTION); } + char creator[LOG_HEADER_CREATOR_END - LOG_HEADER_CREATOR + 1]; + + memcpy(creator, buf + LOG_HEADER_CREATOR, sizeof creator); + /* Ensure that the string is NUL-terminated. */ + creator[LOG_HEADER_CREATOR_END - LOG_HEADER_CREATOR] = 0; + switch (group->format) { case 0: return(recv_find_max_checkpoint_0(&group, max_field)); case LOG_HEADER_FORMAT_CURRENT: case LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED: + case LOG_HEADER_FORMAT_10_3: + case LOG_HEADER_FORMAT_10_3 | LOG_HEADER_FORMAT_ENCRYPTED: break; default: - /* Ensure that the string is NUL-terminated. */ - buf[LOG_HEADER_CREATOR_END] = 0; ib::error() << "Unsupported redo log format." " The redo log was created" - " with " << buf + LOG_HEADER_CREATOR << + " with " << creator << ". Please follow the instructions at " REFMAN "upgrading-downgrading.html"; /* Do not issue a message about a possibility @@ -1018,6 +1076,20 @@ recv_find_max_checkpoint(ulint* max_field) return(DB_ERROR); } + switch (group->format) { + case LOG_HEADER_FORMAT_10_3: + case LOG_HEADER_FORMAT_10_3 | LOG_HEADER_FORMAT_ENCRYPTED: + dberr_t err = recv_log_recover_10_3(); + if (err != DB_SUCCESS) { + ib::error() + << "Downgrade after a crash is not supported." + " The redo log was created with " << creator + << (err == DB_ERROR + ? "." : ", and it appears corrupted."); + } + return(err); + } + return(DB_SUCCESS); } @@ -1166,6 +1238,8 @@ parse_log: redo log been written with something older than InnoDB Plugin 1.0.4. */ ut_ad(0 + /* fil_crypt_rotate_page() writes this */ + || offs == FIL_PAGE_SPACE_ID || offs == IBUF_TREE_SEG_HEADER + IBUF_HEADER + FSEG_HDR_SPACE || offs == IBUF_TREE_SEG_HEADER diff --git a/storage/innobase/rem/rem0rec.cc b/storage/innobase/rem/rem0rec.cc index 37bf6e649a2..6cd8db5d37b 100644 --- a/storage/innobase/rem/rem0rec.cc +++ b/storage/innobase/rem/rem0rec.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -2167,8 +2167,6 @@ rec_get_trx_id( const rec_t* rec, const dict_index_t* index) { - const page_t* page - = page_align(rec); ulint trx_id_col = dict_index_get_sys_col_pos(index, DATA_TRX_ID); const byte* trx_id; @@ -2179,11 +2177,7 @@ rec_get_trx_id( ulint* offsets = offsets_; ut_ad(trx_id_col <= MAX_REF_PARTS); - ut_ad(fil_page_index_page_check(page)); - ut_ad(mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID) - == index->id); ut_ad(dict_index_is_clust(index)); - ut_ad(page_rec_is_leaf(rec)); ut_ad(trx_id_col > 0); ut_ad(trx_id_col != ULINT_UNDEFINED); diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 99f4d37dc6c..8f9b5296015 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1425,17 +1425,6 @@ row_insert_for_mysql( } else if (high_level_read_only) { return(DB_READ_ONLY); } - DBUG_EXECUTE_IF("mark_table_corrupted", { - /* Mark the table corrupted for the clustered index */ - dict_index_t* index = dict_table_get_first_index(table); - ut_ad(dict_index_is_clust(index)); - dict_set_corrupted(index, trx, "INSERT TABLE"); }); - - if (dict_table_is_corrupted(table)) { - - ib::error() << "Table " << table->name << " is corrupt."; - return(DB_TABLE_CORRUPT); - } DBUG_EXECUTE_IF("mark_table_corrupted", { /* Mark the table corrupted for the clustered index */ diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 886d47e44c0..7bfd1715e9b 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -3,7 +3,7 @@ Copyright (c) 1996, 2017, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1413,7 +1413,11 @@ srv_prepare_to_delete_redo_log_files( { ib::info info; if (srv_log_file_size == 0) { - info << "Upgrading redo log: "; + info << ((log_sys->log.format + & ~LOG_HEADER_FORMAT_ENCRYPTED) + != LOG_HEADER_FORMAT_10_3 + ? "Upgrading redo log: " + : "Downgrading redo log: "); } else if (n_files != srv_n_log_files || srv_log_file_size != srv_log_file_size_requested) { diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 4fd9333c0ba..07150577d22 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -1162,28 +1162,6 @@ trx_purge_rseg_get_next_history_log( mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); - - trx_sys_mutex_enter(); - - /* Add debug code to track history list corruption reported - on the MySQL mailing list on Nov 9, 2004. The fut0lst.cc - file-based list was corrupt. The prev node pointer was - FIL_NULL, even though the list length was over 8 million nodes! - We assume that purge truncates the history list in large - size pieces, and if we here reach the head of the list, the - list cannot be longer than 2000 000 undo logs now. */ - - if (trx_sys->rseg_history_len > 2000000) { - ib::warn() << "Purge reached the head of the history" - " list, but its length is still reported as " - << trx_sys->rseg_history_len << "! Make" - " a detailed bug report, and submit it to" - " https://jira.mariadb.org/"; - ut_ad(0); - } - - trx_sys_mutex_exit(); - return; } diff --git a/storage/innobase/ut/ut0dbg.cc b/storage/innobase/ut/ut0dbg.cc index 9e596dcda81..7df189ac560 100644 --- a/storage/innobase/ut/ut0dbg.cc +++ b/storage/innobase/ut/ut0dbg.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -53,7 +53,7 @@ ut_dbg_assertion_failed( " or crashes, even\n" "InnoDB: immediately after the mysqld startup, there may be\n" "InnoDB: corruption in the InnoDB tablespace. Please refer to\n" - "InnoDB: " REFMAN "forcing-innodb-recovery.html\n" + "InnoDB: https://mariadb.com/kb/en/library/xtradbinnodb-recovery-modes/\n" "InnoDB: about forcing recovery.\n", stderr); fflush(stderr); diff --git a/storage/xtradb/buf/buf0lru.cc b/storage/xtradb/buf/buf0lru.cc index c71d45009e4..a84f2a832f3 100644 --- a/storage/xtradb/buf/buf0lru.cc +++ b/storage/xtradb/buf/buf0lru.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -87,6 +87,10 @@ buffer pools. */ frames in the buffer pool, we set this to TRUE */ static ibool buf_lru_switched_on_innodb_mon = FALSE; +/** True if diagnostic message about difficult to find free blocks +in the buffer bool has already printed. */ +static bool buf_lru_free_blocks_error_printed; + /******************************************************************//** These statistics are not 'of' LRU but 'for' LRU. We keep count of I/O and page_zip_decompress() operations. Based on the statistics, @@ -1080,68 +1084,39 @@ buf_LRU_check_size_of_non_data_objects( } /** Diagnose failure to get a free page and request InnoDB monitor output in -the error log if more than two seconds have been spent already. +the error log if it has not yet printed. @param[in] n_iterations how many buf_LRU_get_free_page iterations already completed -@param[in] started_ms timestamp in ms of when the attempt to get the - free page started @param[in] flush_failures how many times single-page flush, if allowed, has failed -@param[out] mon_value_was previous srv_print_innodb_monitor value -@param[out] started_monitor whether InnoDB monitor print has been requested */ static void -buf_LRU_handle_lack_of_free_blocks(ulint n_iterations, ulint started_ms, - ulint flush_failures, - ibool *mon_value_was, - ibool *started_monitor) +buf_LRU_handle_lack_of_free_blocks( + ulint n_iterations, + ulint flush_failures) { - static ulint last_printout_ms = 0; - - /* Legacy algorithm started warning after at least 2 seconds, we - emulate this. */ - const ulint current_ms = ut_time_ms(); - - if ((current_ms > started_ms + 2000) - && (current_ms > last_printout_ms + 2000)) { - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Warning: difficult to find free blocks in\n" - "InnoDB: the buffer pool (%lu search iterations)!\n" - "InnoDB: %lu failed attempts to flush a page!" - " Consider\n" - "InnoDB: increasing the buffer pool size.\n" - "InnoDB: It is also possible that" - " in your Unix version\n" - "InnoDB: fsync is very slow, or" - " completely frozen inside\n" - "InnoDB: the OS kernel. Then upgrading to" - " a newer version\n" - "InnoDB: of your operating system may help." - " Look at the\n" - "InnoDB: number of fsyncs in diagnostic info below.\n" - "InnoDB: Pending flushes (fsync) log: %lu;" - " buffer pool: %lu\n" - "InnoDB: %lu OS file reads, %lu OS file writes," - " %lu OS fsyncs\n" - "InnoDB: Starting InnoDB Monitor to print further\n" - "InnoDB: diagnostics to the standard output.\n", - (ulong) n_iterations, - (ulong) flush_failures, - (ulong) fil_n_pending_log_flushes, - (ulong) fil_n_pending_tablespace_flushes, - (ulong) os_n_file_reads, (ulong) os_n_file_writes, - (ulong) os_n_fsyncs); - - last_printout_ms = current_ms; - *mon_value_was = srv_print_innodb_monitor; - *started_monitor = TRUE; - srv_print_innodb_monitor = TRUE; - os_event_set(lock_sys->timeout_event); + if (n_iterations > 20 && !buf_lru_free_blocks_error_printed) { + ib_logf(IB_LOG_LEVEL_WARN, + "Difficult to find free blocks in" + " the buffer pool (" ULINTPF " search iterations)! " + ULINTPF " failed attempts to flush a page!", + n_iterations, flush_failures); + ib_logf(IB_LOG_LEVEL_INFO, + "Consider increasing the buffer pool size."); + ib_logf(IB_LOG_LEVEL_INFO, + "Pending flushes (fsync) log: " ULINTPF + " buffer pool: " ULINTPF + " OS file reads: " ULINTPF " OS file writes: " + ULINTPF " OS fsyncs: " ULINTPF "", + fil_n_pending_log_flushes, + fil_n_pending_tablespace_flushes, + os_n_file_reads, + os_n_file_writes, + os_n_fsyncs); + + buf_lru_free_blocks_error_printed = true; } - } /** The maximum allowed backoff sleep time duration, microseconds */ @@ -1189,9 +1164,6 @@ buf_LRU_get_free_block( ibool freed = FALSE; ulint n_iterations = 0; ulint flush_failures = 0; - ibool mon_value_was = FALSE; - ibool started_monitor = FALSE; - ulint started_ms = 0; ut_ad(!mutex_own(&buf_pool->LRU_list_mutex)); @@ -1199,42 +1171,20 @@ buf_LRU_get_free_block( loop: buf_LRU_check_size_of_non_data_objects(buf_pool); - /* If there is a block in the free list, take it */ - if (DBUG_EVALUATE_IF("simulate_lack_of_pages", true, false)) { - - block = NULL; - - if (srv_debug_monitor_printed) - DBUG_SET("-d,simulate_lack_of_pages"); - - } else if (DBUG_EVALUATE_IF("simulate_recovery_lack_of_pages", - recv_recovery_on, false)) { - - block = NULL; - - if (srv_debug_monitor_printed) - DBUG_SUICIDE(); - } else { + DBUG_EXECUTE_IF("ib_lru_force_no_free_page", + if (!buf_lru_free_blocks_error_printed) { + n_iterations = 21; + goto not_found;}); - block = buf_LRU_get_free_only(buf_pool); - } + block = buf_LRU_get_free_only(buf_pool); if (block) { ut_ad(buf_pool_from_block(block) == buf_pool); memset(&block->page.zip, 0, sizeof block->page.zip); - - if (started_monitor) { - srv_print_innodb_monitor = - static_cast<my_bool>(mon_value_was); - } - return(block); } - if (!started_ms) - started_ms = ut_time_ms(); - if (srv_empty_free_list_algorithm == SRV_EMPTY_FREE_LIST_BACKOFF && buf_lru_manager_is_active && (srv_shutdown_state == SRV_SHUTDOWN_NONE @@ -1272,10 +1222,7 @@ loop: : FREE_LIST_BACKOFF_LOW_PRIO_DIVIDER)); } - buf_LRU_handle_lack_of_free_blocks(n_iterations, started_ms, - flush_failures, - &mon_value_was, - &started_monitor); + buf_LRU_handle_lack_of_free_blocks(n_iterations, flush_failures); n_iterations++; @@ -1314,13 +1261,8 @@ loop: mutex_exit(&buf_pool->flush_state_mutex); - if (DBUG_EVALUATE_IF("simulate_recovery_lack_of_pages", true, false) - || DBUG_EVALUATE_IF("simulate_lack_of_pages", true, false)) { - - buf_pool->try_LRU_scan = false; - } - freed = FALSE; + if (buf_pool->try_LRU_scan || n_iterations > 0) { /* If no block was in the free list, search from the @@ -1345,9 +1287,11 @@ loop: } - buf_LRU_handle_lack_of_free_blocks(n_iterations, started_ms, - flush_failures, &mon_value_was, - &started_monitor); +#ifndef DBUG_OFF +not_found: +#endif + + buf_LRU_handle_lack_of_free_blocks(n_iterations, flush_failures); /* If we have scanned the whole LRU and still are unable to find a free block then we should sleep here to let the diff --git a/storage/xtradb/dict/dict0dict.cc b/storage/xtradb/dict/dict0dict.cc index 5af3a635a96..9257321c7ef 100644 --- a/storage/xtradb/dict/dict0dict.cc +++ b/storage/xtradb/dict/dict0dict.cc @@ -3409,6 +3409,7 @@ dict_foreign_find_index( if (types_idx != index && !(index->type & DICT_FTS) && !index->to_be_dropped + && !dict_index_is_online_ddl(index) && dict_foreign_qualify_index( table, col_names, columns, n_cols, index, types_idx, diff --git a/storage/xtradb/include/log0crypt.h b/storage/xtradb/include/log0crypt.h index 0ad7e7d7037..b7a221e0a81 100644 --- a/storage/xtradb/include/log0crypt.h +++ b/storage/xtradb/include/log0crypt.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -73,6 +73,8 @@ log_encrypt_before_write( /*=====================*/ ib_uint64_t next_checkpoint_no, /*!< in: log group to be flushed */ byte* block, /*!< in/out: pointer to a log block */ + lsn_t lsn, /*!< in: log sequence number of + the start of the buffer */ const ulint size); /*!< in: size of log blocks */ /******************************************************** @@ -83,6 +85,8 @@ void log_decrypt_after_read( /*===================*/ byte* frame, /*!< in/out: log segment */ + lsn_t lsn, /*!< in: log sequence number of the start + of the buffer */ const ulint size); /*!< in: log segment size */ /* Error codes for crypt info */ diff --git a/storage/xtradb/log/log0crypt.cc b/storage/xtradb/log/log0crypt.cc index a5fbbab17ef..2a0a7abb686 100644 --- a/storage/xtradb/log/log0crypt.cc +++ b/storage/xtradb/log/log0crypt.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -70,22 +70,6 @@ struct crypt_info_t { static std::deque<crypt_info_t> crypt_info; /*********************************************************************//** -Get a log block's start lsn. -@return a log block's start lsn */ -static inline -lsn_t -log_block_get_start_lsn( -/*====================*/ - lsn_t lsn, /*!< in: checkpoint lsn */ - ulint log_block_no) /*!< in: log block number */ -{ - lsn_t start_lsn = - (lsn & (lsn_t)0xffffffff00000000ULL) | - (((log_block_no - 1) & (lsn_t)0x3fffffff) << 9); - return start_lsn; -} - -/*********************************************************************//** Get crypt info from checkpoint. @return a crypt info or NULL if not present. */ static @@ -161,6 +145,8 @@ Crypt_result log_blocks_crypt( /*=============*/ const byte* block, /*!< in: blocks before encrypt/decrypt*/ + lsn_t lsn, /*!< in: log sequence number of the start + of the buffer */ ulint size, /*!< in: size of block */ byte* dst_block, /*!< out: blocks after encrypt/decrypt */ int what, /*!< in: encrypt or decrypt*/ @@ -170,21 +156,18 @@ log_blocks_crypt( Crypt_result rc = MY_AES_OK; uint dst_len; byte aes_ctr_counter[MY_AES_BLOCK_SIZE]; - byte is_encrypt= what == ENCRYPTION_FLAG_ENCRYPT; - lsn_t lsn = is_encrypt ? log_sys->lsn : srv_start_lsn; const uint src_len = OS_FILE_LOG_BLOCK_SIZE - LOG_BLOCK_HDR_SIZE; - for (ulint i = 0; i < size ; i += OS_FILE_LOG_BLOCK_SIZE) { + for (ulint i = 0; i < size ; i += OS_FILE_LOG_BLOCK_SIZE, + lsn += OS_FILE_LOG_BLOCK_SIZE) { ulint log_block_no = log_block_get_hdr_no(log_block); - lsn_t log_block_start_lsn = log_block_get_start_lsn( - lsn, log_block_no); const crypt_info_t* info = crypt_info == NULL ? get_crypt_info(log_block) : crypt_info; #ifdef DEBUG_CRYPT fprintf(stderr, "%s %lu chkpt: %lu key: %u lsn: %lu\n", - is_encrypt ? "crypt" : "decrypt", + what == ENCRYPTION_FLAG_ENCRYPT ? "crypt" : "decrypt", log_block_no, log_block_get_checkpoint_no(log_block), info ? info->key_version : 0, @@ -213,7 +196,7 @@ log_blocks_crypt( // (1-byte, only 5 bits are used). "+" means concatenate. bzero(aes_ctr_counter, MY_AES_BLOCK_SIZE); memcpy(aes_ctr_counter, info->crypt_nonce, 3); - mach_write_to_8(aes_ctr_counter + 3, log_block_start_lsn); + mach_write_to_8(aes_ctr_counter + 3, lsn); mach_write_to_4(aes_ctr_counter + 11, log_block_no); bzero(aes_ctr_counter + 15, 1); @@ -459,19 +442,6 @@ add_crypt_info( } /*********************************************************************//** -Encrypt log blocks. */ -UNIV_INTERN -Crypt_result -log_blocks_encrypt( -/*===============*/ - const byte* block, /*!< in: blocks before encryption */ - const ulint size, /*!< in: size of blocks, must be multiple of a log block */ - byte* dst_block) /*!< out: blocks after encryption */ -{ - return log_blocks_crypt(block, size, dst_block, ENCRYPTION_FLAG_ENCRYPT, NULL); -} - -/*********************************************************************//** Set next checkpoint's key version to latest one, and generate current key. Key version 0 means no encryption. */ UNIV_INTERN @@ -522,6 +492,8 @@ log_encrypt_before_write( /*=====================*/ ib_uint64_t next_checkpoint_no, /*!< in: log group to be flushed */ byte* block, /*!< in/out: pointer to a log block */ + lsn_t lsn, /*!< in: log sequence number of + the start of the buffer */ const ulint size) /*!< in: size of log blocks */ { ut_ad(size % OS_FILE_LOG_BLOCK_SIZE == 0); @@ -540,7 +512,8 @@ log_encrypt_before_write( byte* dst_frame = (byte*)malloc(size); //encrypt log blocks content - Crypt_result result = log_blocks_crypt(block, size, dst_frame, ENCRYPTION_FLAG_ENCRYPT, NULL); + Crypt_result result = log_blocks_crypt( + block, lsn, size, dst_frame, ENCRYPTION_FLAG_ENCRYPT, NULL); if (result == MY_AES_OK) { ut_ad(block[0] == dst_frame[0]); @@ -560,13 +533,16 @@ void log_decrypt_after_read( /*===================*/ byte* frame, /*!< in/out: log segment */ + lsn_t lsn, /*!< in: log sequence number of the start + of the buffer */ const ulint size) /*!< in: log segment size */ { ut_ad(size % OS_FILE_LOG_BLOCK_SIZE == 0); byte* dst_frame = (byte*)malloc(size); // decrypt log blocks content - Crypt_result result = log_blocks_crypt(frame, size, dst_frame, ENCRYPTION_FLAG_DECRYPT, NULL); + Crypt_result result = log_blocks_crypt( + frame, lsn, size, dst_frame, ENCRYPTION_FLAG_DECRYPT, NULL); if (result == MY_AES_OK) { memcpy(frame, dst_frame, size); diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index 1542f4a646f..8f8984f8880 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -1490,7 +1490,7 @@ loop: ut_a(next_offset / UNIV_PAGE_SIZE <= ULINT_MAX); log_encrypt_before_write(log_sys->next_checkpoint_no, - buf, write_len); + buf, start_lsn, write_len); #ifdef DEBUG_CRYPT fprintf(stderr, "WRITE: block: %lu checkpoint: %lu %.8lx %.8lx\n", @@ -2582,7 +2582,7 @@ loop: log_block_get_checksum(buf), source_offset); #endif - log_decrypt_after_read(buf, len); + log_decrypt_after_read(buf, start_lsn, len); #ifdef DEBUG_CRYPT fprintf(stderr, "AFTER DECRYPT: block: %lu checkpoint: %lu %.8lx %.8lx\n", @@ -2890,7 +2890,8 @@ loop: MONITOR_INC(MONITOR_LOG_IO); //TODO (jonaso): This must be dead code?? - log_encrypt_before_write(log_sys->next_checkpoint_no, buf, len); + log_encrypt_before_write(log_sys->next_checkpoint_no, + buf, start_lsn, len); fil_io(OS_FILE_WRITE | OS_FILE_LOG, false, group->archive_space_id, 0, diff --git a/storage/xtradb/trx/trx0purge.cc b/storage/xtradb/trx/trx0purge.cc index ff82bb2ad4e..cbf783628f9 100644 --- a/storage/xtradb/trx/trx0purge.cc +++ b/storage/xtradb/trx/trx0purge.cc @@ -584,32 +584,6 @@ trx_purge_rseg_get_next_history_log( mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); - - mutex_enter(&trx_sys->mutex); - - /* Add debug code to track history list corruption reported - on the MySQL mailing list on Nov 9, 2004. The fut0lst.cc - file-based list was corrupt. The prev node pointer was - FIL_NULL, even though the list length was over 8 million nodes! - We assume that purge truncates the history list in large - size pieces, and if we here reach the head of the list, the - list cannot be longer than 2000 000 undo logs now. */ - - if (trx_sys->rseg_history_len > 2000000) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Warning: purge reached the" - " head of the history list,\n" - "InnoDB: but its length is still" - " reported as %lu! Make a detailed bug\n" - "InnoDB: report, and submit it" - " to https://jira.mariadb.org/\n", - (ulong) trx_sys->rseg_history_len); - ut_ad(0); - } - - mutex_exit(&trx_sys->mutex); - return; } |