diff options
36 files changed, 358 insertions, 283 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 13d9b1be546..4734c6f7d5f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -187,6 +187,7 @@ ENDIF() OPTION(NOT_FOR_DISTRIBUTION "Allow linking with GPLv2-incompatible system libraries. Only set it you never plan to distribute the resulting binaries" OFF) INCLUDE(check_compiler_flag) +INCLUDE(check_linker_flag) OPTION(WITH_ASAN "Enable address sanitizer" OFF) IF (WITH_ASAN) @@ -239,7 +240,7 @@ IF(SECURITY_HARDENED) ENDIF() # security-enhancing flags MY_CHECK_AND_SET_COMPILER_FLAG("-pie -fPIC") - MY_CHECK_AND_SET_COMPILER_FLAG("-Wl,-z,relro,-z,now") + MY_CHECK_AND_SET_LINKER_FLAG("-Wl,-z,relro,-z,now") MY_CHECK_AND_SET_COMPILER_FLAG("-fstack-protector --param=ssp-buffer-size=4") MY_CHECK_AND_SET_COMPILER_FLAG("-D_FORTIFY_SOURCE=2" RELEASE RELWITHDEBINFO) ENDIF() diff --git a/cmake/check_linker_flag.cmake b/cmake/check_linker_flag.cmake new file mode 100644 index 00000000000..ff4b91e89f6 --- /dev/null +++ b/cmake/check_linker_flag.cmake @@ -0,0 +1,27 @@ +include(CheckCXXSourceCompiles) + +FUNCTION(MY_CHECK_AND_SET_LINKER_FLAG flag_to_set) + # Let's avoid expensive compiler tests on Windows: + IF(WIN32) + RETURN() + ENDIF() + STRING(REGEX REPLACE "[-,= +]" "_" result "HAVE_LINK_FLAG_${flag_to_set}") + SET(SAVE_CMAKE_REQUIRED_LINK_OPTIONS "${CMAKE_REQUIRED_LINK_OPTIONS}") + STRING(REGEX REPLACE "^-Wno-" "-W" flag_to_check ${flag_to_set}) + SET(CMAKE_REQUIRED_LINK_OPTIONS ${CMAKE_REQUIRED_LINK_OPTIONS} ${flag_to_check}) + CHECK_CXX_SOURCE_COMPILES("int main(void) { return 0; }" ${result}) + SET(CMAKE_REQUIRED_LINK_OPTIONS "${SAVE_CMAKE_REQUIRED_LINK_OPTIONS}") + IF (${result}) + FOREACH(linktype SHARED MODULE EXE) + IF(ARGN) + FOREACH(type ${ARGN}) + SET(CMAKE_${linktype}_LINKER_FLAGS_${type} + "${CMAKE_${linktype}_LINKER_FLAGS_${type}} ${flag_to_set}" PARENT_SCOPE) + ENDFOREACH() + ELSE() + SET(CMAKE_${linktype}_LINKER_FLAGS + "${CMAKE_${linktype}_LINKER_FLAGS} ${flag_to_set}" PARENT_SCOPE) + ENDIF() + ENDFOREACH() + ENDIF() +ENDFUNCTION() diff --git a/mysql-test/r/stat_tables_innodb.result b/mysql-test/r/stat_tables_innodb.result index a436efb5fea..23b952cc891 100644 --- a/mysql-test/r/stat_tables_innodb.result +++ b/mysql-test/r/stat_tables_innodb.result @@ -855,4 +855,28 @@ drop table t1; # please keep this at the last set @@global.histogram_size=@save_histogram_size; set optimizer_switch=@save_optimizer_switch_for_stat_tables_test; +# +# MDEV-22851: Engine independent index statistics are incorrect for large tables on Windows. +# +CREATE TABLE t1(a INT) ENGINE=INNODB; +INSERT INTO t1 SELECT 1 FROM seq_1_to_60000; +SET @save_use_stat_tables= @@use_stat_tables; +SET use_stat_tables= preferably; +SELECT count(*) FROM t1; +count(*) +60000 +CREATE INDEX idx ON t1(a); +ANALYZE TABLE t1 PERSISTENT FOR COLUMNS (a) INDEXES (idx); +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +SELECT * FROM mysql.index_stats where table_name='t1'; +db_name table_name index_name prefix_arity avg_frequency +test t1 idx 1 60000.0000 +SELECT * FROM mysql.column_stats where table_name='t1'; +db_name table_name column_name min_value max_value nulls_ratio avg_length avg_frequency hist_size hist_type histogram +test t1 a 1 1 0.0000 4.0000 60000.0000 0 NULL NULL +SET use_stat_tables= @save_use_stat_tables; +DROP TABLE t1; +# end of 10.1 tests SET SESSION STORAGE_ENGINE=DEFAULT; diff --git a/mysql-test/t/stat_tables_innodb.test b/mysql-test/t/stat_tables_innodb.test index 04e81de8f9d..008ca6911e9 100644 --- a/mysql-test/t/stat_tables_innodb.test +++ b/mysql-test/t/stat_tables_innodb.test @@ -1,4 +1,5 @@ --source include/have_innodb.inc +--source include/have_sequence.inc SET SESSION STORAGE_ENGINE='InnoDB'; @@ -9,4 +10,23 @@ set optimizer_switch='extended_keys=on'; set optimizer_switch=@save_optimizer_switch_for_stat_tables_test; +--echo # +--echo # MDEV-22851: Engine independent index statistics are incorrect for large tables on Windows. +--echo # + +CREATE TABLE t1(a INT) ENGINE=INNODB; +INSERT INTO t1 SELECT 1 FROM seq_1_to_60000; + +SET @save_use_stat_tables= @@use_stat_tables; +SET use_stat_tables= preferably; +SELECT count(*) FROM t1; +CREATE INDEX idx ON t1(a); +ANALYZE TABLE t1 PERSISTENT FOR COLUMNS (a) INDEXES (idx); +SELECT * FROM mysql.index_stats where table_name='t1'; +SELECT * FROM mysql.column_stats where table_name='t1'; +SET use_stat_tables= @save_use_stat_tables; +DROP TABLE t1; + +--echo # end of 10.1 tests + SET SESSION STORAGE_ENGINE=DEFAULT; diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc index d6ee4f10106..d22921ba1e2 100644 --- a/sql/sql_statistics.cc +++ b/sql/sql_statistics.cc @@ -2108,8 +2108,8 @@ int alloc_statistics_for_table(THD* thd, TABLE *table) sizeof(Index_statistics) * keys); uint key_parts= table->s->ext_key_parts; - ulong *idx_avg_frequency= (ulong*) alloc_root(&table->mem_root, - sizeof(ulong) * key_parts); + ulonglong *idx_avg_frequency= (ulonglong*) alloc_root(&table->mem_root, + sizeof(ulonglong) * key_parts); uint columns= 0; for (field_ptr= table->field; *field_ptr; field_ptr++) @@ -2154,7 +2154,7 @@ int alloc_statistics_for_table(THD* thd, TABLE *table) } } - memset(idx_avg_frequency, 0, sizeof(ulong) * key_parts); + memset(idx_avg_frequency, 0, sizeof(ulonglong) * key_parts); KEY *key_info, *end; for (key_info= table->key_info, end= key_info + table->s->keys; @@ -2270,14 +2270,14 @@ static int alloc_statistics_for_table_share(THD* thd, TABLE_SHARE *table_share) } uint key_parts= table_share->ext_key_parts; - ulong *idx_avg_frequency= table_stats->idx_avg_frequency; + ulonglong *idx_avg_frequency= table_stats->idx_avg_frequency; if (!idx_avg_frequency) { - idx_avg_frequency= (ulong*) alloc_root(&stats_cb->mem_root, - sizeof(ulong) * key_parts); + idx_avg_frequency= (ulonglong*) alloc_root(&stats_cb->mem_root, + sizeof(ulonglong) * key_parts); if (idx_avg_frequency) { - memset(idx_avg_frequency, 0, sizeof(ulong) * key_parts); + memset(idx_avg_frequency, 0, sizeof(ulonglong) * key_parts); table_stats->idx_avg_frequency= idx_avg_frequency; for (key_info= table_share->key_info, end= key_info + keys; key_info < end; diff --git a/sql/sql_statistics.h b/sql/sql_statistics.h index 145c207af58..fa3e8d6b894 100644 --- a/sql/sql_statistics.h +++ b/sql/sql_statistics.h @@ -279,7 +279,9 @@ public: uchar *min_max_record_buffers; /* Record buffers for min/max values */ Column_statistics *column_stats; /* Array of statistical data for columns */ Index_statistics *index_stats; /* Array of statistical data for indexes */ - ulong *idx_avg_frequency; /* Array of records per key for index prefixes */ + + /* Array of records per key for index prefixes */ + ulonglong *idx_avg_frequency; uchar *histograms; /* Sequence of histograms */ }; @@ -331,7 +333,7 @@ private: CHAR values are stripped of trailing spaces. Flexible values are stripped of their length prefixes. */ - ulong avg_length; + ulonglong avg_length; /* The ratio N/D multiplied by the scale factor Scale_factor_avg_frequency, @@ -339,7 +341,7 @@ private: N is the number of rows with not null value in the column, D the number of distinct values among them */ - ulong avg_frequency; + ulonglong avg_frequency; public: @@ -389,12 +391,12 @@ public: void set_avg_length (double val) { - avg_length= (ulong) (val * Scale_factor_avg_length); + avg_length= (ulonglong) (val * Scale_factor_avg_length); } void set_avg_frequency (double val) { - avg_frequency= (ulong) (val * Scale_factor_avg_frequency); + avg_frequency= (ulonglong) (val * Scale_factor_avg_frequency); } bool min_max_values_are_provided() @@ -433,11 +435,11 @@ private: in the first k components, and D is the number of distinct k-component prefixes among them */ - ulong *avg_frequency; + ulonglong *avg_frequency; public: - void init_avg_frequency(ulong *ptr) { avg_frequency= ptr; } + void init_avg_frequency(ulonglong *ptr) { avg_frequency= ptr; } bool avg_frequency_is_inited() { return avg_frequency != NULL; } @@ -448,7 +450,7 @@ public: void set_avg_frequency(uint i, double val) { - avg_frequency[i]= (ulong) (val * Scale_factor_avg_frequency); + avg_frequency[i]= (ulonglong) (val * Scale_factor_avg_frequency); } }; diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 7148a6fbfd0..6cc4bd6dc00 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -566,7 +566,7 @@ buf_dblwr_process() const ulint page_no = page_get_page_no(page); const page_id_t page_id(space_id, page_no); - if (page_no >= space->size) { + if (UNIV_UNLIKELY(page_no >= space->size)) { /* Do not report the warning if the tablespace is scheduled for truncation or was truncated @@ -574,10 +574,12 @@ buf_dblwr_process() if (!srv_is_tablespace_truncated(space_id) && !srv_was_tablespace_truncated(space) && !srv_is_undo_tablespace(space_id)) { - ib::warn() << "A copy of page " << page_id + ib::warn() << "A copy of page " << page_no << " in the doublewrite buffer slot " << page_no_dblwr - << " is not within space bounds"; + << " is beyond the end of tablespace " + << space->name + << " (" << space->size << " pages)"; } continue; } diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 9b395bd3231..26610337d0d 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1391,9 +1391,11 @@ buf_flush_try_neighbors( } } - const ulint space_size = fil_space_get_size(page_id.space()); - if (high > space_size) { - high = space_size; + if (fil_space_t *s = fil_space_acquire_for_io(page_id.space())) { + high = s->max_page_number_for_io(high); + fil_space_release_for_io(s); + } else { + return 0; } DBUG_PRINT("ib_buf", ("flush %u:%u..%u", diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 43831dedb2c..d6f096fc80e 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -279,11 +279,7 @@ buf_read_ahead_random( high = (page_id.page_no() / buf_read_ahead_random_area + 1) * buf_read_ahead_random_area; - /* Remember the tablespace version before we ask the tablespace size - below: if DISCARD + IMPORT changes the actual .ibd file meanwhile, we - do not try to read outside the bounds of the tablespace! */ if (fil_space_t* space = fil_space_acquire(page_id.space())) { - #ifdef UNIV_DEBUG if (srv_file_per_table) { ulint size = 0; @@ -301,9 +297,7 @@ buf_read_ahead_random( } #endif /* UNIV_DEBUG */ - if (high > space->size) { - high = space->size; - } + high = space->max_page_number_for_io(high); fil_space_release(space); } else { return(0); @@ -580,13 +574,10 @@ buf_read_ahead_linear( return(0); } - /* Remember the tablespace version before we ask te tablespace size - below: if DISCARD + IMPORT changes the actual .ibd file meanwhile, we - do not try to read outside the bounds of the tablespace! */ ulint space_size; if (fil_space_t* space = fil_space_acquire(page_id.space())) { - space_size = space->size; + space_size = space->committed_size; fil_space_release(space); if (high > space_size) { diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 42f7a7fc3c9..f9d7070a41b 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -359,34 +359,6 @@ fil_space_get( return(space); } -/** Returns the latch of a file space. -@param[in] id space id -@param[out] flags tablespace flags -@return latch protecting storage allocation */ -rw_lock_t* -fil_space_get_latch( - ulint id, - ulint* flags) -{ - fil_space_t* space; - - ut_ad(fil_system); - - mutex_enter(&fil_system->mutex); - - space = fil_space_get_by_id(id); - - ut_a(space); - - if (flags) { - *flags = space->flags; - } - - mutex_exit(&fil_system->mutex); - - return(&(space->latch)); -} - /** Gets the type of a file space. @param[in] id tablespace identifier @return file type */ @@ -593,13 +565,17 @@ bool fil_node_t::read_page0(bool first) } this->size = ulint(size_bytes / psize); - space->size += this->size; + space->committed_size = space->size += this->size; } else if (space->id != TRX_SYS_SPACE || space->size_in_header) { /* If this is not the first-time open, do nothing. For the system tablespace, we always get invoked as first=false, so we detect the true first-time-open based - on size_in_header and proceed to initiailze the data. */ + on size_in_header and proceed to initialize the data. */ return true; + } else { + /* Initialize the size of predefined tablespaces + to FSP_SIZE. */ + space->committed_size = size; } ut_ad(space->free_limit == 0 || space->free_limit == free_limit); @@ -1126,6 +1102,9 @@ fil_mutex_enter_and_prepare_for_io( ut_a(success); /* InnoDB data files cannot shrink. */ ut_a(space->size >= size); + if (size > space->committed_size) { + space->committed_size = size; + } /* There could be multiple concurrent I/O requests for this tablespace (multiple threads trying to extend diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 2501f4f8ba3..8a393b15952 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -422,7 +422,7 @@ xdes_get_descriptor_with_space_hdr( ulint size; ulint descr_page_no; page_t* descr_page; - ut_ad(mtr_memo_contains(mtr, &space->latch, MTR_MEMO_X_LOCK)); + ut_ad(mtr_memo_contains(mtr, space, MTR_MEMO_SPACE_X_LOCK)); ut_ad(mtr_memo_contains_page(mtr, sp_header, MTR_MEMO_PAGE_SX_FIX)); ut_ad(page_offset(sp_header) == FSP_HEADER_OFFSET); /* Read free limit and space size */ @@ -566,7 +566,7 @@ xdes_lst_get_descriptor( fil_addr_t lst_node, mtr_t* mtr) { - ut_ad(mtr_memo_contains(mtr, &space->latch, MTR_MEMO_X_LOCK)); + ut_ad(mtr_memo_contains(mtr, space, MTR_MEMO_SPACE_X_LOCK)); ut_ad(page_size.equals_to(page_size_t(space->flags))); return(fut_get_ptr(space->id, page_size, lst_node, RW_SX_LATCH, mtr) - XDES_FLST_NODE); @@ -1112,42 +1112,22 @@ fsp_fill_free_list( MLOG_2BYTES, mtr); } - /* Initialize the ibuf bitmap page in a separate - mini-transaction because it is low in the latching - order, and we must be able to release its latch. - Note: Insert-Buffering is disabled for tables that - reside in the temp-tablespace. */ if (space->purpose != FIL_TYPE_TEMPORARY) { - mtr_t ibuf_mtr; - - mtr_start(&ibuf_mtr); - ibuf_mtr.set_named_space(space); - - /* Avoid logging while truncate table - fix-up is active. */ - if (srv_is_tablespace_truncated(space->id)) { - mtr_set_log_mode( - &ibuf_mtr, MTR_LOG_NO_REDO); - } - const page_id_t page_id( space->id, i + FSP_IBUF_BITMAP_OFFSET); block = buf_page_create( - page_id, page_size, &ibuf_mtr); + page_id, page_size, mtr); buf_page_get( - page_id, page_size, RW_SX_LATCH, - &ibuf_mtr); + page_id, page_size, RW_SX_LATCH, mtr); buf_block_dbg_add_level(block, SYNC_FSP_PAGE); - fsp_init_file_page(space, block, &ibuf_mtr); - - ibuf_bitmap_page_init(block, &ibuf_mtr); + fsp_init_file_page(space, block, mtr); - mtr_commit(&ibuf_mtr); + ibuf_bitmap_page_init(block, mtr); } } @@ -1569,7 +1549,7 @@ fsp_free_extent( fsp_header_t* header; xdes_t* descr; - ut_ad(mtr_memo_contains(mtr, &space->latch, MTR_MEMO_X_LOCK)); + ut_ad(mtr_memo_contains(mtr, space, MTR_MEMO_SPACE_X_LOCK)); header = fsp_get_space_header(space, page_size, mtr); diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 2eea443a434..7e01c864a50 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -1984,7 +1984,7 @@ ibuf_add_free_page(void) /* Acquire the fsp latch before the ibuf header, obeying the latching order */ - mtr_x_lock(&space->latch, &mtr); + mtr.x_lock_space(space, __FILE__, __LINE__); header_page = ibuf_header_page_get(&mtr); /* Allocate a new page: NOTE that if the page has been a part of a @@ -2068,7 +2068,7 @@ ibuf_remove_free_page(void) /* Acquire the fsp latch before the ibuf header, obeying the latching order */ - mtr_x_lock(&space->latch, &mtr); + mtr.x_lock_space(space, __FILE__, __LINE__); header_page = ibuf_header_page_get(&mtr); /* Prevent pessimistic inserts to insert buffer trees for a while */ diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 62d87ce2c06..585a284798e 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -135,6 +135,8 @@ struct fil_space_t : ilist_node<unflushed_spaces_tag_t>, /*!< recovered tablespace size in pages; 0 if no size change was read from the redo log, or if the size change was implemented */ + /** the committed size of the tablespace in pages */ + ulint committed_size; ulint flags; /*!< FSP_SPACE_FLAGS and FSP_FLAGS_MEM_ flags; see fsp0types.h, fsp_flags_is_valid(), @@ -193,6 +195,15 @@ struct fil_space_t : ilist_node<unflushed_spaces_tag_t>, /** @return whether the tablespace is about to be dropped */ bool is_stopping() const { return stop_new_ops; } + /** Clamp a page number for batched I/O, such as read-ahead. + @param offset page number limit + @return offset clamped to the tablespace size */ + ulint max_page_number_for_io(ulint offset) const + { + const ulint limit= committed_size; + return limit > offset ? offset : limit; + } + /** @return whether doublewrite buffering is needed */ bool use_doublewrite() const { @@ -571,15 +582,6 @@ extern fil_system_t* fil_system; #include "fil0crypt.h" -/** Returns the latch of a file space. -@param[in] id space id -@param[out] flags tablespace flags -@return latch protecting storage allocation */ -rw_lock_t* -fil_space_get_latch( - ulint id, - ulint* flags); - /** Gets the type of a file space. @param[in] id tablespace identifier @return file type */ diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index 30d2e937f1f..e91590d48d1 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2019, MariaDB Corporation. +Copyright (c) 2013, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -302,6 +302,12 @@ struct mtr_t { const char* file, unsigned line); + /** Exclusively aqcuire a tablespace latch. + @param space tablespace + @param file source code file name of the caller + @param line source code line number */ + void x_lock_space(fil_space_t *space, const char *file, unsigned line); + /** Release an object in the memo stack. @param object object @param type object type: MTR_MEMO_S_LOCK, ... diff --git a/storage/innobase/include/mtr0mtr.ic b/storage/innobase/include/mtr0mtr.ic index 7175ede0d6a..8d68affb1cb 100644 --- a/storage/innobase/include/mtr0mtr.ic +++ b/storage/innobase/include/mtr0mtr.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2019, MariaDB Corporation. +Copyright (c) 2017, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -48,7 +48,7 @@ mtr_t::memo_push(void* object, mtr_memo_type_t type) ut_ad(is_active()); ut_ad(object != NULL); ut_ad(type >= MTR_MEMO_PAGE_S_FIX); - ut_ad(type <= MTR_MEMO_SX_LOCK); + ut_ad(type <= MTR_MEMO_SPACE_X_LOCK); ut_ad(ut_is_2pow(type)); /* If this mtr has x-fixed a clean page then we set diff --git a/storage/innobase/include/mtr0types.h b/storage/innobase/include/mtr0types.h index 985ad7b81ea..2d4cd7b97ac 100644 --- a/storage/innobase/include/mtr0types.h +++ b/storage/innobase/include/mtr0types.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2019, MariaDB Corporation. +Copyright (c) 2017, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -263,7 +263,10 @@ enum mtr_memo_type_t { MTR_MEMO_X_LOCK = RW_X_LATCH << 5, - MTR_MEMO_SX_LOCK = RW_SX_LATCH << 5 + MTR_MEMO_SX_LOCK = RW_SX_LATCH << 5, + + /** acquire X-latch on fil_space_t::latch */ + MTR_MEMO_SPACE_X_LOCK = MTR_MEMO_SX_LOCK << 1 }; #endif /* !UNIV_CHECKSUM */ diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 9fdb10e230c..b649d907c23 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2019, MariaDB Corporation. +Copyright (c) 2017, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -218,6 +218,13 @@ static void memo_slot_release(mtr_memo_slot_t *slot) case MTR_MEMO_SX_LOCK: rw_lock_sx_unlock(reinterpret_cast<rw_lock_t*>(slot->object)); break; + case MTR_MEMO_SPACE_X_LOCK: + { + fil_space_t *space= static_cast<fil_space_t*>(slot->object); + space->committed_size= space->size; + rw_lock_x_unlock(&space->latch); + } + break; case MTR_MEMO_X_LOCK: rw_lock_x_unlock(reinterpret_cast<rw_lock_t*>(slot->object)); break; @@ -251,6 +258,13 @@ struct ReleaseLatches { case MTR_MEMO_S_LOCK: rw_lock_s_unlock(reinterpret_cast<rw_lock_t*>(slot->object)); break; + case MTR_MEMO_SPACE_X_LOCK: + { + fil_space_t *space= static_cast<fil_space_t*>(slot->object); + space->committed_size= space->size; + rw_lock_x_unlock(&space->latch); + } + break; case MTR_MEMO_X_LOCK: rw_lock_x_unlock(reinterpret_cast<rw_lock_t*>(slot->object)); break; @@ -564,13 +578,23 @@ mtr_t::x_lock_space(ulint space_id, const char* file, unsigned line) ut_ad(space); ut_ad(space->id == space_id); - x_lock(&space->latch, file, line); + x_lock_space(space, file, line); ut_ad(space->purpose == FIL_TYPE_TEMPORARY || space->purpose == FIL_TYPE_IMPORT || space->purpose == FIL_TYPE_TABLESPACE); return(space); } +/** Exclusively aqcuire a tablespace latch. +@param space tablespace +@param file source code file name of the caller +@param line source code line number */ +void mtr_t::x_lock_space(fil_space_t *space, const char *file, unsigned line) +{ + rw_lock_x_lock_inline(&space->latch, 0, file, line); + memo_push(space, MTR_MEMO_SPACE_X_LOCK); +} + /** Look up the system tablespace. */ void mtr_t::lookup_sys_space() @@ -764,27 +788,20 @@ with index pages. void mtr_t::release_free_extents(ulint n_reserved) { - fil_space_t* space; - - ut_ad(!m_undo_space); - - if (m_user_space) { + fil_space_t *space= m_user_space; - ut_ad(m_user_space->id == m_user_space_id); - ut_ad(memo_contains(get_memo(), &m_user_space->latch, - MTR_MEMO_X_LOCK)); + ut_ad(!m_undo_space); - space = m_user_space; - } else { - - ut_ad(m_sys_space->id == TRX_SYS_SPACE); - ut_ad(memo_contains(get_memo(), &m_sys_space->latch, - MTR_MEMO_X_LOCK)); - - space = m_sys_space; - } + if (space) + ut_ad(m_user_space->id == m_user_space_id); + else + { + ut_ad(m_sys_space->id == TRX_SYS_SPACE); + space= m_sys_space; + } - space->release_free_extents(n_reserved); + ut_ad(memo_contains(get_memo(), space, MTR_MEMO_SPACE_X_LOCK)); + space->release_free_extents(n_reserved); } #ifdef UNIV_DEBUG diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index ed78b2a154d..981c5046de1 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -527,14 +527,6 @@ protected: /** Space id of the file being iterated over. */ ulint m_space; - /** Minimum page number for which the free list has not been - initialized: the pages >= this limit are, by definition, free; - note that in a single-table tablespace where size < 64 pages, - this number is 64, i.e., we have initialized the space about - the first extent, but have not physically allocted those pages - to the file. @see FSP_LIMIT. */ - ulint m_free_limit; - /** Current size of the space in pages */ ulint m_size; @@ -594,7 +586,6 @@ AbstractCallback::init( ut_a(m_space == ULINT_UNDEFINED); m_size = mach_read_from_4(page + FSP_SIZE); - m_free_limit = mach_read_from_4(page + FSP_FREE_LIMIT); m_space = mach_read_from_4(page + FSP_HEADER_OFFSET + FSP_SPACE_ID); return set_current_xdes(0, page); diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 8479847f779..0d8ebbe98cd 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -3,7 +3,7 @@ Copyright (c) 1996, 2017, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2019, MariaDB Corporation. +Copyright (c) 2013, 2020, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -696,6 +696,7 @@ static bool srv_undo_tablespace_open(const char* name, ulint space_id, if (create_new_db) { space->size = file->size = ulint(size >> srv_page_size_shift); space->size_in_header = SRV_UNDO_TABLESPACE_SIZE_IN_PAGES; + space->committed_size = SRV_UNDO_TABLESPACE_SIZE_IN_PAGES; } else { success = file->read_page0(true); if (!success) { @@ -1044,7 +1045,7 @@ srv_undo_tablespaces_init(bool create_new_db) fsp_header_init( *it, SRV_UNDO_TABLESPACE_SIZE_IN_PAGES, &mtr); - mtr_x_lock(fil_space_get_latch(*it, NULL), &mtr); + mtr_x_lock_space(*it, &mtr); for (ulint i = 0; i < TRX_SYS_N_RSEGS; i++) { diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 2f728b22fb2..2025ac70beb 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -995,7 +995,7 @@ trx_purge_initiate_truncate( mtr_t mtr; const ulint size = SRV_UNDO_TABLESPACE_SIZE_IN_PAGES; mtr.start(); - mtr_x_lock(&space->latch, &mtr); + mtr.x_lock_space(space, __FILE__, __LINE__); fil_truncate_log(space, size, &mtr); fsp_header_init(space_id, size, &mtr); mutex_enter(&fil_system->mutex); diff --git a/storage/innobase/trx/trx0rseg.cc b/storage/innobase/trx/trx0rseg.cc index e76695b7e43..60ebfa3c9a8 100644 --- a/storage/innobase/trx/trx0rseg.cc +++ b/storage/innobase/trx/trx0rseg.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2019, MariaDB Corporation. +Copyright (c) 2017, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -53,8 +53,8 @@ trx_rseg_header_create( buf_block_t* block; ut_ad(mtr); - ut_ad(mtr_memo_contains(mtr, fil_space_get_latch(space, NULL), - MTR_MEMO_X_LOCK)); + ut_ad(mtr_memo_contains(mtr, fil_space_get(space), + MTR_MEMO_SPACE_X_LOCK)); /* Allocate a new file segment for the rollback segment */ block = fseg_create(space, 0, TRX_RSEG + TRX_RSEG_FSEG_HEADER, mtr); diff --git a/storage/innobase/trx/trx0sys.cc b/storage/innobase/trx/trx0sys.cc index 415afc4a90b..2225c69b62e 100644 --- a/storage/innobase/trx/trx0sys.cc +++ b/storage/innobase/trx/trx0sys.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2019, MariaDB Corporation. +Copyright (c) 2017, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index 9f03778e572..a405c3edf5e 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -554,12 +554,18 @@ buf_dblwr_process() continue; } - if (!fil_check_adress_in_tablespace(space_id, page_no)) { + if (!space()->size) { + fil_space_get_size(space_id); + } + + if (UNIV_UNLIKELY(space()->size <= page_no)) { ib_logf(IB_LOG_LEVEL_WARN, - "A copy of page " ULINTPF ":" ULINTPF + "A copy of page " ULINTPF " in the doublewrite buffer slot " ULINTPF - " is not within space bounds", - space_id, page_no, page_no_dblwr); + " is beyond the end of the tablespace " + " %s (" ULINTPF " pages)", + page_no, page_no_dblwr, + space()->name, space()->size); continue; } diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index 9c427395231..2a656d748b1 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2019, MariaDB Corporation. +Copyright (c) 2013, 2020, MariaDB Corporation. Copyright (c) 2013, 2014, Fusion-io This program is free software; you can redistribute it and/or modify it under @@ -1297,8 +1297,11 @@ buf_flush_try_neighbors( /* fprintf(stderr, "Flush area: low %lu high %lu\n", low, high); */ - if (high > fil_space_get_size(space)) { - high = fil_space_get_size(space); + if (fil_space_t *s = fil_space_acquire_for_io(space)) { + high = s->max_page_number_for_io(high); + fil_space_release_for_io(s); + } else { + return 0; } ulint count = 0; diff --git a/storage/xtradb/buf/buf0rea.cc b/storage/xtradb/buf/buf0rea.cc index 1ef9162cab9..a962b94c0c2 100644 --- a/storage/xtradb/buf/buf0rea.cc +++ b/storage/xtradb/buf/buf0rea.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2018, MariaDB Corporation. +Copyright (c) 2013, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -344,19 +344,22 @@ buf_read_ahead_random( return(0); } - /* Remember the tablespace version before we ask te tablespace size - below: if DISCARD + IMPORT changes the actual .ibd file meanwhile, we - do not try to read outside the bounds of the tablespace! */ + if (fil_space_t *s = fil_space_acquire_for_io(space)) { + /* Remember the tablespace version along with the + tablespace size: if DISCARD + IMPORT changes the + actual .ibd file meanwhile, we do not try to read + outside the bounds of the tablespace! */ + tablespace_version = s->tablespace_version; - tablespace_version = fil_space_get_version(space); - - low = (offset / buf_read_ahead_random_area) - * buf_read_ahead_random_area; - high = (offset / buf_read_ahead_random_area + 1) - * buf_read_ahead_random_area; - if (high > fil_space_get_size(space)) { + low = (offset / buf_read_ahead_random_area) + * buf_read_ahead_random_area; + high = (offset / buf_read_ahead_random_area + 1) + * buf_read_ahead_random_area; + high = s->max_page_number_for_io(high); - high = fil_space_get_size(space); + fil_space_release_for_io(s); + } else { + return 0; } if (buf_pool->n_pend_reads @@ -495,22 +498,16 @@ buf_read_page( ulint offset, trx_t* trx) { - ib_int64_t tablespace_version; - ulint count; dberr_t err = DB_SUCCESS; - tablespace_version = fil_space_get_version(space_id); - FilSpace space(space_id, true); if (space()) { - - /* We do the i/o in the synchronous aio mode to save thread - switches: hence TRUE */ - count = buf_read_page_low(&err, true, BUF_READ_ANY_PAGE, space_id, - zip_size, FALSE, - tablespace_version, offset, trx); - + ulint count = buf_read_page_low(&err, /*sync=*/true, + BUF_READ_ANY_PAGE, + space_id, zip_size, FALSE, + space()->tablespace_version, + offset, trx); srv_stats.buf_pool_reads.add(count); } @@ -683,13 +680,23 @@ buf_read_ahead_linear( return(0); } - /* Remember the tablespace version before we ask te tablespace size - below: if DISCARD + IMPORT changes the actual .ibd file meanwhile, we - do not try to read outside the bounds of the tablespace! */ + uint32_t space_high_limit = 0; - tablespace_version = fil_space_get_version(space); + if (fil_space_t *s = fil_space_acquire_for_io(space)) { + /* Remember the tablespace version along with the + tablespace size: if DISCARD + IMPORT changes the + actual .ibd file meanwhile, we do not try to read + outside the bounds of the tablespace! */ + tablespace_version = s->tablespace_version; + + space_high_limit = s->max_page_number_for_io(ULINT_UNDEFINED); + + fil_space_release_for_io(s); + } else { + return 0; + } - if (high > fil_space_get_size(space)) { + if (high > space_high_limit) { /* The area is not whole, return */ return(0); @@ -825,7 +832,7 @@ buf_read_ahead_linear( return(0); } - if (high > fil_space_get_size(space)) { + if (high > space_high_limit) { /* The area is not whole, return */ return(0); diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 4a982df3fda..d77d7320318 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2019, MariaDB Corporation. +Copyright (c) 2014, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -684,7 +684,7 @@ fil_node_open_file( #ifdef UNIV_HOTBACKUP add_size: #endif /* UNIV_HOTBACKUP */ - space->size += node->size; + space->committed_size = space->size += node->size; } ulint atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(space->flags); @@ -1158,6 +1158,9 @@ retry: ut_a(success); /* InnoDB data files cannot shrink. */ ut_a(space->size >= size); + if (size > space->committed_size) { + space->committed_size = size; + } /* There could be multiple concurrent I/O requests for this tablespace (multiple threads trying to extend @@ -1815,25 +1818,6 @@ fil_space_get_zip_size( return(flags); } -/*******************************************************************//** -Checks if the pair space, page_no refers to an existing page in a tablespace -file space. The tablespace must be cached in the memory cache. -@return TRUE if the address is meaningful */ -UNIV_INTERN -ibool -fil_check_adress_in_tablespace( -/*===========================*/ - ulint id, /*!< in: space id */ - ulint page_no)/*!< in: page number */ -{ - if (fil_space_get_size(id) > page_no) { - - return(TRUE); - } - - return(FALSE); -} - /****************************************************************//** Initializes the tablespace memory cache. */ UNIV_INTERN diff --git a/storage/xtradb/fsp/fsp0fsp.cc b/storage/xtradb/fsp/fsp0fsp.cc index 5604a1b48c5..e9d26a5932b 100644 --- a/storage/xtradb/fsp/fsp0fsp.cc +++ b/storage/xtradb/fsp/fsp0fsp.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2018, MariaDB Corporation. +Copyright (c) 2017, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -450,7 +450,7 @@ xdes_get_descriptor_with_space_hdr( page_t* descr_page; ut_ad(mtr_memo_contains(mtr, fil_space_get_latch(space, NULL), - MTR_MEMO_X_LOCK)); + MTR_MEMO_SPACE_X_LOCK)); ut_ad(mtr_memo_contains_page(mtr, sp_header, MTR_MEMO_PAGE_X_FIX)); ut_ad(page_offset(sp_header) == FSP_HEADER_OFFSET); /* Read free limit and space size */ @@ -534,7 +534,7 @@ xdes_lst_get_descriptor( ut_ad(mtr); ut_ad(mtr_memo_contains(mtr, fil_space_get_latch(space, NULL), - MTR_MEMO_X_LOCK)); + MTR_MEMO_SPACE_X_LOCK)); descr = fut_get_ptr(space, zip_size, lst_node, RW_X_LATCH, mtr) - XDES_FLST_NODE; @@ -690,7 +690,7 @@ fsp_header_init(ulint space_id, ulint size, mtr_t* mtr) ut_ad(mtr); - mtr_x_lock(fil_space_get_latch(space_id, &flags), mtr); + mtr_x_space_lock(fil_space_get_latch(space_id, &flags), mtr); zip_size = fsp_flags_get_zip_size(flags); block = buf_page_create(space_id, 0, zip_size, mtr); @@ -817,7 +817,7 @@ fsp_header_inc_size( ut_ad(mtr); - mtr_x_lock(fil_space_get_latch(space, &flags), mtr); + mtr_x_space_lock(fil_space_get_latch(space, &flags), mtr); header = fsp_get_space_header(space, fsp_flags_get_zip_size(flags), @@ -846,7 +846,7 @@ fsp_header_get_tablespace_size(void) mtr_start(&mtr); - mtr_x_lock(fil_space_get_latch(0, NULL), &mtr); + mtr_x_space_lock(fil_space_get_latch(0, NULL), &mtr); header = fsp_get_space_header(0, 0, &mtr); @@ -1073,7 +1073,6 @@ fsp_fill_free_list( ulint frag_n_used; ulint actual_increase; ulint i; - mtr_t ibuf_mtr; ut_ad(page_offset(header) == FSP_HEADER_OFFSET); @@ -1141,26 +1140,17 @@ fsp_fill_free_list( MLOG_2BYTES, mtr); } - /* Initialize the ibuf bitmap page in a separate - mini-transaction because it is low in the latching - order, and we must be able to release its latch - before returning from the fsp routine */ - - mtr_start(&ibuf_mtr); - block = buf_page_create(space, - i + FSP_IBUF_BITMAP_OFFSET, - zip_size, &ibuf_mtr); + i + FSP_IBUF_BITMAP_OFFSET, + zip_size, mtr); buf_page_get(space, zip_size, i + FSP_IBUF_BITMAP_OFFSET, - RW_X_LATCH, &ibuf_mtr); + RW_X_LATCH, mtr); buf_block_dbg_add_level(block, SYNC_FSP_PAGE); - fsp_init_file_page(block, &ibuf_mtr); + fsp_init_file_page(block, mtr); - ibuf_bitmap_page_init(block, &ibuf_mtr); - - mtr_commit(&ibuf_mtr); + ibuf_bitmap_page_init(block, mtr); } descr = xdes_get_descriptor_with_space_hdr(header, space, i, @@ -2047,7 +2037,7 @@ fseg_create_general( header = byte_offset + buf_block_get_frame(block); } - mtr_x_lock(latch, mtr); + mtr_x_space_lock(latch, mtr); if (!has_done_reservation) { success = fsp_reserve_free_extents(&n_reserved, space, 2, @@ -2194,7 +2184,7 @@ fseg_n_reserved_pages( latch = fil_space_get_latch(space, &flags); zip_size = fsp_flags_get_zip_size(flags); - mtr_x_lock(latch, mtr); + mtr_x_space_lock(latch, mtr); inode = fseg_inode_get(header, space, zip_size, mtr); @@ -2617,7 +2607,7 @@ fseg_alloc_free_page_general( zip_size = fsp_flags_get_zip_size(flags); - mtr_x_lock(latch, mtr); + mtr_x_space_lock(latch, mtr); inode = fseg_inode_get(seg_header, space, zip_size, mtr); @@ -2734,7 +2724,7 @@ fsp_reserve_free_extents( latch = fil_space_get_latch(space, &flags); zip_size = fsp_flags_get_zip_size(flags); - mtr_x_lock(latch, mtr); + mtr_x_space_lock(latch, mtr); space_header = fsp_get_space_header(space, zip_size, mtr); try_again: @@ -2873,7 +2863,7 @@ fsp_get_available_space_in_free_extents( zip_size = fsp_flags_get_zip_size(flags); - mtr_x_lock(latch, &mtr); + mtr_x_space_lock(latch, &mtr); mutex_exit(&dict_sys->mutex); @@ -3169,7 +3159,7 @@ fseg_free_page( latch = fil_space_get_latch(space, &flags); zip_size = fsp_flags_get_zip_size(flags); - mtr_x_lock(latch, mtr); + mtr_x_space_lock(latch, mtr); seg_inode = fseg_inode_get(seg_header, space, zip_size, mtr); @@ -3203,7 +3193,7 @@ fseg_page_is_free( zip_size = dict_tf_get_zip_size(flags); mtr_start(&mtr); - mtr_x_lock(latch, &mtr); + mtr_x_space_lock(latch, &mtr); seg_inode = fseg_inode_get(seg_header, space, zip_size, &mtr); @@ -3328,7 +3318,7 @@ fseg_free_step( latch = fil_space_get_latch(space, &flags); zip_size = fsp_flags_get_zip_size(flags); - mtr_x_lock(latch, mtr); + mtr_x_space_lock(latch, mtr); descr = xdes_get_descriptor(space, zip_size, header_page, mtr); @@ -3415,7 +3405,7 @@ fseg_free_step_not_header( latch = fil_space_get_latch(space, &flags); zip_size = fsp_flags_get_zip_size(flags); - mtr_x_lock(latch, mtr); + mtr_x_space_lock(latch, mtr); inode = fseg_inode_get(header, space, zip_size, mtr); @@ -3541,7 +3531,7 @@ fseg_validate_low( ulint zip_size; mtr_start(&mtr); - mtr_x_lock(fil_space_get_latch(space, &flags), &mtr); + mtr_x_space_lock(fil_space_get_latch(space, &flags), &mtr); zip_size = fsp_flags_get_zip_size(flags); descr = xdes_lst_get_descriptor(space, zip_size, @@ -3564,7 +3554,7 @@ fseg_validate_low( ulint zip_size; mtr_start(&mtr); - mtr_x_lock(fil_space_get_latch(space, &flags), &mtr); + mtr_x_space_lock(fil_space_get_latch(space, &flags), &mtr); zip_size = fsp_flags_get_zip_size(flags); descr = xdes_lst_get_descriptor(space, zip_size, @@ -3590,7 +3580,7 @@ fseg_validate_low( ulint zip_size; mtr_start(&mtr); - mtr_x_lock(fil_space_get_latch(space, &flags), &mtr); + mtr_x_space_lock(fil_space_get_latch(space, &flags), &mtr); zip_size = fsp_flags_get_zip_size(flags); descr = xdes_lst_get_descriptor(space, zip_size, @@ -3628,7 +3618,7 @@ fseg_validate( space = page_get_space_id(page_align(header)); - mtr_x_lock(fil_space_get_latch(space, &flags), mtr); + mtr_x_space_lock(fil_space_get_latch(space, &flags), mtr); zip_size = fsp_flags_get_zip_size(flags); inode = fseg_inode_get(header, space, zip_size, mtr); @@ -3704,7 +3694,7 @@ fseg_print( space = page_get_space_id(page_align(header)); - mtr_x_lock(fil_space_get_latch(space, &flags), mtr); + mtr_x_space_lock(fil_space_get_latch(space, &flags), mtr); zip_size = fsp_flags_get_zip_size(flags); inode = fseg_inode_get(header, space, zip_size, mtr); @@ -3753,10 +3743,10 @@ fsp_validate( /* Start first a mini-transaction mtr2 to lock out all other threads from the fsp system */ mtr_start(&mtr2); - mtr_x_lock(latch, &mtr2); + mtr_x_space_lock(latch, &mtr2); mtr_start(&mtr); - mtr_x_lock(latch, &mtr); + mtr_x_space_lock(latch, &mtr); header = fsp_get_space_header(space, zip_size, &mtr); @@ -3783,7 +3773,7 @@ fsp_validate( /* Validate FSP_FREE list */ mtr_start(&mtr); - mtr_x_lock(latch, &mtr); + mtr_x_space_lock(latch, &mtr); header = fsp_get_space_header(space, zip_size, &mtr); node_addr = flst_get_first(header + FSP_FREE, &mtr); @@ -3792,7 +3782,7 @@ fsp_validate( while (!fil_addr_is_null(node_addr)) { mtr_start(&mtr); - mtr_x_lock(latch, &mtr); + mtr_x_space_lock(latch, &mtr); descr_count++; descr = xdes_lst_get_descriptor(space, zip_size, @@ -3807,7 +3797,7 @@ fsp_validate( /* Validate FSP_FREE_FRAG list */ mtr_start(&mtr); - mtr_x_lock(latch, &mtr); + mtr_x_space_lock(latch, &mtr); header = fsp_get_space_header(space, zip_size, &mtr); node_addr = flst_get_first(header + FSP_FREE_FRAG, &mtr); @@ -3816,7 +3806,7 @@ fsp_validate( while (!fil_addr_is_null(node_addr)) { mtr_start(&mtr); - mtr_x_lock(latch, &mtr); + mtr_x_space_lock(latch, &mtr); descr_count++; descr = xdes_lst_get_descriptor(space, zip_size, @@ -3834,7 +3824,7 @@ fsp_validate( /* Validate FSP_FULL_FRAG list */ mtr_start(&mtr); - mtr_x_lock(latch, &mtr); + mtr_x_space_lock(latch, &mtr); header = fsp_get_space_header(space, zip_size, &mtr); node_addr = flst_get_first(header + FSP_FULL_FRAG, &mtr); @@ -3843,7 +3833,7 @@ fsp_validate( while (!fil_addr_is_null(node_addr)) { mtr_start(&mtr); - mtr_x_lock(latch, &mtr); + mtr_x_space_lock(latch, &mtr); descr_count++; descr = xdes_lst_get_descriptor(space, zip_size, @@ -3858,7 +3848,7 @@ fsp_validate( /* Validate segments */ mtr_start(&mtr); - mtr_x_lock(latch, &mtr); + mtr_x_space_lock(latch, &mtr); header = fsp_get_space_header(space, zip_size, &mtr); @@ -3873,7 +3863,7 @@ fsp_validate( n = 0; do { mtr_start(&mtr); - mtr_x_lock(latch, &mtr); + mtr_x_space_lock(latch, &mtr); seg_inode_page = fut_get_ptr( space, zip_size, node_addr, RW_X_LATCH, &mtr) @@ -3902,7 +3892,7 @@ fsp_validate( } mtr_start(&mtr); - mtr_x_lock(latch, &mtr); + mtr_x_space_lock(latch, &mtr); header = fsp_get_space_header(space, zip_size, &mtr); @@ -3918,7 +3908,7 @@ fsp_validate( do { mtr_start(&mtr); - mtr_x_lock(latch, &mtr); + mtr_x_space_lock(latch, &mtr); seg_inode_page = fut_get_ptr( space, zip_size, node_addr, RW_X_LATCH, &mtr) @@ -4002,11 +3992,11 @@ fsp_print( mtr_start(&mtr2); - mtr_x_lock(latch, &mtr2); + mtr_x_space_lock(latch, &mtr2); mtr_start(&mtr); - mtr_x_lock(latch, &mtr); + mtr_x_space_lock(latch, &mtr); header = fsp_get_space_header(space, zip_size, &mtr); @@ -4038,7 +4028,7 @@ fsp_print( /* Print segments */ mtr_start(&mtr); - mtr_x_lock(latch, &mtr); + mtr_x_space_lock(latch, &mtr); header = fsp_get_space_header(space, zip_size, &mtr); @@ -4053,7 +4043,7 @@ fsp_print( do { mtr_start(&mtr); - mtr_x_lock(latch, &mtr); + mtr_x_space_lock(latch, &mtr); seg_inode_page = fut_get_ptr( space, zip_size, node_addr, RW_X_LATCH, &mtr) @@ -4075,7 +4065,7 @@ fsp_print( } mtr_start(&mtr); - mtr_x_lock(latch, &mtr); + mtr_x_space_lock(latch, &mtr); header = fsp_get_space_header(space, zip_size, &mtr); @@ -4090,7 +4080,7 @@ fsp_print( do { mtr_start(&mtr); - mtr_x_lock(latch, &mtr); + mtr_x_space_lock(latch, &mtr); seg_inode_page = fut_get_ptr( space, zip_size, node_addr, RW_X_LATCH, &mtr) @@ -4147,7 +4137,8 @@ fsp_page_is_free_func( ut_ad(mtr); - mtr_x_lock_func(fil_space_get_latch(space, &flags), file, line, mtr); + mtr_x_space_lock_func(fil_space_get_latch(space, &flags), file, line, + mtr); ulint zip_size = fsp_flags_get_zip_size(flags); xdes_t* descr = xdes_get_descriptor(space, zip_size, page_no, mtr); diff --git a/storage/xtradb/ibuf/ibuf0ibuf.cc b/storage/xtradb/ibuf/ibuf0ibuf.cc index cd65ea15729..6f11daa7363 100644 --- a/storage/xtradb/ibuf/ibuf0ibuf.cc +++ b/storage/xtradb/ibuf/ibuf0ibuf.cc @@ -588,7 +588,7 @@ ibuf_init_at_db_start(void) mutex_enter(&ibuf_mutex); - mtr_x_lock(fil_space_get_latch(IBUF_SPACE_ID, NULL), &mtr); + mtr_x_space_lock(fil_space_get_latch(IBUF_SPACE_ID, NULL), &mtr); header_page = ibuf_header_page_get(&mtr); @@ -2139,7 +2139,7 @@ ibuf_add_free_page(void) /* Acquire the fsp latch before the ibuf header, obeying the latching order */ - mtr_x_lock(fil_space_get_latch(IBUF_SPACE_ID, &flags), &mtr); + mtr_x_space_lock(fil_space_get_latch(IBUF_SPACE_ID, &flags), &mtr); zip_size = fsp_flags_get_zip_size(flags); header_page = ibuf_header_page_get(&mtr); @@ -2223,7 +2223,7 @@ ibuf_remove_free_page(void) /* Acquire the fsp latch before the ibuf header, obeying the latching order */ - mtr_x_lock(fil_space_get_latch(IBUF_SPACE_ID, &flags), &mtr); + mtr_x_space_lock(fil_space_get_latch(IBUF_SPACE_ID, &flags), &mtr); zip_size = fsp_flags_get_zip_size(flags); header_page = ibuf_header_page_get(&mtr); diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index a4274f4d8a6..6a823e131ac 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2019, MariaDB Corporation. +Copyright (c) 2013, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -300,6 +300,8 @@ struct fil_space_t { /*!< recovered tablespace size in pages; 0 if no size change was read from the redo log, or if the size change was implemented */ + /** the committed size of the tablespace in pages */ + ulint committed_size; ulint flags; /*!< FSP_SPACE_FLAGS and FSP_FLAGS_MEM_ flags; see fsp0fsp.h, fsp_flags_is_valid(), @@ -364,6 +366,15 @@ struct fil_space_t { { return stop_new_ops; } + + /** Clamp a page number for batched I/O, such as read-ahead. + @param offset page number limit + @return offset clamped to the tablespace size */ + ulint max_page_number_for_io(ulint offset) const + { + const ulint limit= committed_size; + return limit > offset ? offset : limit; + } }; /** Value of fil_space_t::magic_n */ @@ -574,16 +585,6 @@ ulint fil_space_get_zip_size( /*===================*/ ulint id); /*!< in: space id */ -/*******************************************************************//** -Checks if the pair space, page_no refers to an existing page in a tablespace -file space. The tablespace must be cached in the memory cache. -@return TRUE if the address is meaningful */ -UNIV_INTERN -ibool -fil_check_adress_in_tablespace( -/*===========================*/ - ulint id, /*!< in: space id */ - ulint page_no);/*!< in: page number */ /****************************************************************//** Initializes the tablespace memory cache. */ UNIV_INTERN diff --git a/storage/xtradb/include/mtr0mtr.h b/storage/xtradb/include/mtr0mtr.h index 2d3da4d2a25..7d115eed5fd 100644 --- a/storage/xtradb/include/mtr0mtr.h +++ b/storage/xtradb/include/mtr0mtr.h @@ -2,6 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. +Copyright (c) 2013, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -57,6 +58,7 @@ first 3 values must be RW_S_LATCH, RW_X_LATCH, RW_NO_LATCH */ #endif /* UNIV_DEBUG */ #define MTR_MEMO_S_LOCK 55 #define MTR_MEMO_X_LOCK 56 +#define MTR_MEMO_SPACE_X_LOCK 57 /** @name Log item types The log items are declared 'byte' so that the compiler can warn if val @@ -293,6 +295,8 @@ This macro locks an rw-lock in s-mode. */ This macro locks an rw-lock in x-mode. */ #define mtr_x_lock(B, MTR) mtr_x_lock_func((B), __FILE__, __LINE__,\ (MTR)) +#define mtr_x_space_lock(B, MTR) mtr_x_space_lock_func(B, __FILE__, __LINE__,\ + MTR) /*********************************************************************//** NOTE! Use the macro above! Locks a lock in s-mode. */ @@ -315,6 +319,14 @@ mtr_x_lock_func( const char* file, /*!< in: file name */ ulint line, /*!< in: line number */ mtr_t* mtr); /*!< in: mtr */ + +/** Acquire exclusive tablespace latch. +@param lock fil_space_t::latch +@param file source code file name of the caller +@param line source code line number of the caller +@param mtr mini-transaction */ +inline void mtr_x_space_lock_func(prio_rw_lock_t *lock, + const char *file, ulint line, mtr_t *mtr); #endif /* !UNIV_HOTBACKUP */ /***************************************************//** diff --git a/storage/xtradb/include/mtr0mtr.ic b/storage/xtradb/include/mtr0mtr.ic index a44728463ae..0aea0994267 100644 --- a/storage/xtradb/include/mtr0mtr.ic +++ b/storage/xtradb/include/mtr0mtr.ic @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -78,7 +79,7 @@ mtr_memo_push( ut_ad(object); ut_ad(type >= MTR_MEMO_PAGE_S_FIX); - ut_ad(type <= MTR_MEMO_X_LOCK); + ut_ad(type <= MTR_MEMO_SPACE_X_LOCK); ut_ad(mtr); ut_ad(mtr->magic_n == MTR_MAGIC_N); ut_ad(mtr->state == MTR_ACTIVE); @@ -293,4 +294,16 @@ mtr_x_lock_func( mtr_memo_push(mtr, lock, MTR_MEMO_X_LOCK); } + +/** Acquire exclusive tablespace latch. +@param lock fil_space_t::latch +@param file source code file name of the caller +@param line source code line number of the caller +@param mtr mini-transaction */ +inline void mtr_x_space_lock_func(prio_rw_lock_t *lock, + const char *file, ulint line, mtr_t *mtr) +{ + rw_lock_x_lock_inline(lock, 0, file, line); + mtr_memo_push(mtr, lock, MTR_MEMO_SPACE_X_LOCK); +} #endif /* !UNIV_HOTBACKUP */ diff --git a/storage/xtradb/mtr/mtr0mtr.cc b/storage/xtradb/mtr/mtr0mtr.cc index 77af6e9eb60..a04f9cc84a3 100644 --- a/storage/xtradb/mtr/mtr0mtr.cc +++ b/storage/xtradb/mtr/mtr0mtr.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -88,6 +89,16 @@ mtr_memo_slot_release_func( case MTR_MEMO_X_LOCK: rw_lock_x_unlock((prio_rw_lock_t*) object); break; + case MTR_MEMO_SPACE_X_LOCK: + { + fil_space_t* space = reinterpret_cast<fil_space_t*>( + static_cast<char*>(object) + - reinterpret_cast<char*>( + &static_cast<fil_space_t*>(0)->latch)); + space->committed_size = space->size; + rw_lock_x_unlock(&space->latch); + } + break; #ifdef UNIV_DEBUG default: ut_ad(slot->type == MTR_MEMO_MODIFY); diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index bafea49fae9..f3375f838ba 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -561,14 +561,6 @@ protected: /** Space id of the file being iterated over. */ ulint m_space; - /** Minimum page number for which the free list has not been - initialized: the pages >= this limit are, by definition, free; - note that in a single-table tablespace where size < 64 pages, - this number is 64, i.e., we have initialized the space about - the first extent, but have not physically allocted those pages - to the file. @see FSP_LIMIT. */ - ulint m_free_limit; - /** Current size of the space in pages */ ulint m_size; @@ -647,7 +639,6 @@ AbstractCallback::init( ut_a(m_space == ULINT_UNDEFINED); m_size = mach_read_from_4(page + FSP_SIZE); - m_free_limit = mach_read_from_4(page + FSP_FREE_LIMIT); m_space = mach_read_from_4(page + FSP_HEADER_OFFSET + FSP_SPACE_ID); return set_current_xdes(0, page); diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index 6ea430699d4..0ad755c1e3a 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -3,7 +3,7 @@ Copyright (c) 1996, 2017, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2019, MariaDB Corporation. +Copyright (c) 2013, 2020, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1240,6 +1240,10 @@ check_first_page: } } + if (fil_space_t* s = fil_space_get(0)) { + s->committed_size = s->size; + } + return(DB_SUCCESS); } @@ -1408,6 +1412,10 @@ srv_undo_tablespace_open( if (fil_node_create(name, (ulint) n_pages, space, FALSE)) { err = DB_SUCCESS; } + + if (fil_space_t* s = fil_space_get(space)) { + s->committed_size = n_pages; + } } return(err); diff --git a/storage/xtradb/trx/trx0rseg.cc b/storage/xtradb/trx/trx0rseg.cc index 1d0eb6d29fd..8ce6fb6729f 100644 --- a/storage/xtradb/trx/trx0rseg.cc +++ b/storage/xtradb/trx/trx0rseg.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2011, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -65,7 +65,7 @@ trx_rseg_header_create( ut_ad(mtr); ut_ad(mtr_memo_contains(mtr, fil_space_get_latch(space, NULL), - MTR_MEMO_X_LOCK)); + MTR_MEMO_SPACE_X_LOCK)); /* Allocate a new file segment for the rollback segment */ block = fseg_create(space, 0, TRX_RSEG + TRX_RSEG_FSEG_HEADER, mtr); @@ -312,7 +312,7 @@ trx_rseg_create(ulint space) /* To obey the latching order, acquire the file space x-latch before the trx_sys->mutex. */ - mtr_x_lock(fil_space_get_latch(space, NULL), &mtr); + mtr_x_space_lock(fil_space_get_latch(space, NULL), &mtr); slot_no = trx_sysf_rseg_find_free(&mtr); diff --git a/storage/xtradb/trx/trx0sys.cc b/storage/xtradb/trx/trx0sys.cc index fcdce291cf0..94a79a1f944 100644 --- a/storage/xtradb/trx/trx0sys.cc +++ b/storage/xtradb/trx/trx0sys.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2018, MariaDB Corporation. +Copyright (c) 2017, 2020, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -554,7 +554,7 @@ trx_sysf_create( then enter the kernel: we must do it in this order to conform to the latching order rules. */ - mtr_x_lock(fil_space_get_latch(TRX_SYS_SPACE, NULL), mtr); + mtr_x_space_lock(fil_space_get_latch(TRX_SYS_SPACE, NULL), mtr); /* Create the trx sys file block in a new allocated file segment */ block = fseg_create(TRX_SYS_SPACE, 0, TRX_SYS + TRX_SYS_FSEG_HEADER, |