diff options
Diffstat (limited to 'storage/innobase')
77 files changed, 1800 insertions, 1774 deletions
diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index 743a2f1fbfa..bd18fd2d45f 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -351,8 +351,8 @@ btr_root_adjust_on_import( } else { /* Check that the table flags and the tablespace flags match. */ - ulint tf = dict_tf_to_fsp_flags(table->flags); - ulint sf = table->space->flags; + uint32_t tf = dict_tf_to_fsp_flags(table->flags); + uint32_t sf = table->space->flags; sf &= ~FSP_FLAGS_MEM_MASK; tf &= ~FSP_FLAGS_MEM_MASK; if (fil_space_t::is_flags_equal(tf, sf) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 9d509637f44..ceac301a502 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -6893,7 +6893,6 @@ btr_store_big_rec_extern_fields( byte* field_ref; ulint extern_len; ulint store_len; - ulint space_id; ulint i; mtr_t mtr; mem_heap_t* heap = NULL; @@ -6916,7 +6915,6 @@ btr_store_big_rec_extern_fields( btr_blob_log_check_t redo_log(pcur, btr_mtr, offsets, &rec_block, &rec, op); page_zip = buf_block_get_page_zip(rec_block); - space_id = rec_block->page.id().space(); ut_a(fil_page_index_page_check(page_align(rec)) || op == BTR_STORE_INSERT_BULK); @@ -7034,6 +7032,7 @@ btr_store_big_rec_extern_fields( ut_a(block != NULL); + const uint32_t space_id = block->page.id().space(); const uint32_t page_no = block->page.id().page_no(); if (prev_page_no != FIL_NULL) { diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 228db3c4d4e..e15d7b7140a 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -330,9 +330,6 @@ buf_pool_t::chunk_t::map *buf_pool_t::chunk_t::map_reg; buf_pool_t::chunk_t::map *buf_pool_t::chunk_t::map_ref; #ifdef UNIV_DEBUG -/** Disable resizing buffer pool to make assertion code not expensive. */ -my_bool buf_disable_resize_buffer_pool_debug = TRUE; - /** This is used to insert validation operations in execution in the debug version */ static Atomic_counter<size_t> buf_dbg_counter; @@ -578,15 +575,12 @@ bool buf_is_zeroes(span<const byte> buf) } /** Check if a page is corrupt. -@param[in] check_lsn whether the LSN should be checked -@param[in] read_buf database page -@param[in] fsp_flags tablespace flags +@param check_lsn whether FIL_PAGE_LSN should be checked +@param read_buf database page +@param fsp_flags contents of FIL_SPACE_FLAGS @return whether the page is corrupted */ -bool -buf_page_is_corrupted( - bool check_lsn, - const byte* read_buf, - ulint fsp_flags) +bool buf_page_is_corrupted(bool check_lsn, const byte *read_buf, + uint32_t fsp_flags) { #ifndef UNIV_INNOCHECKSUM DBUG_EXECUTE_IF("buf_page_import_corrupt_failure", return(true); ); @@ -4257,10 +4251,10 @@ buf_print_io( /** Verify that post encryption checksum match with the calculated checksum. This function should be called only if tablespace contains crypt data metadata. -@param[in] page page frame -@param[in] fsp_flags tablespace flags -@return true if true if page is encrypted and OK, false otherwise */ -bool buf_page_verify_crypt_checksum(const byte* page, ulint fsp_flags) +@param page page frame +@param fsp_flags contents of FSP_SPACE_FLAGS +@return whether the page is encrypted and valid */ +bool buf_page_verify_crypt_checksum(const byte *page, uint32_t fsp_flags) { if (!fil_space_t::full_crc32(fsp_flags)) { return fil_space_verify_crypt_checksum( diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index d3c871ea69a..9c78c5c563e 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -352,7 +352,7 @@ void buf_dblwr_t::recover() if (recv_sys.parse_start_lsn > lsn) /* Pages written before the checkpoint are not useful for recovery. */ continue; - const ulint space_id= page_get_space_id(page); + const uint32_t space_id= page_get_space_id(page); const page_id_t page_id(space_id, page_no); if (recv_sys.scanned_lsn < lsn) diff --git a/storage/innobase/buf/buf0dump.cc b/storage/innobase/buf/buf0dump.cc index 96b046c93bc..bc39e6bfaec 100644 --- a/storage/innobase/buf/buf0dump.cc +++ b/storage/innobase/buf/buf0dump.cc @@ -633,7 +633,7 @@ buf_load() /* Avoid calling the expensive fil_space_t::get() for each page within the same tablespace. dump[] is sorted by (space, page), so all pages from a given tablespace are consecutive. */ - ulint cur_space_id = dump[0].space(); + uint32_t cur_space_id = dump[0].space(); fil_space_t* space = fil_space_t::get(cur_space_id); ulint zip_size = space ? space->zip_size() : 0; @@ -645,10 +645,9 @@ buf_load() for (i = 0; i < dump_n && !SHUTTING_DOWN(); i++) { /* space_id for this iteration of the loop */ - const ulint this_space_id = dump[i].space(); + const uint32_t this_space_id = dump[i].space(); - if (this_space_id == SRV_TMP_SPACE_ID) { - /* Ignore the innodb_temporary tablespace. */ + if (this_space_id >= SRV_SPACE_ID_UPPER_BOUND) { continue; } diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 80ce1d8f2ed..fdc73128bb7 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -40,11 +40,8 @@ Created 11/11/1995 Heikki Tuuri #include "log0crypt.h" #include "srv0mon.h" #include "fil0pagecompress.h" -#ifdef HAVE_LZO -# include "lzo/lzo1x.h" -#elif defined HAVE_SNAPPY -# include "snappy-c.h" -#endif +#include "lzo/lzo1x.h" +#include "snappy-c.h" /** Number of pages flushed via LRU. Protected by buf_pool.mutex. Also included in buf_flush_page_count. */ @@ -86,10 +83,6 @@ static struct ulint flush_pass; } page_cleaner; -#ifdef UNIV_DEBUG -my_bool innodb_page_cleaner_disabled_debug; -#endif /* UNIV_DEBUG */ - /* @} */ #ifdef UNIV_DEBUG @@ -219,7 +212,7 @@ deleting the data file of that tablespace. The pages still remain a part of LRU and are evicted from the list as they age towards the tail of the LRU. @param id tablespace identifier */ -void buf_flush_remove_pages(ulint id) +void buf_flush_remove_pages(uint32_t id) { const page_id_t first(id, 0), end(id + 1, 0); ut_ad(id); @@ -579,11 +572,10 @@ static void buf_tmp_reserve_compression_buf(buf_tmp_buffer_t* slot) /* Both Snappy and LZO compression methods require that the output buffer be bigger than input buffer. Adjust the allocated size. */ ulint size= srv_page_size; -#ifdef HAVE_LZO - size+= LZO1X_1_15_MEM_COMPRESS; -#elif defined HAVE_SNAPPY - size= snappy_max_compressed_length(size); -#endif + if (provider_service_lzo->is_loaded) + size+= LZO1X_1_15_MEM_COMPRESS; + else if (provider_service_snappy->is_loaded) + size= snappy_max_compressed_length(size); slot->comp_buf= static_cast<byte*>(aligned_malloc(size, srv_page_size)); } @@ -2331,12 +2323,6 @@ do_checkpoint: goto unemployed; } -#ifdef UNIV_DEBUG - while (innodb_page_cleaner_disabled_debug && !buf_flush_sync_lsn && - srv_shutdown_state == SRV_SHUTDOWN_NONE) - std::this_thread::sleep_for(std::chrono::milliseconds(100)); -#endif /* UNIV_DEBUG */ - #ifndef DBUG_OFF next: #endif /* !DBUG_OFF */ diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index aafd1c048e4..2dc779e81d0 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -505,7 +505,7 @@ void buf_read_page_background(fil_space_t *space, const page_id_t page_id, case DB_DECRYPTION_FAILED: ib::error() << "Background Page read failed to " - "read or decrypt " << page_id; + "read, uncompress, or decrypt " << page_id; break; default: ib::fatal() << "Error " << err << " in background read of " @@ -689,11 +689,9 @@ failed: } /** Issues read requests for pages which recovery wants to read in. -@param[in] space_id tablespace id -@param[in] page_nos array of page numbers to read, with the -highest page number the last in the array -@param[in] n number of page numbers in the array */ -void buf_read_recv_pages(ulint space_id, const uint32_t* page_nos, ulint n) +@param space_id tablespace identifier +@param page_nos page numbers to read, in ascending order */ +void buf_read_recv_pages(uint32_t space_id, st_::span<uint32_t> page_nos) { fil_space_t* space = fil_space_t::get(space_id); @@ -704,7 +702,7 @@ void buf_read_recv_pages(ulint space_id, const uint32_t* page_nos, ulint n) const ulint zip_size = space->zip_size(); - for (ulint i = 0; i < n; i++) { + for (ulint i = 0; i < page_nos.size(); i++) { /* Ignore if the page already present in freed ranges. */ if (space->freed_ranges.contains(page_nos[i])) { @@ -745,7 +743,7 @@ void buf_read_recv_pages(ulint space_id, const uint32_t* page_nos, ulint n) } - DBUG_PRINT("ib_buf", ("recovery read (%u pages) for %s", n, - space->chain.start->name)); + DBUG_PRINT("ib_buf", ("recovery read (%zu pages) for %s", + page_nos.size(), space->chain.start->name)); space->release(); } diff --git a/storage/innobase/bzip2.cmake b/storage/innobase/bzip2.cmake deleted file mode 100644 index 91dd2bf0fcd..00000000000 --- a/storage/innobase/bzip2.cmake +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (C) 2014, SkySQL Ab. All Rights Reserved. -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free Software -# Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA - -SET(WITH_INNODB_BZIP2 AUTO CACHE STRING - "Build with bzip2. Possible values are 'ON', 'OFF', 'AUTO' and default is 'AUTO'") - -MACRO (MYSQL_CHECK_BZIP2) - IF (WITH_INNODB_BZIP2 STREQUAL "ON" OR WITH_INNODB_BZIP2 STREQUAL "AUTO") - CHECK_INCLUDE_FILES(bzlib.h HAVE_BZLIB2_H) - CHECK_LIBRARY_EXISTS(bz2 BZ2_bzBuffToBuffCompress "" HAVE_BZLIB2_COMPRESS) - CHECK_LIBRARY_EXISTS(bz2 BZ2_bzBuffToBuffDecompress "" HAVE_BZLIB2_DECOMPRESS) - - IF (HAVE_BZLIB2_COMPRESS AND HAVE_BZLIB2_DECOMPRESS AND HAVE_BZLIB2_H) - SET(HAVE_INNODB_BZLIB2 TRUE) - ADD_DEFINITIONS(-DHAVE_BZIP2=1) - LINK_LIBRARIES(bz2) - ELSE() - IF (WITH_INNODB_BZIP2 STREQUAL "ON") - MESSAGE(FATAL_ERROR "Required bzip2 library is not found") - ENDIF() - ENDIF() - ENDIF() - ADD_FEATURE_INFO(INNODB_BZIP2 HAVE_INNODB_BZLIB2 - "BZIP2 compression in the InnoDB storage engine") -ENDMACRO() diff --git a/storage/innobase/dict/dict0boot.cc b/storage/innobase/dict/dict0boot.cc index 40c8df1498e..9e5031da430 100644 --- a/storage/innobase/dict/dict0boot.cc +++ b/storage/innobase/dict/dict0boot.cc @@ -44,7 +44,7 @@ dict_hdr_get_new_id( (not assigned if NULL) */ index_id_t* index_id, /*!< out: index id (not assigned if NULL) */ - ulint* space_id) /*!< out: space id + uint32_t* space_id) /*!< out: space id (not assigned if NULL) */ { ib_id_t id; @@ -245,8 +245,8 @@ dict_boot(void) header. */ dict_sys.recover_row_id(mach_read_from_8(dict_hdr + DICT_HDR_ROW_ID)); - if (ulint max_space_id = mach_read_from_4(dict_hdr - + DICT_HDR_MAX_SPACE_ID)) { + if (uint32_t max_space_id + = mach_read_from_4(dict_hdr + DICT_HDR_MAX_SPACE_ID)) { max_space_id--; fil_assign_new_space_id(&max_space_id); } diff --git a/storage/innobase/dict/dict0crea.cc b/storage/innobase/dict/dict0crea.cc index 0f5e1761940..f1bc2fcd980 100644 --- a/storage/innobase/dict/dict0crea.cc +++ b/storage/innobase/dict/dict0crea.cc @@ -348,8 +348,8 @@ dict_build_table_def_step( dict_table_t* table = node->table; ut_ad(!table->is_temporary()); ut_ad(!table->space); - ut_ad(table->space_id == ULINT_UNDEFINED); - dict_hdr_get_new_id(&table->id, NULL, NULL); + ut_ad(table->space_id == UINT32_MAX); + dict_hdr_get_new_id(&table->id, nullptr, nullptr); /* Always set this bit for all new created tables */ DICT_TF2_FLAG_SET(table, DICT_TF2_FTS_AUX_HEX_NAME); @@ -367,10 +367,10 @@ dict_build_table_def_step( DBUG_EXECUTE_IF( "ib_create_table_fail_out_of_space_ids", - table->space_id = ULINT_UNDEFINED; + table->space_id = UINT32_MAX; ); - if (table->space_id == ULINT_UNDEFINED) { + if (table->space_id == UINT32_MAX) { return DB_ERROR; } } else { @@ -1096,8 +1096,6 @@ dict_create_table_step( } if (node->state == TABLE_ADD_TO_CACHE) { - DBUG_EXECUTE_IF("ib_ddl_crash_during_create", DBUG_SUICIDE();); - node->table->can_be_evicted = true; node->table->add_to_cache(); diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 4253326d46a..bb820374266 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -1039,8 +1039,18 @@ dict_table_open_on_name( if (!(ignore_err & ~DICT_ERR_IGNORE_FK_NOKEY) && !table->is_readable() && table->corrupted) { - ib::error() << "Table " << table->name - << " is corrupted. Please drop the table and recreate."; + ulint algo = table->space->get_compression_algo(); + if (algo <= PAGE_ALGORITHM_LAST && !fil_comp_algo_loaded(algo)) { + my_printf_error(ER_PROVIDER_NOT_LOADED, + "Table %s is compressed with %s, which is not currently loaded. " + "Please load the %s provider plugin to open the table", + MYF(ME_ERROR_LOG), table->name, + page_compression_algorithms[algo], page_compression_algorithms[algo]); + } else { + my_printf_error(ER_TABLE_CORRUPT, + "Table %s is corrupted. Please drop the table and recreate.", + MYF(ME_ERROR_LOG), table->name); + } dict_sys.unfreeze(); DBUG_RETURN(nullptr); } diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index 5baf353280e..20ab06c8461 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -618,11 +618,10 @@ dict_sys_tables_type_valid(ulint type, bool not_redundant) @param[in] not_redundant whether ROW_FORMAT=REDUNDANT is not used @return table flags */ static -ulint -dict_sys_tables_type_to_tf(ulint type, bool not_redundant) +uint32_t dict_sys_tables_type_to_tf(uint32_t type, bool not_redundant) { ut_ad(dict_sys_tables_type_valid(type, not_redundant)); - ulint flags = not_redundant ? 1 : 0; + uint32_t flags = not_redundant ? 1 : 0; /* ZIP_SSIZE, ATOMIC_BLOBS, DATA_DIR, PAGE_COMPRESSION, PAGE_COMPRESSION_LEVEL are the same. */ @@ -653,14 +652,13 @@ dict_sys_tables_rec_read( const rec_t* rec, const span<const char>& name, table_id_t* table_id, - ulint* space_id, - ulint* n_cols, - ulint* flags, - ulint* flags2) + uint32_t* space_id, + uint32_t* n_cols, + uint32_t* flags, + uint32_t* flags2) { const byte* field; ulint len; - ulint type; field = rec_get_nth_field_old( rec, DICT_FLD__SYS_TABLES__ID, &len); @@ -676,7 +674,7 @@ dict_sys_tables_rec_read( field = rec_get_nth_field_old( rec, DICT_FLD__SYS_TABLES__TYPE, &len); ut_a(len == 4); - type = mach_read_from_4(field); + uint32_t type = mach_read_from_4(field); /* Handle MDEV-12873 InnoDB SYS_TABLES.TYPE incompatibility for PAGE_COMPRESSED=YES in MariaDB 10.2.2 to 10.2.6. @@ -767,8 +765,8 @@ dict_sys_tables_rec_read( if (!dict_sys_tables_type_valid(type, not_redundant)) { sql_print_error("InnoDB: Table %.*s in InnoDB" " data dictionary contains invalid flags." - " SYS_TABLES.TYPE=" ULINTPF - " SYS_TABLES.N_COLS=" ULINTPF, + " SYS_TABLES.TYPE=" UINT32PF + " SYS_TABLES.N_COLS=" UINT32PF, int(name.size()), name.data(), type, *n_cols); return(false); } @@ -792,8 +790,8 @@ dict_sys_tables_rec_read( sql_print_error("InnoDB: Table %.*s in InnoDB" " data dictionary" " contains invalid flags." - " SYS_TABLES.TYPE=" ULINTPF - " SYS_TABLES.MIX_LEN=" ULINTPF, + " SYS_TABLES.TYPE=" UINT32PF + " SYS_TABLES.MIX_LEN=" UINT32PF, int(name.size()), name.data(), type, *flags2); return(false); @@ -816,9 +814,9 @@ Search SYS_TABLES and check each tablespace mentioned that has not already been added to the fil_system. If it is valid, add it to the file_system list. @return the highest space ID found. */ -static ulint dict_check_sys_tables() +static uint32_t dict_check_sys_tables() { - ulint max_space_id = 0; + uint32_t max_space_id = 0; btr_pcur_t pcur; mtr_t mtr; @@ -833,10 +831,10 @@ static ulint dict_check_sys_tables() rec; rec = dict_getnext_system_low(&pcur, &mtr)) { ulint len; table_id_t table_id; - ulint space_id; - ulint n_cols; - ulint flags; - ulint flags2; + uint32_t space_id; + uint32_t n_cols; + uint32_t flags; + uint32_t flags2; /* If a table record is not useable, ignore it and continue on to the next record. Error messages were logged. */ @@ -934,10 +932,10 @@ void dict_check_tablespaces_and_store_max_id() /* Initialize the max space_id from sys header */ mtr.start(); - ulint max_space_id = mach_read_from_4(DICT_HDR_MAX_SPACE_ID - + DICT_HDR - + dict_hdr_get(&mtr) - ->page.frame); + uint32_t max_space_id = mach_read_from_4(DICT_HDR_MAX_SPACE_ID + + DICT_HDR + + dict_hdr_get(&mtr) + ->page.frame); mtr.commit(); fil_set_max_space_id_if_bigger(max_space_id); @@ -2043,12 +2041,8 @@ const char *dict_load_table_low(const span<const char> &name, const rec_t *rec, dict_table_t **table) { table_id_t table_id; - ulint space_id; - ulint n_cols; - ulint t_num; - ulint flags; - ulint flags2; - ulint n_v_col; + uint32_t space_id, t_num, flags, flags2; + ulint n_cols, n_v_col; if (const char* error_text = dict_sys_tables_rec_check(rec)) { *table = NULL; diff --git a/storage/innobase/dict/dict0mem.cc b/storage/innobase/dict/dict0mem.cc index 821dd8e5138..f4a42555f1d 100644 --- a/storage/innobase/dict/dict0mem.cc +++ b/storage/innobase/dict/dict0mem.cc @@ -162,7 +162,7 @@ dict_table_t *dict_table_t::create(const span<const char> &name, table->mdl_name.m_name= table->name.m_name; table->is_system_db= dict_mem_table_is_system(table->name.m_name); table->space= space; - table->space_id= space ? space->id : ULINT_UNDEFINED; + table->space_id= space ? space->id : UINT32_MAX; table->n_t_cols= static_cast<unsigned>(n_cols + DATA_N_SYS_COLS) & dict_index_t::MAX_N_FIELDS; table->n_v_cols= static_cast<unsigned>(n_v_cols) & diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index 570903d4327..fd744e6e2f5 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2009, 2019, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2021, MariaDB Corporation. +Copyright (c) 2015, 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -3262,6 +3262,11 @@ unlocked_free_and_exit: } } + ret= trx->bulk_insert_apply(); + if (ret != DB_SUCCESS) { + goto rollback_and_exit; + } + trx->commit(); goto free_and_exit; } @@ -3824,6 +3829,13 @@ dict_stats_update( return(DB_SUCCESS); } + if (trx_id_t bulk_trx_id = table->bulk_trx_id) { + if (trx_sys.find(nullptr, bulk_trx_id, false)) { + dict_stats_empty_table(table, false); + return DB_SUCCESS; + } + } + switch (stats_upd_option) { case DICT_STATS_RECALC_PERSISTENT: diff --git a/storage/innobase/dict/dict0stats_bg.cc b/storage/innobase/dict/dict0stats_bg.cc index 645f5818f57..e035c0688ee 100644 --- a/storage/innobase/dict/dict0stats_bg.cc +++ b/storage/innobase/dict/dict0stats_bg.cc @@ -46,11 +46,6 @@ Created Apr 25, 2012 Vasil Dimov #define MIN_RECALC_INTERVAL 10 /* seconds */ static void dict_stats_schedule(int ms); -#ifdef UNIV_DEBUG -/** Used by SET GLOBAL innodb_dict_stats_disabled_debug = 1; */ -my_bool innodb_dict_stats_disabled_debug; -#endif /* UNIV_DEBUG */ - /** Protects recalc_pool */ static mysql_mutex_t recalc_pool_mutex; @@ -376,21 +371,6 @@ done: return update_now; } -#ifdef UNIV_DEBUG -/** Disables dict stats thread. It's used by: - SET GLOBAL innodb_dict_stats_disabled_debug = 1 (0). -@param[in] save immediate result from check function */ -void dict_stats_disabled_debug_update(THD*, st_mysql_sys_var*, void*, - const void* save) -{ - const bool disable = *static_cast<const my_bool*>(save); - if (disable) - dict_stats_shutdown(); - else - dict_stats_start(); -} -#endif /* UNIV_DEBUG */ - static tpool::timer* dict_stats_timer; static std::mutex dict_stats_mutex; diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 69f18de9428..5a792a59664 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -767,20 +767,20 @@ static dberr_t fil_space_decrypt_for_non_full_checksum( /** Decrypt a page. @param[in] space_id tablespace id +@param[in] fsp_flags Tablespace flags @param[in] crypt_data crypt_data @param[in] tmp_frame Temporary buffer @param[in] physical_size page size -@param[in] fsp_flags Tablespace flags @param[in,out] src_frame Page to decrypt @param[out] err DB_SUCCESS or DB_DECRYPTION_FAILED @return DB_SUCCESS or error */ dberr_t fil_space_decrypt( - ulint space_id, + uint32_t space_id, + uint32_t fsp_flags, fil_space_crypt_t* crypt_data, byte* tmp_frame, ulint physical_size, - ulint fsp_flags, byte* src_frame) { if (fil_space_t::full_crc32(fsp_flags)) { @@ -811,9 +811,10 @@ fil_space_decrypt( ut_ad(space->crypt_data != NULL && space->crypt_data->is_encrypted()); ut_ad(space->referenced()); - if (DB_SUCCESS != fil_space_decrypt(space->id, space->crypt_data, + if (DB_SUCCESS != fil_space_decrypt(space->id, space->flags, + space->crypt_data, tmp_frame, physical_size, - space->flags, src_frame)) { + src_frame)) { return nullptr; } @@ -1403,7 +1404,7 @@ inline fil_space_t *fil_system_t::default_encrypt_next(fil_space_t *space, mysql_mutex_assert_owner(&mutex); auto it= space && space->is_in_default_encrypt - ? sized_ilist<fil_space_t, rotation_list_tag_t>::iterator(space) + ? sized_ilist<fil_space_t, default_encrypt_tag_t>::iterator(space) : default_encrypt_tables.begin(); const auto end= default_encrypt_tables.end(); @@ -1896,7 +1897,7 @@ fil_crypt_rotate_pages( const key_state_t* key_state, rotate_thread_t* state) { - ulint space_id = state->space->id; + const uint32_t space_id = state->space->id; uint32_t end = std::min(state->offset + uint32_t(state->batch), state->space->free_limit); diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 764ee73f814..37ac293c259 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -54,14 +54,11 @@ Created 10/25/1995 Heikki Tuuri # include <dirent.h> #endif -/** Determine if the space id is a user tablespace id or not. -@param space_id tablespace identifier -@return true if it is a user tablespace ID */ -inline bool fil_is_user_tablespace_id(ulint space_id) -{ - return space_id != TRX_SYS_SPACE && space_id != SRV_TMP_SPACE_ID && - !srv_is_undo_tablespace(space_id); -} +#include "lz4.h" +#include "lzo/lzo1x.h" +#include "lzma.h" +#include "bzlib.h" +#include "snappy-c.h" /** Try to close a file to adhere to the innodb_open_files limit. @param print_info whether to diagnose why a file cannot be closed @@ -77,7 +74,7 @@ bool fil_space_t::try_to_close(bool print_info) case FIL_TYPE_IMPORT: break; case FIL_TYPE_TABLESPACE: - if (!fil_is_user_tablespace_id(space.id)) + if (is_predefined_tablespace(space.id)) continue; } @@ -127,11 +124,8 @@ The tablespace must exist in the memory cache. @param[in] new_path_in new file name, or NULL if it is located in the normal data directory @return true if success */ -static bool -fil_rename_tablespace( - ulint id, - const char* old_path, - const char* new_path_in); +static bool fil_rename_tablespace(uint32_t id, const char *old_path, + const char *new_path_in); /* IMPLEMENTATION OF THE TABLESPACE MEMORY CACHE @@ -219,14 +213,11 @@ fil_validate_skip(void) } #endif /* UNIV_DEBUG */ -/*******************************************************************//** -Returns the table space by a given id, NULL if not found. -It is unsafe to dereference the returned pointer. It is fine to check -for NULL. */ -fil_space_t* -fil_space_get_by_id( -/*================*/ - ulint id) /*!< in: space id */ +/** Look up a tablespace. +@param tablespace identifier +@return tablespace +@retval nullptr if not found */ +fil_space_t *fil_space_get_by_id(uint32_t id) { fil_space_t* space; @@ -249,48 +240,38 @@ or the caller should be in single-threaded crash recovery mode Normally, fil_space_t::get() should be used instead. @param[in] id tablespace ID @return tablespace, or NULL if not found */ -fil_space_t* -fil_space_get( - ulint id) +fil_space_t *fil_space_get(uint32_t id) { - mysql_mutex_lock(&fil_system.mutex); - fil_space_t* space = fil_space_get_by_id(id); - mysql_mutex_unlock(&fil_system.mutex); - return(space); + mysql_mutex_lock(&fil_system.mutex); + fil_space_t *space= fil_space_get_by_id(id); + mysql_mutex_unlock(&fil_system.mutex); + return space; } -/** Validate the compression algorithm for full crc32 format. -@param[in] space tablespace object -@return whether the compression algorithm support */ -static bool fil_comp_algo_validate(const fil_space_t* space) +/** Check if the compression algorithm is loaded +@param[in] comp_algo ulint compression algorithm +@return whether the compression algorithm is loaded */ +bool fil_comp_algo_loaded(ulint comp_algo) { - if (!space->full_crc32()) { - return true; - } - - DBUG_EXECUTE_IF("fil_comp_algo_validate_fail", - return false;); - - ulint comp_algo = space->get_compression_algo(); switch (comp_algo) { case PAGE_UNCOMPRESSED: case PAGE_ZLIB_ALGORITHM: -#ifdef HAVE_LZ4 + return true; + case PAGE_LZ4_ALGORITHM: -#endif /* HAVE_LZ4 */ -#ifdef HAVE_LZO + return provider_service_lz4->is_loaded; + case PAGE_LZO_ALGORITHM: -#endif /* HAVE_LZO */ -#ifdef HAVE_LZMA + return provider_service_lzo->is_loaded; + case PAGE_LZMA_ALGORITHM: -#endif /* HAVE_LZMA */ -#ifdef HAVE_BZIP2 + return provider_service_lzma->is_loaded; + case PAGE_BZIP2_ALGORITHM: -#endif /* HAVE_BZIP2 */ -#ifdef HAVE_SNAPPY + return provider_service_bzip2->is_loaded; + case PAGE_SNAPPY_ALGORITHM: -#endif /* HAVE_SNAPPY */ - return true; + return provider_service_snappy->is_loaded; } return false; @@ -389,9 +370,26 @@ static bool fil_node_open_file_low(fil_node_t *node) return false; } + ulint comp_algo = node->space->get_compression_algo(); + bool comp_algo_invalid = false; + if (node->size); - else if (!node->read_page0() || !fil_comp_algo_validate(node->space)) + else if (!node->read_page0() || + // validate compression algorithm for full crc32 format + (node->space->full_crc32() && + (comp_algo_invalid = !fil_comp_algo_loaded(comp_algo)))) { + if (comp_algo_invalid) + { + if (comp_algo <= PAGE_ALGORITHM_LAST) + ib::warn() << "'" << node->name << "' is compressed with " + << page_compression_algorithms[comp_algo] + << ", which is not currently loaded"; + else + ib::warn() << "'" << node->name << "' is compressed with " + << "invalid algorithm: " << comp_algo; + } + os_file_close(node->handle); node->handle= OS_FILE_CLOSED; return false; @@ -418,7 +416,7 @@ static bool fil_node_open_file(fil_node_t *node) { mysql_mutex_assert_owner(&fil_system.mutex); ut_ad(!node->is_open()); - ut_ad(fil_is_user_tablespace_id(node->space->id) || + ut_ad(!is_predefined_tablespace(node->space->id) || srv_operation == SRV_OPERATION_BACKUP || srv_operation == SRV_OPERATION_RESTORE || srv_operation == SRV_OPERATION_RESTORE_DELTA); @@ -861,13 +859,10 @@ fil_space_free_low( /** Frees a space object from the tablespace memory cache. Closes the files in the chain but does not delete them. There must not be any pending i/o's or flushes on the files. -@param[in] id tablespace identifier -@param[in] x_latched whether the caller holds X-mode space->latch +@param id tablespace identifier +@param x_latched whether the caller holds exclusive fil_space_t::latch @return true if success */ -bool -fil_space_free( - ulint id, - bool x_latched) +bool fil_space_free(uint32_t id, bool x_latched) { ut_ad(id != TRX_SYS_SPACE); @@ -915,7 +910,7 @@ fil_space_free( @param mode encryption mode @return pointer to created tablespace, to be filled in with add() @retval nullptr on failure (such as when the same tablespace exists) */ -fil_space_t *fil_space_t::create(ulint id, ulint flags, +fil_space_t *fil_space_t::create(uint32_t id, uint32_t flags, fil_type_t purpose, fil_space_crypt_t *crypt_data, fil_encryption_t mode) @@ -1021,18 +1016,13 @@ Assigns a new space id for a new single-table tablespace. This works simply by incrementing the global counter. If 4 billion id's is not enough, we may need to recycle id's. @return true if assigned, false if not */ -bool -fil_assign_new_space_id( -/*====================*/ - ulint* space_id) /*!< in/out: space id */ +bool fil_assign_new_space_id(uint32_t *space_id) { - ulint id; + uint32_t id = *space_id; bool success; mysql_mutex_lock(&fil_system.mutex); - id = *space_id; - if (id < fil_system.max_assigned_id) { id = fil_system.max_assigned_id; } @@ -1058,7 +1048,7 @@ fil_assign_new_space_id( << ". To reset the counter to zero" " you have to dump all your tables and" " recreate the whole InnoDB installation."; - *space_id = ULINT_UNDEFINED; + *space_id = UINT32_MAX; } mysql_mutex_unlock(&fil_system.mutex); @@ -1091,7 +1081,7 @@ bool fil_space_t::read_page0() } /** Look up a tablespace and ensure that its first page has been validated. */ -static fil_space_t *fil_space_get_space(ulint id) +static fil_space_t *fil_space_get_space(uint32_t id) { if (fil_space_t *space= fil_space_get_by_id(id)) if (space->read_page0()) @@ -1099,7 +1089,8 @@ static fil_space_t *fil_space_get_space(ulint id) return nullptr; } -void fil_space_set_recv_size_and_flags(ulint id, uint32_t size, uint32_t flags) +void fil_space_set_recv_size_and_flags(uint32_t id, uint32_t size, + uint32_t flags) { ut_ad(id < SRV_SPACE_ID_UPPER_BOUND); mysql_mutex_lock(&fil_system.mutex); @@ -1377,14 +1368,9 @@ void fil_space_t::close_all() /*******************************************************************//** Sets the max tablespace id counter if the given number is bigger than the previous value. */ -void -fil_set_max_space_id_if_bigger( -/*===========================*/ - ulint max_id) /*!< in: maximum known id */ +void fil_set_max_space_id_if_bigger(uint32_t max_id) { - if (max_id >= SRV_SPACE_ID_UPPER_BOUND) { - ib::fatal() << "Max tablespace id is too high, " << max_id; - } + ut_a(max_id < SRV_SPACE_ID_UPPER_BOUND); mysql_mutex_lock(&fil_system.mutex); @@ -1420,7 +1406,7 @@ fil_write_flushed_lsn( mach_write_to_8(buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, lsn); - ulint fsp_flags = mach_read_from_4( + uint32_t fsp_flags = mach_read_from_4( buf + FSP_HEADER_OFFSET + FSP_SPACE_FLAGS); if (fil_space_t::full_crc32(fsp_flags)) { @@ -1442,7 +1428,7 @@ fil_write_flushed_lsn( @param id tablespace identifier @return tablespace @retval nullptr if the tablespace is missing or inaccessible */ -fil_space_t *fil_space_t::get(ulint id) +fil_space_t *fil_space_t::get(uint32_t id) { mysql_mutex_lock(&fil_system.mutex); fil_space_t *space= fil_space_get_by_id(id); @@ -1462,7 +1448,7 @@ fil_space_t *fil_space_t::get(ulint id) @param first_page_no first page number in the file @param path file path @param new_path new file path for type=FILE_RENAME */ -inline void mtr_t::log_file_op(mfile_type_t type, ulint space_id, +inline void mtr_t::log_file_op(mfile_type_t type, uint32_t space_id, const char *path, const char *new_path) { ut_ad((new_path != nullptr) == (type == FILE_RENAME)); @@ -1521,13 +1507,8 @@ inline void mtr_t::log_file_op(mfile_type_t type, ulint space_id, @param[in] old_name tablespace file name @param[in] new_name tablespace file name after renaming @param[in,out] mtr mini-transaction */ -static -void -fil_name_write_rename_low( - ulint space_id, - const char* old_name, - const char* new_name, - mtr_t* mtr) +static void fil_name_write_rename_low(uint32_t space_id, const char *old_name, + const char *new_name, mtr_t *mtr) { ut_ad(!is_predefined_tablespace(space_id)); mtr->log_file_op(FILE_RENAME, space_id, old_name, new_name); @@ -1537,35 +1518,28 @@ fil_name_write_rename_low( @param[in] space_id tablespace id @param[in] old_name tablespace file name @param[in] new_name tablespace file name after renaming */ -static void -fil_name_write_rename( - ulint space_id, - const char* old_name, - const char* new_name) +static void fil_name_write_rename(uint32_t space_id, + const char *old_name, const char* new_name) { - mtr_t mtr; - mtr.start(); - fil_name_write_rename_low(space_id, old_name, new_name, &mtr); - mtr.commit(); - log_write_up_to(mtr.commit_lsn(), true); + mtr_t mtr; + mtr.start(); + fil_name_write_rename_low(space_id, old_name, new_name, &mtr); + mtr.commit(); + log_write_up_to(mtr.commit_lsn(), true); } /** Write FILE_MODIFY for a file. @param[in] space_id tablespace id @param[in] name tablespace file name @param[in,out] mtr mini-transaction */ -static -void -fil_name_write( - ulint space_id, - const char* name, - mtr_t* mtr) +static void fil_name_write(uint32_t space_id, const char *name, + mtr_t *mtr) { ut_ad(!is_predefined_tablespace(space_id)); mtr->log_file_op(FILE_MODIFY, space_id, name); } -fil_space_t *fil_space_t::check_pending_operations(ulint id) +fil_space_t *fil_space_t::check_pending_operations(uint32_t id) { ut_a(!is_system_tablespace(id)); mysql_mutex_lock(&fil_system.mutex); @@ -1592,7 +1566,7 @@ being_deleted: return nullptr; /* Issue a warning every 10.24 seconds, starting after 2.56 seconds */ if ((count & 511) == 128) - sql_print_warning("InnoDB: Waiting for tablespace " ULINTPF + sql_print_warning("InnoDB: Waiting for tablespace " UINT32PF " to be deleted", id); std::this_thread::sleep_for(std::chrono::milliseconds(20)); mysql_mutex_lock(&fil_system.mutex); @@ -1631,7 +1605,7 @@ being_deleted: /** Close a single-table tablespace on failed IMPORT TABLESPACE. The tablespace must be cached in the memory cache. Free all pages used by the tablespace. */ -void fil_close_tablespace(ulint id) +void fil_close_tablespace(uint32_t id) { ut_ad(!is_system_tablespace(id)); fil_space_t* space = fil_space_t::check_pending_operations(id); @@ -1671,7 +1645,7 @@ void fil_close_tablespace(ulint id) @param id tablespace identifier @return detached file handle (to be closed by the caller) @return OS_FILE_CLOSED if no file existed */ -pfs_os_file_t fil_delete_tablespace(ulint id) +pfs_os_file_t fil_delete_tablespace(uint32_t id) { ut_ad(!is_system_tablespace(id)); pfs_os_file_t handle= OS_FILE_CLOSED; @@ -1880,11 +1854,8 @@ The tablespace must exist in the memory cache. @param[in] new_path_in new file name, or NULL if it is located in the normal data directory @return true if success */ -static bool -fil_rename_tablespace( - ulint id, - const char* old_path, - const char* new_path_in) +static bool fil_rename_tablespace(uint32_t id, const char *old_path, + const char *new_path_in) { fil_space_t* space; fil_node_t* node; @@ -1969,10 +1940,10 @@ must be >= FIL_IBD_FILE_INITIAL_SIZE @retval NULL on error */ fil_space_t* fil_ibd_create( - ulint space_id, + uint32_t space_id, const table_name_t name, const char* path, - ulint flags, + uint32_t flags, uint32_t size, fil_encryption_t mode, uint32_t key_id, @@ -2146,8 +2117,8 @@ fil_space_t* fil_ibd_open( bool validate, fil_type_t purpose, - ulint id, - ulint flags, + uint32_t id, + uint32_t flags, fil_space_t::name_type name, const char* path_in, dberr_t* err) @@ -2167,7 +2138,7 @@ fil_ibd_open( /* Table flags can be ULINT_UNDEFINED if dict_tf_to_fsp_flags_failure is set. */ - if (flags == ULINT_UNDEFINED) { + if (flags == UINT32_MAX) { corrupted: local_err = DB_CORRUPTION; func_exit: @@ -2511,10 +2482,7 @@ of the file in validate_for_recovery(). @param[out] space the tablespace, or NULL on error @return status of the operation */ enum fil_load_status -fil_ibd_load( - ulint space_id, - const char* filename, - fil_space_t*& space) +fil_ibd_load(uint32_t space_id, const char *filename, fil_space_t *&space) { /* If the a space is already in the file system cache with this space ID, then there is nothing to do. */ @@ -2633,7 +2601,7 @@ tablespace_check: /* Adjust the memory-based flags that would normally be set by dict_tf_to_fsp_flags(). In recovery, we have no data dictionary. */ - ulint flags = file.flags(); + uint32_t flags = file.flags(); if (fil_space_t::is_compressed(flags)) { flags |= page_zip_level << FSP_FLAGS_MEM_COMPRESSION_LEVEL; @@ -2667,7 +2635,7 @@ tablespace_check: (Typically when upgrading from MariaDB 10.1.0..10.1.20.) @param[in,out] space tablespace @param[in] flags desired tablespace flags */ -void fsp_flags_try_adjust(fil_space_t* space, ulint flags) +void fsp_flags_try_adjust(fil_space_t *space, uint32_t flags) { ut_ad(!srv_read_only_mode); ut_ad(fil_space_t::is_valid_flags(flags, space->id)); @@ -2717,14 +2685,15 @@ startup, there may be many tablespaces which are not yet in the memory cache. @param[in] table_flags table flags @return the tablespace @retval NULL if no matching tablespace exists in the memory cache */ -fil_space_t *fil_space_for_table_exists_in_mem(ulint id, ulint table_flags) +fil_space_t *fil_space_for_table_exists_in_mem(uint32_t id, + uint32_t table_flags) { - const ulint expected_flags = dict_tf_to_fsp_flags(table_flags); + const uint32_t expected_flags = dict_tf_to_fsp_flags(table_flags); mysql_mutex_lock(&fil_system.mutex); if (fil_space_t* space = fil_space_get_by_id(id)) { - ulint tf = expected_flags & ~FSP_FLAGS_MEM_MASK; - ulint sf = space->flags & ~FSP_FLAGS_MEM_MASK; + uint32_t tf = expected_flags & ~FSP_FLAGS_MEM_MASK; + uint32_t sf = space->flags & ~FSP_FLAGS_MEM_MASK; if (!fil_space_t::is_flags_equal(tf, sf) && !fil_space_t::is_flags_equal(sf, tf)) { @@ -3161,12 +3130,6 @@ fil_names_clear( bool do_write) { mtr_t mtr; - ulint mtr_checkpoint_size = RECV_SCAN_SIZE - 1; - - DBUG_EXECUTE_IF( - "increase_mtr_checkpoint_size", - mtr_checkpoint_size = 75 * 1024; - ); mysql_mutex_assert_owner(&log_sys.mutex); ut_ad(lsn); @@ -3175,9 +3138,8 @@ fil_names_clear( for (auto it = fil_system.named_spaces.begin(); it != fil_system.named_spaces.end(); ) { - if (mtr.get_log()->size() - + (3 + 5 + 1) + strlen(it->chain.start->name) - >= mtr_checkpoint_size) { + if (mtr.get_log()->size() + strlen(it->chain.start->name) + >= RECV_SCAN_SIZE - (3 + 5)) { /* Prevent log parse buffer overflow */ mtr.commit_files(); mtr.start(); diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index 9961bdf056c..b6971558201 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -55,21 +55,11 @@ Updated 14/02/2015 #include <fcntl.h> #endif #include "row0mysql.h" -#ifdef HAVE_LZ4 #include "lz4.h" -#endif -#ifdef HAVE_LZO #include "lzo/lzo1x.h" -#endif -#ifdef HAVE_LZMA #include "lzma.h" -#endif -#ifdef HAVE_BZIP2 #include "bzlib.h" -#endif -#ifdef HAVE_SNAPPY #include "snappy-c.h" -#endif /** Compress a page for the given compression algorithm. @param[in] buf page to be compressed @@ -94,6 +84,7 @@ static ulint fil_page_compress_low( /* fall through */ case PAGE_UNCOMPRESSED: return 0; + case PAGE_ZLIB_ALGORITHM: { ulong len = uLong(write_size); @@ -104,23 +95,15 @@ static ulint fil_page_compress_low( } } break; -#ifdef HAVE_LZ4 + case PAGE_LZ4_ALGORITHM: -# ifdef HAVE_LZ4_COMPRESS_DEFAULT write_size = LZ4_compress_default( reinterpret_cast<const char*>(buf), reinterpret_cast<char*>(out_buf) + header_len, int(srv_page_size), int(write_size)); -# else - write_size = LZ4_compress_limitedOutput( - reinterpret_cast<const char*>(buf), - reinterpret_cast<char*>(out_buf) + header_len, - int(srv_page_size), int(write_size)); -# endif return write_size; -#endif /* HAVE_LZ4 */ -#ifdef HAVE_LZO + case PAGE_LZO_ALGORITHM: { lzo_uint len = write_size; @@ -133,8 +116,7 @@ static ulint fil_page_compress_low( } break; } -#endif /* HAVE_LZO */ -#ifdef HAVE_LZMA + case PAGE_LZMA_ALGORITHM: { size_t out_pos = 0; @@ -147,9 +129,7 @@ static ulint fil_page_compress_low( } break; } -#endif /* HAVE_LZMA */ -#ifdef HAVE_BZIP2 case PAGE_BZIP2_ALGORITHM: { unsigned len = unsigned(write_size); if (BZ_OK == BZ2_bzBuffToBuffCompress( @@ -163,9 +143,7 @@ static ulint fil_page_compress_low( } break; } -#endif /* HAVE_BZIP2 */ -#ifdef HAVE_SNAPPY case PAGE_SNAPPY_ALGORITHM: { size_t len = snappy_max_compressed_length(srv_page_size); @@ -179,7 +157,6 @@ static ulint fil_page_compress_low( } break; } -#endif /* HAVE_SNAPPY */ } return 0; @@ -195,7 +172,7 @@ static ulint fil_page_compress_low( static ulint fil_page_compress_for_full_crc32( const byte* buf, byte* out_buf, - ulint flags, + uint32_t flags, ulint block_size, bool encrypted) { @@ -378,7 +355,7 @@ static ulint fil_page_compress_for_non_full_crc32( ulint fil_page_compress( const byte* buf, byte* out_buf, - ulint flags, + uint32_t flags, ulint block_size, bool encrypted) { @@ -432,7 +409,7 @@ static bool fil_page_decompress_low( uLong(actual_size)) && len == srv_page_size); } -#ifdef HAVE_LZ4 + case PAGE_LZ4_ALGORITHM: return LZ4_decompress_safe( reinterpret_cast<const char*>(buf) + header_len, @@ -440,8 +417,7 @@ static bool fil_page_decompress_low( static_cast<int>(actual_size), static_cast<int>(srv_page_size)) == static_cast<int>(srv_page_size); -#endif /* HAVE_LZ4 */ -#ifdef HAVE_LZO + case PAGE_LZO_ALGORITHM: { lzo_uint len_lzo = srv_page_size; @@ -450,8 +426,7 @@ static bool fil_page_decompress_low( actual_size, tmp_buf, &len_lzo, NULL) && len_lzo == srv_page_size); } -#endif /* HAVE_LZO */ -#ifdef HAVE_LZMA + case PAGE_LZMA_ALGORITHM: { size_t src_pos = 0; @@ -464,8 +439,7 @@ static bool fil_page_decompress_low( srv_page_size) && dst_pos == srv_page_size; } -#endif /* HAVE_LZMA */ -#ifdef HAVE_BZIP2 + case PAGE_BZIP2_ALGORITHM: { uint dst_pos = static_cast<uint>(srv_page_size); @@ -476,8 +450,7 @@ static bool fil_page_decompress_low( static_cast<uint>(actual_size), 1, 0) && dst_pos == srv_page_size; } -#endif /* HAVE_BZIP2 */ -#ifdef HAVE_SNAPPY + case PAGE_SNAPPY_ALGORITHM: { size_t olen = srv_page_size; @@ -489,7 +462,6 @@ static bool fil_page_decompress_low( reinterpret_cast<char*>(tmp_buf), &olen) && olen == srv_page_size; } -#endif /* HAVE_SNAPPY */ } return false; @@ -502,7 +474,8 @@ static bool fil_page_decompress_low( @return size of the compressed data @retval 0 if decompression failed @retval srv_page_size if the page was not compressed */ -ulint fil_page_decompress_for_full_crc32(byte* tmp_buf, byte* buf, ulint flags) +static size_t fil_page_decompress_for_full_crc32(byte *tmp_buf, byte *buf, + uint32_t flags) { ut_ad(fil_space_t::full_crc32(flags)); bool compressed = false; @@ -547,9 +520,7 @@ ulint fil_page_decompress_for_full_crc32(byte* tmp_buf, byte* buf, ulint flags) @return size of the compressed data @retval 0 if decompression failed @retval srv_page_size if the page was not compressed */ -ulint fil_page_decompress_for_non_full_crc32( - byte* tmp_buf, - byte* buf) +static size_t fil_page_decompress_for_non_full_crc32(byte *tmp_buf, byte *buf) { ulint header_len; uint comp_algo; @@ -600,10 +571,7 @@ ulint fil_page_decompress_for_non_full_crc32( @return size of the compressed data @retval 0 if decompression failed @retval srv_page_size if the page was not compressed */ -ulint fil_page_decompress( - byte* tmp_buf, - byte* buf, - ulint flags) +ulint fil_page_decompress(byte *tmp_buf, byte *buf, uint32_t flags) { if (fil_space_t::full_crc32(flags)) { return fil_page_decompress_for_full_crc32(tmp_buf, buf, flags); diff --git a/storage/innobase/fsp/fsp0file.cc b/storage/innobase/fsp/fsp0file.cc index d775fd9c657..e85674401ee 100644 --- a/storage/innobase/fsp/fsp0file.cc +++ b/storage/innobase/fsp/fsp0file.cc @@ -304,8 +304,8 @@ Datafile::read_first_page(bool read_only_mode) + m_first_page); m_flags = fsp_header_get_flags(m_first_page); if (!fil_space_t::is_valid_flags(m_flags, m_space_id)) { - ulint cflags = fsp_flags_convert_from_101(m_flags); - if (cflags == ULINT_UNDEFINED) { + uint32_t cflags = fsp_flags_convert_from_101(m_flags); + if (cflags == UINT32_MAX) { ib::error() << "Invalid flags " << ib::hex(m_flags) << " in " << m_filepath; @@ -342,8 +342,7 @@ in order for this function to validate it. @param[in] flags The expected tablespace flags. @retval DB_SUCCESS if tablespace is valid, DB_ERROR if not. m_is_valid is also set true on success, else false. */ -dberr_t -Datafile::validate_to_dd(ulint space_id, ulint flags) +dberr_t Datafile::validate_to_dd(uint32_t space_id, uint32_t flags) { dberr_t err; @@ -433,7 +432,7 @@ Datafile::validate_for_recovery() } } - if (m_space_id == ULINT_UNDEFINED) { + if (m_space_id == UINT32_MAX) { return DB_SUCCESS; /* empty file */ } @@ -626,15 +625,15 @@ Datafile::find_space_id() page_size <<= 1) { /* map[space_id] = count of pages */ typedef std::map< - ulint, - ulint, - std::less<ulint>, - ut_allocator<std::pair<const ulint, ulint> > > + uint32_t, + uint32_t, + std::less<uint32_t>, + ut_allocator<std::pair<const uint32_t, uint32_t> > > Pages; Pages verify; - ulint page_count = 64; - ulint valid_pages = 0; + uint32_t page_count = 64; + uint32_t valid_pages = 0; /* Adjust the number of pages to analyze based on file size */ while ((page_count * page_size) > file_size) { @@ -648,14 +647,14 @@ Datafile::find_space_id() byte* page = static_cast<byte*>( aligned_malloc(page_size, page_size)); - ulint fsp_flags; + uint32_t fsp_flags; /* provide dummy value if the first os_file_read() fails */ switch (srv_checksum_algorithm) { case SRV_CHECKSUM_ALGORITHM_STRICT_FULL_CRC32: case SRV_CHECKSUM_ALGORITHM_FULL_CRC32: fsp_flags = 1U << FSP_FLAGS_FCRC32_POS_MARKER | FSP_FLAGS_FCRC32_PAGE_SSIZE() - | innodb_compression_algorithm + | uint(innodb_compression_algorithm) << FSP_FLAGS_FCRC32_POS_COMPRESSED_ALGO; break; default: @@ -695,7 +694,7 @@ Datafile::find_space_id() if (noncompressed_ok || compressed_ok) { - ulint space_id = mach_read_from_4(page + uint32_t space_id = mach_read_from_4(page + FIL_PAGE_SPACE_ID); if (space_id > 0) { @@ -773,14 +772,14 @@ Datafile::restore_from_doublewrite() return(true); } - ulint flags = mach_read_from_4( + uint32_t flags = mach_read_from_4( FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); if (!fil_space_t::is_valid_flags(flags, m_space_id)) { flags = fsp_flags_convert_from_101(flags); /* recv_dblwr_t::validate_page() inside find_page() checked this already. */ - ut_ad(flags != ULINT_UNDEFINED); + ut_ad(flags != UINT32_MAX); /* The flags on the page should be converted later. */ } diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index d563e0868f1..60e62a4f922 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -1104,7 +1104,7 @@ fsp_alloc_free_page( { fil_addr_t first; xdes_t* descr; - const ulint space_id = space->id; + const uint32_t space_id = space->id; ut_d(space->modify_check(*mtr)); buf_block_t* block = fsp_get_header(space, mtr); @@ -1545,7 +1545,7 @@ static fseg_inode_t* fseg_inode_try_get( const fseg_header_t* header, - ulint space, + uint32_t space, ulint zip_size, mtr_t* mtr, buf_block_t** block) @@ -1582,7 +1582,7 @@ static fseg_inode_t* fseg_inode_get( const fseg_header_t* header, - ulint space, + uint32_t space, ulint zip_size, mtr_t* mtr, buf_block_t** block = NULL) @@ -2021,7 +2021,7 @@ fseg_alloc_free_page_low( xdes_t* ret_descr; /*!< the extent of the allocated page */ buf_block_t* xdes; ulint n; - const ulint space_id = space->id; + const uint32_t space_id = space->id; ut_ad((direction >= FSP_UP) && (direction <= FSP_NO_DIR)); ut_ad(mach_read_from_4(seg_inode + FSEG_MAGIC_N) @@ -2278,13 +2278,12 @@ fseg_alloc_free_page_general( in which the page should be initialized. */ { fseg_inode_t* inode; - ulint space_id; fil_space_t* space; buf_block_t* iblock; buf_block_t* block; uint32_t n_reserved; - space_id = page_get_space_id(page_align(seg_header)); + const uint32_t space_id = page_get_space_id(page_align(seg_header)); space = mtr->x_lock_space(space_id); inode = fseg_inode_get(seg_header, space_id, space->zip_size(), mtr, &iblock); diff --git a/storage/innobase/fsp/fsp0space.cc b/storage/innobase/fsp/fsp0space.cc index b069250ff9f..6bdf9fcc4d8 100644 --- a/storage/innobase/fsp/fsp0space.cc +++ b/storage/innobase/fsp/fsp0space.cc @@ -59,7 +59,7 @@ Tablespace::shutdown() m_files.clear(); ut_free(m_path); m_path = NULL; - m_space_id = ULINT_UNDEFINED; + m_space_id = UINT32_MAX; } /** Note that the data file was found. @@ -118,7 +118,7 @@ Tablespace::open_or_create(bool is_temp) /* Create the tablespace entry for the multi-file tablespace in the tablespace manager. */ - ulint fsp_flags = 0; + uint32_t fsp_flags; switch (srv_checksum_algorithm) { case SRV_CHECKSUM_ALGORITHM_FULL_CRC32: diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index bfb3075edc6..56bea723dc8 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -114,6 +114,12 @@ this program; if not, write to the Free Software Foundation, Inc., #include "ut0mem.h" #include "row0ext.h" +#include "lz4.h" +#include "lzo/lzo1x.h" +#include "lzma.h" +#include "bzlib.h" +#include "snappy-c.h" + #include <limits> #define thd_get_trx_isolation(X) ((enum_tx_isolation)thd_tx_isolation(X)) @@ -845,11 +851,6 @@ innodb_compression_algorithm_validate( for update function */ struct st_mysql_value* value); /*!< in: incoming string */ -static ibool innodb_have_lzo=IF_LZO(1, 0); -static ibool innodb_have_lz4=IF_LZ4(1, 0); -static ibool innodb_have_lzma=IF_LZMA(1, 0); -static ibool innodb_have_bzip2=IF_BZIP2(1, 0); -static ibool innodb_have_snappy=IF_SNAPPY(1, 0); static ibool innodb_have_punch_hole=IF_PUNCH_HOLE(1, 0); static @@ -1041,11 +1042,11 @@ static SHOW_VAR innodb_status_variables[]= { &export_vars.innodb_pages_encrypted, SHOW_LONGLONG}, {"num_pages_decrypted", &export_vars.innodb_pages_decrypted, SHOW_LONGLONG}, - {"have_lz4", &innodb_have_lz4, SHOW_BOOL}, - {"have_lzo", &innodb_have_lzo, SHOW_BOOL}, - {"have_lzma", &innodb_have_lzma, SHOW_BOOL}, - {"have_bzip2", &innodb_have_bzip2, SHOW_BOOL}, - {"have_snappy", &innodb_have_snappy, SHOW_BOOL}, + {"have_lz4", &(provider_service_lz4->is_loaded), SHOW_BOOL}, + {"have_lzo", &(provider_service_lzo->is_loaded), SHOW_BOOL}, + {"have_lzma", &(provider_service_lzma->is_loaded), SHOW_BOOL}, + {"have_bzip2", &(provider_service_bzip2->is_loaded), SHOW_BOOL}, + {"have_snappy", &(provider_service_snappy->is_loaded), SHOW_BOOL}, {"have_punch_hole", &innodb_have_punch_hole, SHOW_BOOL}, /* Defragmentation */ @@ -1767,16 +1768,11 @@ If correct, set the associated page_size_shift which is the power of 2 for this page size. @param[in] page_size Page Size to evaluate @return an associated page_size_shift if valid, 0 if invalid. */ -inline -ulong -innodb_page_size_validate( - ulong page_size) +inline uint32_t innodb_page_size_validate(ulong page_size) { - ulong n; - DBUG_ENTER("innodb_page_size_validate"); - for (n = UNIV_PAGE_SIZE_SHIFT_MIN; + for (uint32_t n = UNIV_PAGE_SIZE_SHIFT_MIN; n <= UNIV_PAGE_SIZE_SHIFT_MAX; n++) { if (page_size == static_cast<ulong>(1 << n)) { @@ -3743,23 +3739,36 @@ static const char* ha_innobase_exts[] = { @retval 0 if no system-versioned data was affected by the transaction */ static ulonglong innodb_prepare_commit_versioned(THD* thd, ulonglong *trx_id) { - if (const trx_t* trx = thd_to_trx(thd)) { - *trx_id = trx->id; - - for (const auto& t : trx->mod_tables) { - if (t.second.is_versioned()) { - DBUG_ASSERT(t.first->versioned_by_id()); - DBUG_ASSERT(trx->rsegs.m_redo.rseg); + if (trx_t *trx= thd_to_trx(thd)) + { + *trx_id= trx->id; + bool versioned= false; - return trx_sys.get_new_trx_id(); - } - } + for (auto &t : trx->mod_tables) + { + if (t.second.is_versioned()) + { + DBUG_ASSERT(t.first->versioned_by_id()); + DBUG_ASSERT(trx->rsegs.m_redo.rseg); + versioned= true; + if (!trx->bulk_insert) + break; + } + if (t.second.is_bulk_insert()) + { + ut_ad(trx->bulk_insert); + ut_ad(!trx->check_unique_secondary); + ut_ad(!trx->check_foreigns); + if (t.second.write_bulk(t.first, trx)) + return ULONGLONG_MAX; + } + } - return 0; - } + return versioned ? trx_sys.get_new_trx_id() : 0; + } - *trx_id = 0; - return 0; + *trx_id= 0; + return 0; } /** Initialize and normalize innodb_buffer_pool_size. */ @@ -3775,6 +3784,25 @@ static void innodb_buffer_pool_size_init() innobase_buffer_pool_size = srv_buf_pool_size; } + +static bool +compression_algorithm_is_not_loaded(ulong compression_algorithm, myf flags) +{ + bool is_loaded[PAGE_ALGORITHM_LAST+1]= { 1, 1, provider_service_lz4->is_loaded, + provider_service_lzo->is_loaded, provider_service_lzma->is_loaded, + provider_service_bzip2->is_loaded, provider_service_snappy->is_loaded }; + + DBUG_ASSERT(compression_algorithm <= PAGE_ALGORITHM_LAST); + + if (is_loaded[compression_algorithm]) + return 0; + + my_printf_error(HA_ERR_UNSUPPORTED, "InnoDB: compression algorithm %s (%u)" + " is not available. Please, load the corresponding provider plugin.", flags, + page_compression_algorithms[compression_algorithm], compression_algorithm); + return 1; +} + /** Initialize, validate and normalize the InnoDB startup parameters. @return failure code @retval 0 on success @@ -3810,50 +3838,8 @@ static int innodb_init_params() DBUG_RETURN(HA_ERR_INITIALIZATION); } -#ifndef HAVE_LZ4 - if (innodb_compression_algorithm == PAGE_LZ4_ALGORITHM) { - sql_print_error("InnoDB: innodb_compression_algorithm = %lu unsupported.\n" - "InnoDB: liblz4 is not installed. \n", - innodb_compression_algorithm); - DBUG_RETURN(HA_ERR_INITIALIZATION); - } -#endif - -#ifndef HAVE_LZO - if (innodb_compression_algorithm == PAGE_LZO_ALGORITHM) { - sql_print_error("InnoDB: innodb_compression_algorithm = %lu unsupported.\n" - "InnoDB: liblzo is not installed. \n", - innodb_compression_algorithm); - DBUG_RETURN(HA_ERR_INITIALIZATION); - } -#endif - -#ifndef HAVE_LZMA - if (innodb_compression_algorithm == PAGE_LZMA_ALGORITHM) { - sql_print_error("InnoDB: innodb_compression_algorithm = %lu unsupported.\n" - "InnoDB: liblzma is not installed. \n", - innodb_compression_algorithm); - DBUG_RETURN(HA_ERR_INITIALIZATION); - } -#endif - -#ifndef HAVE_BZIP2 - if (innodb_compression_algorithm == PAGE_BZIP2_ALGORITHM) { - sql_print_error("InnoDB: innodb_compression_algorithm = %lu unsupported.\n" - "InnoDB: libbz2 is not installed. \n", - innodb_compression_algorithm); - DBUG_RETURN(HA_ERR_INITIALIZATION); - } -#endif - -#ifndef HAVE_SNAPPY - if (innodb_compression_algorithm == PAGE_SNAPPY_ALGORITHM) { - sql_print_error("InnoDB: innodb_compression_algorithm = %lu unsupported.\n" - "InnoDB: libsnappy is not installed. \n", - innodb_compression_algorithm); - DBUG_RETURN(HA_ERR_INITIALIZATION); - } -#endif + if (compression_algorithm_is_not_loaded(innodb_compression_algorithm, ME_ERROR_LOG)) + DBUG_RETURN(HA_ERR_INITIALIZATION); if ((srv_encrypt_tables || srv_encrypt_log || innodb_encrypt_temporary_tables) @@ -4579,9 +4565,6 @@ innobase_commit( if (commit_trx || (!thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN))) { - DBUG_EXECUTE_IF("crash_innodb_before_commit", - DBUG_SUICIDE();); - /* Run the fast part of commit if we did not already. */ if (!trx->active_commit_ordered) { innobase_commit_ordered_2(trx, thd); @@ -4616,6 +4599,10 @@ innobase_commit( SQL statement */ trx_mark_sql_stat_end(trx); + if (UNIV_UNLIKELY(trx->error_state != DB_SUCCESS)) { + trx_rollback_for_mysql(trx); + DBUG_RETURN(1); + } } /* Reset the number AUTO-INC rows required */ @@ -10880,9 +10867,6 @@ err_col: ut_ad(dict_sys.sys_tables_exist()); err = row_create_table_for_mysql(table, m_trx); - - DBUG_EXECUTE_IF("ib_crash_during_create_for_encryption", - DBUG_SUICIDE();); } switch (err) { @@ -15707,6 +15691,7 @@ ha_innobase::extra( row_ins_duplicate_error_in_clust() will acquire a shared lock instead of an exclusive lock. */ stmt_boundary: + trx->bulk_insert_apply(); trx->end_bulk_insert(*m_prebuilt->table); trx->bulk_insert = false; break; @@ -15727,6 +15712,9 @@ ha_innobase::extra( if (trx->is_bulk_insert()) { /* Allow a subsequent INSERT into an empty table if !unique_checks && !foreign_key_checks. */ + if (dberr_t err = trx->bulk_insert_apply()) { + return err; + } break; } goto stmt_boundary; @@ -18906,10 +18894,23 @@ static MYSQL_SYSVAR_ULONG(purge_batch_size, srv_purge_batch_size, 1, /* Minimum value */ 5000, 0); /* Maximum value */ +extern void srv_update_purge_thread_count(uint n); + +static +void +innodb_purge_threads_update(THD*, struct st_mysql_sys_var*, void*, const void*save ) +{ + srv_update_purge_thread_count(*static_cast<const uint*>(save)); +} + static MYSQL_SYSVAR_UINT(purge_threads, srv_n_purge_threads, - PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY, + PLUGIN_VAR_OPCMDARG, "Number of tasks for purging transaction history", - NULL, NULL, 4, 1, innodb_purge_threads_MAX, 0); + NULL, innodb_purge_threads_update, + 4, /* Default setting */ + 1, /* Minimum value */ + innodb_purge_threads_MAX, /* Maximum value */ + 0); static MYSQL_SYSVAR_UINT(fast_shutdown, srv_fast_shutdown, PLUGIN_VAR_OPCMDARG, @@ -19389,7 +19390,7 @@ static MYSQL_SYSVAR_STR(undo_directory, srv_undo_dir, "Directory where undo tablespace files live, this path can be absolute.", NULL, NULL, NULL); -static MYSQL_SYSVAR_ULONG(undo_tablespaces, srv_undo_tablespaces, +static MYSQL_SYSVAR_UINT(undo_tablespaces, srv_undo_tablespaces, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "Number of undo tablespaces to use.", NULL, NULL, @@ -19670,29 +19671,7 @@ static MYSQL_SYSVAR_UINT(saved_page_number_debug, srv_saved_page_number_debug, PLUGIN_VAR_OPCMDARG, "An InnoDB page number.", NULL, NULL, 0, 0, UINT_MAX32, 0); - -static MYSQL_SYSVAR_BOOL(disable_resize_buffer_pool_debug, - buf_disable_resize_buffer_pool_debug, PLUGIN_VAR_NOCMDARG, - "Disable resizing buffer pool to make assertion code not expensive.", - NULL, NULL, TRUE); - -static MYSQL_SYSVAR_BOOL(page_cleaner_disabled_debug, - innodb_page_cleaner_disabled_debug, PLUGIN_VAR_OPCMDARG, - "Disable page cleaner", - NULL, NULL, FALSE); - -static MYSQL_SYSVAR_BOOL(dict_stats_disabled_debug, - innodb_dict_stats_disabled_debug, - PLUGIN_VAR_OPCMDARG, - "Disable dict_stats thread", - NULL, dict_stats_disabled_debug_update, FALSE); - -static MYSQL_SYSVAR_BOOL(master_thread_disabled_debug, - srv_master_thread_disabled_debug, - PLUGIN_VAR_OPCMDARG, - "Disable master thread", - NULL, srv_master_thread_disabled_debug_update, FALSE); -#endif /* UNIV_DEBUG */ +#endif static MYSQL_SYSVAR_BOOL(force_primary_key, srv_force_primary_key, @@ -19700,7 +19679,7 @@ static MYSQL_SYSVAR_BOOL(force_primary_key, "Do not allow creating a table without primary key (off by default)", NULL, NULL, FALSE); -static const char *page_compression_algorithms[]= { "none", "zlib", "lz4", "lzo", "lzma", "bzip2", "snappy", 0 }; +const char *page_compression_algorithms[]= { "none", "zlib", "lz4", "lzo", "lzma", "bzip2", "snappy", 0 }; static TYPELIB page_compression_algorithms_typelib= { array_elements(page_compression_algorithms) - 1, 0, @@ -19932,10 +19911,6 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(data_file_size_debug), MYSQL_SYSVAR(fil_make_page_dirty_debug), MYSQL_SYSVAR(saved_page_number_debug), - MYSQL_SYSVAR(disable_resize_buffer_pool_debug), - MYSQL_SYSVAR(page_cleaner_disabled_debug), - MYSQL_SYSVAR(dict_stats_disabled_debug), - MYSQL_SYSVAR(master_thread_disabled_debug), #endif /* UNIV_DEBUG */ MYSQL_SYSVAR(force_primary_key), MYSQL_SYSVAR(fatal_semaphore_wait_threshold), @@ -20924,21 +20899,6 @@ innodb_buffer_pool_size_validate( return(1); } -#ifdef UNIV_DEBUG - if (buf_disable_resize_buffer_pool_debug == TRUE) { - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - ER_WRONG_ARGUMENTS, - "Cannot update innodb_buffer_pool_size," - " because innodb_disable_resize_buffer_pool_debug" - " is set."); - ib::warn() << "Cannot update innodb_buffer_pool_size," - " because innodb_disable_resize_buffer_pool_debug" - " is set."; - return(1); - } -#endif /* UNIV_DEBUG */ - - mysql_mutex_lock(&buf_pool.mutex); if (srv_buf_pool_old_size != srv_buf_pool_size) { @@ -21000,70 +20960,14 @@ innodb_compression_algorithm_validate( for update function */ struct st_mysql_value* value) /*!< in: incoming string */ { - ulong compression_algorithm; DBUG_ENTER("innobase_compression_algorithm_validate"); if (check_sysvar_enum(thd, var, save, value)) { DBUG_RETURN(1); } - compression_algorithm = *reinterpret_cast<ulong*>(save); - (void)compression_algorithm; - -#ifndef HAVE_LZ4 - if (compression_algorithm == PAGE_LZ4_ALGORITHM) { - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - HA_ERR_UNSUPPORTED, - "InnoDB: innodb_compression_algorithm = %lu unsupported.\n" - "InnoDB: liblz4 is not installed. \n", - compression_algorithm); - DBUG_RETURN(1); - } -#endif - -#ifndef HAVE_LZO - if (compression_algorithm == PAGE_LZO_ALGORITHM) { - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - HA_ERR_UNSUPPORTED, - "InnoDB: innodb_compression_algorithm = %lu unsupported.\n" - "InnoDB: liblzo is not installed. \n", - compression_algorithm); - DBUG_RETURN(1); - } -#endif - -#ifndef HAVE_LZMA - if (compression_algorithm == PAGE_LZMA_ALGORITHM) { - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - HA_ERR_UNSUPPORTED, - "InnoDB: innodb_compression_algorithm = %lu unsupported.\n" - "InnoDB: liblzma is not installed. \n", - compression_algorithm); - DBUG_RETURN(1); - } -#endif - -#ifndef HAVE_BZIP2 - if (compression_algorithm == PAGE_BZIP2_ALGORITHM) { - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - HA_ERR_UNSUPPORTED, - "InnoDB: innodb_compression_algorithm = %lu unsupported.\n" - "InnoDB: libbz2 is not installed. \n", - compression_algorithm); - DBUG_RETURN(1); - } -#endif - -#ifndef HAVE_SNAPPY - if (compression_algorithm == PAGE_SNAPPY_ALGORITHM) { - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - HA_ERR_UNSUPPORTED, - "InnoDB: innodb_compression_algorithm = %lu unsupported.\n" - "InnoDB: libsnappy is not installed. \n", - compression_algorithm); - DBUG_RETURN(1); - } -#endif + if (compression_algorithm_is_not_loaded(*(ulong*)save, ME_WARNING)) + DBUG_RETURN(1); DBUG_RETURN(0); } diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index a68a17d3401..1965994ac37 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -10478,8 +10478,8 @@ commit_cache_norebuild( space->flags |= FSP_FLAGS_MASK_PAGE_COMPRESSION; } else if (!space->is_compressed()) { - space->flags - |= innodb_compression_algorithm + space->flags |= static_cast<uint32_t>( + innodb_compression_algorithm) << FSP_FLAGS_FCRC32_POS_COMPRESSED_ALGO; } mysql_mutex_unlock(&fil_system.mutex); diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 6e19248d578..e666c285bf0 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -2256,7 +2256,7 @@ static void ibuf_read_merge_pages(const uint32_t* space_ids, #endif for (ulint i = 0; i < n_stored; i++) { - const ulint space_id = space_ids[i]; + const uint32_t space_id = space_ids[i]; fil_space_t* s = fil_space_t::get(space_id); if (!s) { tablespace_deleted: @@ -2942,7 +2942,7 @@ void ibuf_update_max_tablespace_id(void) /*===============================*/ { - ulint max_space_id; + uint32_t max_space_id; const rec_t* rec; const byte* field; ulint len; @@ -4466,7 +4466,7 @@ reset_bit: /** Delete all change buffer entries for a tablespace, in DISCARD TABLESPACE, IMPORT TABLESPACE, or crash recovery. @param[in] space missing or to-be-discarded tablespace */ -void ibuf_delete_for_discarded_space(ulint space) +void ibuf_delete_for_discarded_space(uint32_t space) { mem_heap_t* heap; btr_pcur_t pcur; diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 3f7cc4fb516..703087d1b7f 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -74,11 +74,6 @@ the blocks on free list. If LRU list is very small then we can end up in thrashing. */ #define BUF_LRU_MIN_LEN 256 -# ifdef UNIV_DEBUG -extern my_bool buf_disable_resize_buffer_pool_debug; /*!< if TRUE, resizing - buffer pool is not allowed. */ -# endif /* UNIV_DEBUG */ - /** This structure defines information we will fetch from each buffer pool. It will be used to print table IO stats */ struct buf_pool_info_t @@ -347,16 +342,13 @@ buf_block_get_modify_clock( bool buf_is_zeroes(st_::span<const byte> buf); /** Check if a page is corrupt. -@param[in] check_lsn whether the LSN should be checked -@param[in] read_buf database page -@param[in] fsp_flags tablespace flags +@param check_lsn whether FIL_PAGE_LSN should be checked +@param read_buf database page +@param fsp_flags contents of FIL_SPACE_FLAGS @return whether the page is corrupted */ -bool -buf_page_is_corrupted( - bool check_lsn, - const byte* read_buf, - ulint fsp_flags) - MY_ATTRIBUTE((warn_unused_result)); +bool buf_page_is_corrupted(bool check_lsn, const byte *read_buf, + uint32_t fsp_flags) + MY_ATTRIBUTE((warn_unused_result)); inline void *aligned_malloc(size_t size, size_t align) { @@ -385,7 +377,8 @@ stored in 26th position. @param[in] read_buf database page @param[in] fsp_flags tablespace flags @return key version of the page. */ -inline uint32_t buf_page_get_key_version(const byte* read_buf, ulint fsp_flags) +inline uint32_t buf_page_get_key_version(const byte* read_buf, + uint32_t fsp_flags) { static_assert(FIL_PAGE_FCRC32_KEY_VERSION == 0, "compatibility"); return fil_space_t::full_crc32(fsp_flags) @@ -400,7 +393,7 @@ stored in page type. @param[in] read_buf database page @param[in] fsp_flags tablespace flags @return true if page is compressed. */ -inline bool buf_page_is_compressed(const byte* read_buf, ulint fsp_flags) +inline bool buf_page_is_compressed(const byte* read_buf, uint32_t fsp_flags) { uint16_t page_type= fil_page_get_type(read_buf); return fil_space_t::full_crc32(fsp_flags) @@ -503,12 +496,10 @@ buf_pool_size_align( /** Verify that post encryption checksum match with the calculated checksum. This function should be called only if tablespace contains crypt data metadata. -@param[in] page page frame -@param[in] fsp_flags tablespace flags -@return true if page is encrypted and OK, false otherwise */ -bool buf_page_verify_crypt_checksum( - const byte* page, - ulint fsp_flags); +@param page page frame +@param fsp_flags contents of FSP_SPACE_FLAGS +@return whether the page is encrypted and valid */ +bool buf_page_verify_crypt_checksum(const byte *page, uint32_t fsp_flags); /** Calculate a ROW_FORMAT=COMPRESSED page checksum and update the page. @param[in,out] page page to update diff --git a/storage/innobase/include/buf0flu.h b/storage/innobase/include/buf0flu.h index 027d1102004..665fd1115e7 100644 --- a/storage/innobase/include/buf0flu.h +++ b/storage/innobase/include/buf0flu.h @@ -41,19 +41,12 @@ extern ulint buf_lru_freed_page_count; /** Flag indicating if the page_cleaner is in active state. */ extern Atomic_relaxed<bool> buf_page_cleaner_is_active; -#ifdef UNIV_DEBUG - -/** Value of MySQL global variable used to disable page cleaner. */ -extern my_bool innodb_page_cleaner_disabled_debug; - -#endif /* UNIV_DEBUG */ - /** Remove all dirty pages belonging to a given tablespace when we are deleting the data file of that tablespace. The pages still remain a part of LRU and are evicted from the list as they age towards the tail of the LRU. @param id tablespace identifier */ -void buf_flush_remove_pages(ulint id); +void buf_flush_remove_pages(uint32_t id); /*******************************************************************//** Relocates a buffer control block on the flush_list. diff --git a/storage/innobase/include/buf0rea.h b/storage/innobase/include/buf0rea.h index 8d6b28194dc..986a792b97e 100644 --- a/storage/innobase/include/buf0rea.h +++ b/storage/innobase/include/buf0rea.h @@ -101,12 +101,10 @@ which could result in a deadlock if the OS does not support asynchronous io. ulint buf_read_ahead_linear(const page_id_t page_id, ulint zip_size, bool ibuf); -/** Issues read requests for pages which recovery wants to read in. -@param[in] space_id tablespace id -@param[in] page_nos array of page numbers to read, with the -highest page number the last in the array -@param[in] n number of page numbers in the array */ -void buf_read_recv_pages(ulint space_id, const uint32_t* page_nos, ulint n); +/** Issue read requests for pages that need to be recovered. +@param space_id tablespace identifier +@param page_nos page numbers to read, in ascending order */ +void buf_read_recv_pages(uint32_t space_id, st_::span<uint32_t> page_nos); /** @name Modes used in read-ahead @{ */ /** read only pages belonging to the insert buffer tree */ diff --git a/storage/innobase/include/buf0types.h b/storage/innobase/include/buf0types.h index 4e29b41124e..6bc46dbbc04 100644 --- a/storage/innobase/include/buf0types.h +++ b/storage/innobase/include/buf0types.h @@ -91,12 +91,10 @@ class page_id_t public: /** Constructor from (space, page_no). - @param[in] space tablespace id - @param[in] page_no page number */ - page_id_t(ulint space, uint32_t page_no) : m_id(uint64_t{space} << 32 | page_no) - { - ut_ad(space <= 0xFFFFFFFFU); - } + @param space tablespace id + @param page_no page number */ + page_id_t(uint32_t space, uint32_t page_no) : + m_id(uint64_t{space} << 32 | page_no) {} page_id_t(uint64_t id) : m_id(id) {} bool operator==(const page_id_t& rhs) const { return m_id == rhs.m_id; } diff --git a/storage/innobase/include/dict0boot.h b/storage/innobase/include/dict0boot.h index 869905c6b97..07ecfe4edff 100644 --- a/storage/innobase/include/dict0boot.h +++ b/storage/innobase/include/dict0boot.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2018, 2020, MariaDB Corporation. +Copyright (c) 2018, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -45,7 +45,7 @@ dict_hdr_get_new_id( (not assigned if NULL) */ index_id_t* index_id, /*!< out: index id (not assigned if NULL) */ - ulint* space_id); /*!< out: space id + uint32_t* space_id); /*!< out: space id (not assigned if NULL) */ /** Update dict_sys.row_id in the dictionary header file page. */ void dict_hdr_flush_row_id(row_id_t id); diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index 07acd0ecb74..7493359e16a 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -871,11 +871,8 @@ fil_space_t::flags | 0 | 0 | 1 | 1 ================================================================== @param[in] table_flags dict_table_t::flags @return tablespace flags (fil_space_t::flags) */ -UNIV_INLINE -ulint -dict_tf_to_fsp_flags(ulint table_flags) - MY_ATTRIBUTE((const)); - +inline uint32_t dict_tf_to_fsp_flags(unsigned table_flags) + MY_ATTRIBUTE((const)); /** Extract the ROW_FORMAT=COMPRESSED page size from table flags. @param[in] flags flags diff --git a/storage/innobase/include/dict0dict.inl b/storage/innobase/include/dict0dict.inl index a98ca38d9b3..8af2d93f831 100644 --- a/storage/innobase/include/dict0dict.inl +++ b/storage/innobase/include/dict0dict.inl @@ -618,19 +618,16 @@ fil_space_t::flags | 0 | 0 | 1 | 1 ================================================================== @param[in] table_flags dict_table_t::flags @return tablespace flags (fil_space_t::flags) */ -UNIV_INLINE -ulint -dict_tf_to_fsp_flags(ulint table_flags) +inline uint32_t dict_tf_to_fsp_flags(unsigned table_flags) { - ulint fsp_flags; - ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL( + uint32_t fsp_flags; + uint32_t page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL( table_flags); ut_ad((DICT_TF_GET_PAGE_COMPRESSION(table_flags) == 0) == (page_compression_level == 0)); - DBUG_EXECUTE_IF("dict_tf_to_fsp_flags_failure", - return(ULINT_UNDEFINED);); + DBUG_EXECUTE_IF("dict_tf_to_fsp_flags_failure", return UINT32_MAX;); /* No ROW_FORMAT=COMPRESSED for innodb_checksum_algorithm=full_crc32 */ if ((srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_FULL_CRC32 @@ -641,7 +638,8 @@ dict_tf_to_fsp_flags(ulint table_flags) | FSP_FLAGS_FCRC32_PAGE_SSIZE(); if (page_compression_level) { - fsp_flags |= innodb_compression_algorithm + fsp_flags |= static_cast<uint32_t>( + innodb_compression_algorithm) << FSP_FLAGS_FCRC32_POS_COMPRESSED_ALGO; } } else { diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index dac738fbc68..0b690bb865e 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -112,7 +112,7 @@ are described in fsp0fsp.h. */ /** This bitmask is used in SYS_TABLES.N_COLS to set and test whether the Compact page format is used, i.e ROW_FORMAT != REDUNDANT */ -#define DICT_N_COLS_COMPACT 0x80000000UL +constexpr uint32_t DICT_N_COLS_COMPACT= 1U << 31; /** Width of the COMPACT flag */ #define DICT_TF_WIDTH_COMPACT 1 @@ -1184,6 +1184,13 @@ public: /** @return whether this is the change buffer */ bool is_ibuf() const { return UNIV_UNLIKELY(type & DICT_IBUF); } + /** @return whether this is a normal B-tree index + (not the change buffer, not SPATIAL or FULLTEXT) */ + bool is_btree() const { + return UNIV_LIKELY(!(type & (DICT_IBUF | DICT_SPATIAL + | DICT_FTS | DICT_CORRUPT))); + } + /** @return whether the index includes virtual columns */ bool has_virtual() const { return type & DICT_VIRTUAL; } @@ -2023,7 +2030,7 @@ public: /** The tablespace of the table */ fil_space_t* space; /** Tablespace ID */ - ulint space_id; + uint32_t space_id; /** Stores information about: 1 row format (redundant or compact), diff --git a/storage/innobase/include/dict0stats_bg.h b/storage/innobase/include/dict0stats_bg.h index f047d404693..5604420134b 100644 --- a/storage/innobase/include/dict0stats_bg.h +++ b/storage/innobase/include/dict0stats_bg.h @@ -34,11 +34,6 @@ Created Apr 26, 2012 Vasil Dimov extern mysql_pfs_key_t recalc_pool_mutex_key; #endif /* HAVE_PSI_INTERFACE */ -#ifdef UNIV_DEBUG -/** Value of MySQL global used to disable dict_stats thread. */ -extern my_bool innodb_dict_stats_disabled_debug; -#endif /* UNIV_DEBUG */ - /** Delete a table from the auto recalc pool, and ensure that no statistics are being updated on it. */ void dict_stats_recalc_pool_del(table_id_t id, bool have_mdl_exclusive); @@ -53,14 +48,6 @@ Free resources allocated by dict_stats_thread_init(), must be called after dict_stats task has exited. */ void dict_stats_deinit(); -#ifdef UNIV_DEBUG -/** Disables dict stats thread. It's used by: - SET GLOBAL innodb_dict_stats_disabled_debug = 1 (0). -@param[in] save immediate result from check function */ -void dict_stats_disabled_debug_update(THD*, st_mysql_sys_var*, void*, - const void* save); -#endif /* UNIV_DEBUG */ - /** Start the dict stats timer. */ void dict_stats_start(); diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index c3abdad90ed..d5df0ca126a 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -291,19 +291,19 @@ byte* fil_space_encrypt( /** Decrypt a page. @param]in] space_id space id +@param[in] fsp_flags Tablespace flags @param[in] crypt_data crypt_data @param[in] tmp_frame Temporary buffer @param[in] physical_size page size -@param[in] fsp_flags Tablespace flags @param[in,out] src_frame Page to decrypt @return DB_SUCCESS or error */ dberr_t fil_space_decrypt( - ulint space_id, + uint32_t space_id, + uint32_t fsp_flags, fil_space_crypt_t* crypt_data, byte* tmp_frame, ulint physical_size, - ulint fsp_flags, byte* src_frame); /****************************************************************** diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index cf6873f8850..63f83ff0f96 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -43,7 +43,7 @@ Created 10/25/1995 Heikki Tuuri #include <mutex> struct unflushed_spaces_tag_t; -struct rotation_list_tag_t; +struct default_encrypt_tag_t; struct space_list_tag_t; struct named_spaces_tag_t; @@ -82,14 +82,14 @@ enum srv_flush_t extern ulong srv_file_flush_method; /** Undo tablespaces starts with space_id. */ -extern ulint srv_undo_space_id_start; +extern uint32_t srv_undo_space_id_start; /** The number of UNDO tablespaces that are open and ready to use. */ -extern ulint srv_undo_tablespaces_open; +extern uint32_t srv_undo_tablespaces_open; /** Check whether given space id is undo tablespace id @param[in] space_id space id to check @return true if it is undo tablespace else false. */ -inline bool srv_is_undo_tablespace(ulint space_id) +inline bool srv_is_undo_tablespace(uint32_t space_id) { return srv_undo_space_id_start > 0 && space_id >= srv_undo_space_id_start && @@ -336,7 +336,7 @@ enum fil_encryption_t }; struct fil_space_t final : ilist_node<unflushed_spaces_tag_t>, - ilist_node<rotation_list_tag_t>, + ilist_node<default_encrypt_tag_t>, ilist_node<space_list_tag_t>, ilist_node<named_spaces_tag_t> #else @@ -352,8 +352,6 @@ struct fil_space_t final latch.destroy(); } - ulint id; /*!< space id */ - /** fil_system.spaces chain node */ fil_space_t *hash; lsn_t max_lsn; @@ -363,6 +361,8 @@ struct fil_space_t final Protected by log_sys.mutex. If and only if this is nonzero, the tablespace will be in named_spaces. */ + /** tablespace identifier */ + uint32_t id; /** whether undo tablespace truncation is in progress */ bool is_being_truncated; fil_type_t purpose;/*!< purpose */ @@ -522,7 +522,7 @@ public: @param id tablespace identifier @return tablespace @retval nullptr if no tablespace was found */ - static fil_space_t *check_pending_operations(ulint id); + static fil_space_t *check_pending_operations(uint32_t id); private: MY_ATTRIBUTE((warn_unused_result)) @@ -668,92 +668,78 @@ public: freed_ranges.clear(); } #endif /* !UNIV_INNOCHECKSUM */ - /** FSP_SPACE_FLAGS and FSP_FLAGS_MEM_ flags; - check fsp0types.h to more info about flags. */ - ulint flags; - - /** Determine if full_crc32 is used for a data file - @param[in] flags tablespace flags (FSP_SPACE_FLAGS) - @return whether the full_crc32 algorithm is active */ - static bool full_crc32(ulint flags) { - return flags & FSP_FLAGS_FCRC32_MASK_MARKER; - } - /** Determine if full_crc32 is used along with compression */ - static bool is_full_crc32_compressed(ulint flags) + /** FSP_SPACE_FLAGS and FSP_FLAGS_MEM_ flags; + check fsp0types.h to more info about flags. */ + uint32_t flags; + + /** Determine if full_crc32 is used for a data file + @param[in] flags tablespace flags (FSP_SPACE_FLAGS) + @return whether the full_crc32 algorithm is active */ + static bool full_crc32(uint32_t flags) + { return flags & FSP_FLAGS_FCRC32_MASK_MARKER; } + /** @return whether innodb_checksum_algorithm=full_crc32 is active */ + bool full_crc32() const { return full_crc32(flags); } + /** Determine if full_crc32 is used along with PAGE_COMPRESSED */ + static bool is_full_crc32_compressed(uint32_t flags) { - if (full_crc32(flags)) - { - ulint algo= FSP_FLAGS_FCRC32_GET_COMPRESSED_ALGO(flags); - DBUG_ASSERT(algo <= PAGE_ALGORITHM_LAST); - return algo > 0; + if (!full_crc32(flags)) + return false; + auto algo= FSP_FLAGS_FCRC32_GET_COMPRESSED_ALGO(flags); + DBUG_ASSERT(algo <= PAGE_ALGORITHM_LAST); + return algo != 0; + } + /** Determine the logical page size. + @param flags tablespace flags (FSP_SPACE_FLAGS) + @return the logical page size + @retval 0 if the flags are invalid */ + static unsigned logical_size(uint32_t flags) + { + switch (full_crc32(flags) + ? FSP_FLAGS_FCRC32_GET_PAGE_SSIZE(flags) + : FSP_FLAGS_GET_PAGE_SSIZE(flags)) { + case 3: return 4096; + case 4: return 8192; + case 5: return full_crc32(flags) ? 16384 : 0; + case 0: return full_crc32(flags) ? 0 : 16384; + case 6: return 32768; + case 7: return 65536; + default: return 0; } - - return false; } - /** @return whether innodb_checksum_algorithm=full_crc32 is active */ - bool full_crc32() const { return full_crc32(flags); } - /** Determine the logical page size. - @param flags tablespace flags (FSP_FLAGS) - @return the logical page size - @retval 0 if the flags are invalid */ - static unsigned logical_size(ulint flags) { - - ulint page_ssize = 0; - - if (full_crc32(flags)) { - page_ssize = FSP_FLAGS_FCRC32_GET_PAGE_SSIZE(flags); - } else { - page_ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); - } - - switch (page_ssize) { - case 3: return 4096; - case 4: return 8192; - case 5: - { ut_ad(full_crc32(flags)); return 16384; } - case 0: - { ut_ad(!full_crc32(flags)); return 16384; } - case 6: return 32768; - case 7: return 65536; - default: return 0; - } - } - /** Determine the ROW_FORMAT=COMPRESSED page size. - @param flags tablespace flags (FSP_FLAGS) - @return the ROW_FORMAT=COMPRESSED page size - @retval 0 if ROW_FORMAT=COMPRESSED is not used */ - static unsigned zip_size(ulint flags) { - - if (full_crc32(flags)) { - return 0; - } + /** Determine the ROW_FORMAT=COMPRESSED page size. + @param flags tablespace flags (FSP_SPACE_FLAGS) + @return the ROW_FORMAT=COMPRESSED page size + @retval 0 if ROW_FORMAT=COMPRESSED is not used */ + static unsigned zip_size(uint32_t flags) + { + if (full_crc32(flags)) + return 0; + const uint32_t zip_ssize= FSP_FLAGS_GET_ZIP_SSIZE(flags); + return zip_ssize ? (UNIV_ZIP_SIZE_MIN >> 1) << zip_ssize : 0; + } + /** Determine the physical page size. + @param flags tablespace flags (FSP_SPACE_FLAGS) + @return the physical page size */ + static unsigned physical_size(uint32_t flags) + { + if (full_crc32(flags)) + return logical_size(flags); - ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); - return zip_ssize - ? (UNIV_ZIP_SIZE_MIN >> 1) << zip_ssize : 0; - } - /** Determine the physical page size. - @param flags tablespace flags (FSP_FLAGS) - @return the physical page size */ - static unsigned physical_size(ulint flags) { + const uint32_t zip_ssize= FSP_FLAGS_GET_ZIP_SSIZE(flags); + return zip_ssize + ? (UNIV_ZIP_SIZE_MIN >> 1) << zip_ssize + : unsigned(srv_page_size); + } - if (full_crc32(flags)) { - return logical_size(flags); - } + /** @return the ROW_FORMAT=COMPRESSED page size + @retval 0 if ROW_FORMAT=COMPRESSED is not used */ + unsigned zip_size() const { return zip_size(flags); } + /** @return the physical page size */ + unsigned physical_size() const { return physical_size(flags); } - ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); - return zip_ssize - ? (UNIV_ZIP_SIZE_MIN >> 1) << zip_ssize - : unsigned(srv_page_size); - } - /** @return the ROW_FORMAT=COMPRESSED page size - @retval 0 if ROW_FORMAT=COMPRESSED is not used */ - unsigned zip_size() const { return zip_size(flags); } - /** @return the physical page size */ - unsigned physical_size() const { return physical_size(flags); } - /** Check whether the compression enabled in tablespace. + /** Check whether PAGE_COMPRESSED is enabled. @param[in] flags tablespace flags */ - static bool is_compressed(ulint flags) + static bool is_compressed(uint32_t flags) { return is_full_crc32_compressed(flags) || FSP_FLAGS_HAS_PAGE_COMPRESSION(flags); @@ -761,187 +747,151 @@ public: /** @return whether the compression enabled for the tablespace. */ bool is_compressed() const { return is_compressed(flags); } - /** Get the compression algorithm for full crc32 format. - @param[in] flags tablespace flags - @return algorithm type of tablespace */ - static ulint get_compression_algo(ulint flags) - { - return full_crc32(flags) - ? FSP_FLAGS_FCRC32_GET_COMPRESSED_ALGO(flags) - : 0; - } - /** @return the page_compressed algorithm - @retval 0 if not page_compressed */ - ulint get_compression_algo() const { - return fil_space_t::get_compression_algo(flags); - } - /** Determine if the page_compressed page contains an extra byte - for exact compressed stream length - @param[in] flags tablespace flags - @return whether the extra byte is needed */ - static bool full_crc32_page_compressed_len(ulint flags) - { - DBUG_ASSERT(full_crc32(flags)); - switch (get_compression_algo(flags)) { - case PAGE_LZ4_ALGORITHM: - case PAGE_LZO_ALGORITHM: - case PAGE_SNAPPY_ALGORITHM: - return true; - } - return false; - } - - /** Whether the full checksum matches with non full checksum flags. - @param[in] flags flags present - @param[in] expected expected flags - @return true if it is equivalent */ - static bool is_flags_full_crc32_equal(ulint flags, ulint expected) - { - ut_ad(full_crc32(flags)); - ulint fcrc32_psize = FSP_FLAGS_FCRC32_GET_PAGE_SSIZE(flags); - - if (full_crc32(expected)) { - /* The data file may have been created with a - different innodb_compression_algorithm. But - we only support one innodb_page_size for all files. */ - return fcrc32_psize - == FSP_FLAGS_FCRC32_GET_PAGE_SSIZE(expected); - } - - ulint non_fcrc32_psize = FSP_FLAGS_GET_PAGE_SSIZE(expected); - - if (!non_fcrc32_psize) { - if (fcrc32_psize != 5) { - return false; - } - } else if (fcrc32_psize != non_fcrc32_psize) { - return false; - } - - return true; - } - /** Whether old tablespace flags match full_crc32 flags. - @param[in] flags flags present - @param[in] expected expected flags - @return true if it is equivalent */ - static bool is_flags_non_full_crc32_equal(ulint flags, ulint expected) - { - ut_ad(!full_crc32(flags)); - - if (!full_crc32(expected)) { - return false; - } - - ulint non_fcrc32_psize = FSP_FLAGS_GET_PAGE_SSIZE(flags); - ulint fcrc32_psize = FSP_FLAGS_FCRC32_GET_PAGE_SSIZE( - expected); - - if (!non_fcrc32_psize) { - if (fcrc32_psize != 5) { - return false; - } - } else if (fcrc32_psize != non_fcrc32_psize) { - return false; - } - - return true; - } - /** Whether both fsp flags are equivalent */ - static bool is_flags_equal(ulint flags, ulint expected) - { - if (!((flags ^ expected) & ~(1U << FSP_FLAGS_POS_RESERVED))) { - return true; - } - - return full_crc32(flags) - ? is_flags_full_crc32_equal(flags, expected) - : is_flags_non_full_crc32_equal(flags, expected); - } - /** Validate the tablespace flags for full crc32 format. - @param[in] flags the content of FSP_SPACE_FLAGS - @return whether the flags are correct in full crc32 format */ - static bool is_fcrc32_valid_flags(ulint flags) - { - ut_ad(flags & FSP_FLAGS_FCRC32_MASK_MARKER); - const ulint page_ssize = physical_size(flags); - if (page_ssize < 3 || page_ssize & 8) { - return false; - } - - flags >>= FSP_FLAGS_FCRC32_POS_COMPRESSED_ALGO; - - return flags <= PAGE_ALGORITHM_LAST; - } - /** Validate the tablespace flags. - @param[in] flags content of FSP_SPACE_FLAGS - @param[in] is_ibd whether this is an .ibd file - (not system tablespace) - @return whether the flags are correct. */ - static bool is_valid_flags(ulint flags, bool is_ibd) - { - DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", - return false;); - - if (full_crc32(flags)) { - return is_fcrc32_valid_flags(flags); - } + /** Get the compression algorithm for full crc32 format. + @param flags contents of FSP_SPACE_FLAGS + @return PAGE_COMPRESSED algorithm of full_crc32 tablespace + @retval 0 if not PAGE_COMPRESSED or not full_crc32 */ + static unsigned get_compression_algo(uint32_t flags) + { + return full_crc32(flags) + ? FSP_FLAGS_FCRC32_GET_COMPRESSED_ALGO(flags) + : 0; + } + /** @return the page_compressed algorithm + @retval 0 if not page_compressed */ + unsigned get_compression_algo() const { return get_compression_algo(flags); } + /** Determine if the page_compressed page contains an extra byte + for exact compressed stream length + @param flags contents of FSP_SPACE_FLAGS + @return whether the extra byte is needed */ + static bool full_crc32_page_compressed_len(uint32_t flags) + { + DBUG_ASSERT(full_crc32(flags)); + switch (get_compression_algo(flags)) { + case PAGE_LZ4_ALGORITHM: + case PAGE_LZO_ALGORITHM: + case PAGE_SNAPPY_ALGORITHM: + return true; + } + return false; + } - if (flags == 0) { - return true; - } + /** Whether the full checksum matches with non full checksum flags. + @param flags contents of FSP_SPACE_FLAGS + @param expected expected flags + @return true if it is equivalent */ + static bool is_flags_full_crc32_equal(uint32_t flags, uint32_t expected) + { + ut_ad(full_crc32(flags)); + uint32_t fcrc32_psize= FSP_FLAGS_FCRC32_GET_PAGE_SSIZE(flags); + + if (full_crc32(expected)) + /* The data file may have been created with a + different innodb_compression_algorithm. But + we only support one innodb_page_size for all files. */ + return fcrc32_psize == FSP_FLAGS_FCRC32_GET_PAGE_SSIZE(expected); + + uint32_t non_fcrc32_psize = FSP_FLAGS_GET_PAGE_SSIZE(expected); + if (!non_fcrc32_psize) + return fcrc32_psize == 5; + return fcrc32_psize == non_fcrc32_psize; + } - if (flags & ~FSP_FLAGS_MASK) { - return false; - } + /** Whether old tablespace flags match full_crc32 flags. + @param flags contents of FSP_SPACE_FLAGS + @param expected expected flags + @return true if it is equivalent */ + static bool is_flags_non_full_crc32_equal(uint32_t flags, uint32_t expected) + { + ut_ad(!full_crc32(flags)); + if (!full_crc32(expected)) + return false; - if ((flags & (FSP_FLAGS_MASK_POST_ANTELOPE - | FSP_FLAGS_MASK_ATOMIC_BLOBS)) - == FSP_FLAGS_MASK_ATOMIC_BLOBS) { - /* If the "atomic blobs" flag (indicating - ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED) flag - is set, then the "post Antelope" - (ROW_FORMAT!=REDUNDANT) flag must also be set. */ - return false; - } + uint32_t non_fcrc32_psize= FSP_FLAGS_GET_PAGE_SSIZE(flags); + uint32_t fcrc32_psize = FSP_FLAGS_FCRC32_GET_PAGE_SSIZE(expected); - /* Bits 10..14 should be 0b0000d where d is the DATA_DIR flag - of MySQL 5.6 and MariaDB 10.0, which we ignore. - In the buggy FSP_SPACE_FLAGS written by MariaDB 10.1.0 to 10.1.20, - bits 10..14 would be nonzero 0bsssaa where sss is - nonzero PAGE_SSIZE (3, 4, 6, or 7) - and aa is ATOMIC_WRITES (not 0b11). */ - if (FSP_FLAGS_GET_RESERVED(flags) & ~1U) { - return false; - } + if (!non_fcrc32_psize) + return fcrc32_psize == 5; + return fcrc32_psize == non_fcrc32_psize; + } - const ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); - if (ssize == 1 || ssize == 2 || ssize == 5 || ssize & 8) { - /* the page_size is not between 4k and 64k; - 16k should be encoded as 0, not 5 */ - return false; - } + /** Whether both fsp flags are equivalent */ + static bool is_flags_equal(uint32_t flags, uint32_t expected) + { + if (!((flags ^ expected) & ~(1U << FSP_FLAGS_POS_RESERVED))) + return true; + return full_crc32(flags) + ? is_flags_full_crc32_equal(flags, expected) + : is_flags_non_full_crc32_equal(flags, expected); + } - const ulint zssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); - if (zssize == 0) { - /* not ROW_FORMAT=COMPRESSED */ - } else if (zssize > (ssize ? ssize : 5)) { - /* Invalid KEY_BLOCK_SIZE */ - return false; - } else if (~flags & (FSP_FLAGS_MASK_POST_ANTELOPE - | FSP_FLAGS_MASK_ATOMIC_BLOBS)) { - /* both these flags should be set for - ROW_FORMAT=COMPRESSED */ - return false; - } + /** Validate the tablespace flags for full crc32 format. + @param flags contents of FSP_SPACE_FLAGS + @return whether the flags are correct in full crc32 format */ + static bool is_fcrc32_valid_flags(uint32_t flags) + { + ut_ad(flags & FSP_FLAGS_FCRC32_MASK_MARKER); + const ulint page_ssize= physical_size(flags); + if (page_ssize < 3 || page_ssize & 8) + return false; + flags >>= FSP_FLAGS_FCRC32_POS_COMPRESSED_ALGO; + return flags <= PAGE_ALGORITHM_LAST; + } + /** Validate the tablespace flags. + @param flags contents of FSP_SPACE_FLAGS + @param is_ibd whether this is an .ibd file (not system tablespace) + @return whether the flags are correct */ + static bool is_valid_flags(uint32_t flags, bool is_ibd) + { + DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", return false;); + if (full_crc32(flags)) + return is_fcrc32_valid_flags(flags); - /* The flags do look valid. But, avoid misinterpreting - buggy MariaDB 10.1 format flags for - PAGE_COMPRESSED=1 PAGE_COMPRESSION_LEVEL={0,2,3} - as valid-looking PAGE_SSIZE if this is known to be - an .ibd file and we are using the default innodb_page_size=16k. */ - return(ssize == 0 || !is_ibd - || srv_page_size != UNIV_PAGE_SIZE_ORIG); - } + if (flags == 0) + return true; + if (~FSP_FLAGS_MASK & flags) + return false; + + if (FSP_FLAGS_MASK_ATOMIC_BLOBS == + (flags & (FSP_FLAGS_MASK_POST_ANTELOPE | FSP_FLAGS_MASK_ATOMIC_BLOBS))) + /* If the "atomic blobs" flag (indicating + ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED) flag is set, then the + ROW_FORMAT!=REDUNDANT flag must also be set. */ + return false; + + /* Bits 10..14 should be 0b0000d where d is the DATA_DIR flag + of MySQL 5.6 and MariaDB 10.0, which we ignore. + In the buggy FSP_SPACE_FLAGS written by MariaDB 10.1.0 to 10.1.20, + bits 10..14 would be nonzero 0bsssaa where sss is + nonzero PAGE_SSIZE (3, 4, 6, or 7) + and aa is ATOMIC_WRITES (not 0b11). */ + if (FSP_FLAGS_GET_RESERVED(flags) & ~1U) + return false; + + const uint32_t ssize= FSP_FLAGS_GET_PAGE_SSIZE(flags); + if (ssize == 1 || ssize == 2 || ssize == 5 || ssize & 8) + /* the page_size is not between 4k and 64k; + 16k should be encoded as 0, not 5 */ + return false; + + const uint32_t zssize= FSP_FLAGS_GET_ZIP_SSIZE(flags); + if (zssize == 0) + /* not ROW_FORMAT=COMPRESSED */; + else if (zssize > (ssize ? ssize : 5)) + /* Invalid KEY_BLOCK_SIZE */ + return false; + else if (~flags & + (FSP_FLAGS_MASK_POST_ANTELOPE | FSP_FLAGS_MASK_ATOMIC_BLOBS)) + /* both these flags must set for ROW_FORMAT=COMPRESSED */ + return false; + + /* The flags do look valid. But, avoid misinterpreting + buggy MariaDB 10.1 format flags for + PAGE_COMPRESSED=1 PAGE_COMPRESSION_LEVEL={0,2,3} + as valid-looking PAGE_SSIZE if this is known to be + an .ibd file and we are using the default innodb_page_size=16k. */ + return(ssize == 0 || !is_ibd || srv_page_size != UNIV_PAGE_SIZE_ORIG); + } #ifndef UNIV_INNOCHECKSUM MY_ATTRIBUTE((warn_unused_result)) @@ -953,7 +903,7 @@ public: @param mode encryption mode @return pointer to created tablespace, to be filled in with add() @retval nullptr on failure (such as when the same tablespace exists) */ - static fil_space_t *create(ulint id, ulint flags, + static fil_space_t *create(uint32_t id, uint32_t flags, fil_type_t purpose, fil_space_crypt_t *crypt_data, fil_encryption_t mode= FIL_ENCRYPTION_DEFAULT); @@ -962,7 +912,7 @@ public: @param id tablespace identifier @return tablespace @retval nullptr if the tablespace is missing or inaccessible */ - static fil_space_t *get(ulint id); + static fil_space_t *get(uint32_t id); /** Add/remove the free page in the freed ranges list. @param[in] offset page number to be added @@ -1415,14 +1365,12 @@ or the caller should be in single-threaded crash recovery mode Normally, fil_space_t::get() should be used instead. @param[in] id tablespace ID @return tablespace, or NULL if not found */ -fil_space_t* -fil_space_get( - ulint id) - MY_ATTRIBUTE((warn_unused_result)); +fil_space_t *fil_space_get(uint32_t id) + MY_ATTRIBUTE((warn_unused_result)); -/** The tablespace memory cache; also the totality of logs (the log -data space) is stored here; below we talk about tablespaces */ -struct fil_system_t { +/** The tablespace memory cache */ +struct fil_system_t +{ /** Constructor. @@ -1481,12 +1429,8 @@ public: ulint n_open; /** last time we noted n_open exceeding the limit; protected by mutex */ time_t n_open_exceeded_time; - ulint max_assigned_id;/*!< maximum space id in the existing - tables, or assigned during the time - mysqld has been up; at an InnoDB - startup we scan the data dictionary - and set here the maximum of the - space id's of the tables there */ + /** maximum persistent tablespace id that has ever been assigned */ + uint32_t max_assigned_id; /** nonzero if fil_node_open_file_low() should avoid moving the tablespace to the end of space_list, for FIFO policy of try_to_close() */ ulint freeze_space_list; @@ -1499,13 +1443,13 @@ public: the latest redo log checkpoint. Protected only by log_sys.mutex. */ - /** List of all file spaces need key rotation */ - ilist<fil_space_t, rotation_list_tag_t> default_encrypt_tables; + /** list of all ENCRYPTED=DEFAULT tablespaces that need + to be converted to the current value of innodb_encrypt_tables */ + ilist<fil_space_t, default_encrypt_tag_t> default_encrypt_tables; - bool space_id_reuse_warned; - /*!< whether fil_space_t::create() - has issued a warning about - potential space_id reuse */ + /** whether fil_space_t::create() has issued a warning about + potential space_id reuse */ + bool space_id_reuse_warned; /** Return the next tablespace from default_encrypt_tables list. @param space previous tablespace (nullptr to start from the start) @@ -1627,36 +1571,27 @@ Assigns a new space id for a new single-table tablespace. This works simply by incrementing the global counter. If 4 billion id's is not enough, we may need to recycle id's. @return true if assigned, false if not */ -bool -fil_assign_new_space_id( -/*====================*/ - ulint* space_id); /*!< in/out: space id */ +bool fil_assign_new_space_id(uint32_t *space_id); /** Frees a space object from the tablespace memory cache. Closes the files in the chain but does not delete them. There must not be any pending i/o's or flushes on the files. -@param[in] id tablespace identifier -@param[in] x_latched whether the caller holds X-mode space->latch +@param id tablespace identifier +@param x_latched whether the caller holds exclusive fil_space_t::latch @return true if success */ -bool -fil_space_free( - ulint id, - bool x_latched); +bool fil_space_free(uint32_t id, bool x_latched); /** Set the recovered size of a tablespace in pages. @param id tablespace ID @param size recovered size in pages @param flags tablespace flags */ -void fil_space_set_recv_size_and_flags(ulint id, uint32_t size, +void fil_space_set_recv_size_and_flags(uint32_t id, uint32_t size, uint32_t flags); /*******************************************************************//** Sets the max tablespace id counter if the given number is bigger than the previous value. */ -void -fil_set_max_space_id_if_bigger( -/*===========================*/ - ulint max_id);/*!< in: maximum known id */ +void fil_set_max_space_id_if_bigger(uint32_t max_id); /** Write the flushed LSN to the page header of the first page in the system tablespace. @@ -1672,12 +1607,12 @@ MY_ATTRIBUTE((warn_unused_result)) @param id tablespace identifier @return detached file handle (to be closed by the caller) @return OS_FILE_CLOSED if no file existed */ -pfs_os_file_t fil_delete_tablespace(ulint id); +pfs_os_file_t fil_delete_tablespace(uint32_t id); /** Close a single-table tablespace on failed IMPORT TABLESPACE. The tablespace must be cached in the memory cache. Free all pages used by the tablespace. */ -void fil_close_tablespace(ulint id); +void fil_close_tablespace(uint32_t id); /*******************************************************************//** Allocates and builds a file name from a path, a table or tablespace name @@ -1707,10 +1642,10 @@ must be >= FIL_IBD_FILE_INITIAL_SIZE @retval NULL on error */ fil_space_t* fil_ibd_create( - ulint space_id, + uint32_t space_id, const table_name_t name, const char* path, - ulint flags, + uint32_t flags, uint32_t size, fil_encryption_t mode, uint32_t key_id, @@ -1721,7 +1656,7 @@ fil_ibd_create( (Typically when upgrading from MariaDB 10.1.0..10.1.20.) @param[in,out] space tablespace @param[in] flags desired tablespace flags */ -void fsp_flags_try_adjust(fil_space_t* space, ulint flags); +void fsp_flags_try_adjust(fil_space_t *space, uint32_t flags); /********************************************************************//** Tries to open a single-table tablespace and optionally checks the space id is @@ -1757,8 +1692,8 @@ fil_space_t* fil_ibd_open( bool validate, fil_type_t purpose, - ulint id, - ulint flags, + uint32_t id, + uint32_t flags, fil_space_t::name_type name, const char* path_in, dberr_t* err = NULL) @@ -1783,12 +1718,8 @@ enum fil_load_status { @param[out] space the tablespace, or NULL on error @return status of the operation */ enum fil_load_status -fil_ibd_load( - ulint space_id, - const char* filename, - fil_space_t*& space) - MY_ATTRIBUTE((warn_unused_result)); - +fil_ibd_load(uint32_t space_id, const char *filename, fil_space_t *&space) + MY_ATTRIBUTE((warn_unused_result)); /** Determine if a matching tablespace exists in the InnoDB tablespace memory cache. Note that if we have not done a crash recovery at the database @@ -1797,7 +1728,8 @@ startup, there may be many tablespaces which are not yet in the memory cache. @param[in] table_flags table flags @return the tablespace @retval NULL if no matching tablespace exists in the memory cache */ -fil_space_t *fil_space_for_table_exists_in_mem(ulint id, ulint table_flags); +fil_space_t *fil_space_for_table_exists_in_mem(uint32_t id, + uint32_t table_flags); /** Try to extend a tablespace if it is smaller than the specified size. @param[in,out] space tablespace @@ -1828,12 +1760,11 @@ fil_delete_file( /*============*/ const char* path); /*!< in: filepath of the ibd tablespace */ -/*******************************************************************//** -Returns the table space by a given id, NULL if not found. */ -fil_space_t* -fil_space_get_by_id( -/*================*/ - ulint id); /*!< in: space id */ +/** Look up a tablespace. +@param tablespace identifier +@return tablespace +@retval nullptr if not found */ +fil_space_t *fil_space_get_by_id(uint32_t id); /** Note that a non-predefined persistent tablespace has been modified by redo log. @@ -1872,6 +1803,9 @@ inline bool fil_names_write_if_was_clean(fil_space_t* space) return(was_clean); } + +bool fil_comp_algo_loaded(ulint comp_algo); + /** On a log checkpoint, reset fil_names_dirty_and_write() flags and write out FILE_MODIFY and FILE_CHECKPOINT if needed. @param[in] lsn checkpoint LSN diff --git a/storage/innobase/include/fil0pagecompress.h b/storage/innobase/include/fil0pagecompress.h index a22867ad56a..2927da3c2fb 100644 --- a/storage/innobase/include/fil0pagecompress.h +++ b/storage/innobase/include/fil0pagecompress.h @@ -40,7 +40,7 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com ulint fil_page_compress( const byte* buf, byte* out_buf, - ulint flags, + uint32_t flags, ulint block_size, bool encrypted) MY_ATTRIBUTE((nonnull, warn_unused_result)); @@ -52,9 +52,6 @@ ulint fil_page_compress( @return size of the compressed data @retval 0 if decompression failed @retval srv_page_size if the page was not compressed */ -ulint fil_page_decompress( - byte* tmp_buf, - byte* buf, - ulint flags) - MY_ATTRIBUTE((nonnull, warn_unused_result)); +ulint fil_page_decompress(byte *tmp_buf, byte *buf, uint32_t flags) + MY_ATTRIBUTE((nonnull, warn_unused_result)); #endif diff --git a/storage/innobase/include/fsp0file.h b/storage/innobase/include/fsp0file.h index 8c11d61c5aa..9dfb3cc7561 100644 --- a/storage/innobase/include/fsp0file.h +++ b/storage/innobase/include/fsp0file.h @@ -56,7 +56,7 @@ public: m_size(), m_order(), m_type(SRV_NOT_RAW), - m_space_id(ULINT_UNDEFINED), + m_space_id(UINT32_MAX), m_flags(), m_exists(), m_is_valid(), @@ -67,7 +67,7 @@ public: /* No op */ } - Datafile(ulint flags, uint32_t size, ulint order) + Datafile(uint32_t flags, uint32_t size, ulint order) : m_filepath(), m_filename(), @@ -76,7 +76,7 @@ public: m_size(size), m_order(order), m_type(SRV_NOT_RAW), - m_space_id(ULINT_UNDEFINED), + m_space_id(UINT32_MAX), m_flags(flags), m_exists(), m_is_valid(), @@ -154,7 +154,7 @@ public: } /** Initialize the tablespace flags */ - void init(ulint flags) { m_flags= flags; } + void init(uint32_t flags) { m_flags= flags; } /** Release the resources. */ virtual void shutdown(); @@ -198,7 +198,7 @@ public: @param[in] flags The expected tablespace flags. @retval DB_SUCCESS if tablespace is valid, DB_ERROR if not. m_is_valid is also set true on success, else false. */ - dberr_t validate_to_dd(ulint space_id, ulint flags) + dberr_t validate_to_dd(uint32_t space_id, uint32_t flags) MY_ATTRIBUTE((warn_unused_result)); /** Validates this datafile for the purpose of recovery. @@ -254,24 +254,15 @@ public: /** Get Datafile::m_space_id. @return m_space_id */ - ulint space_id() const - { - return(m_space_id); - } + uint32_t space_id() const { return m_space_id; } /** Get Datafile::m_flags. @return m_flags */ - ulint flags() const - { - return(m_flags); - } + uint32_t flags() const { return m_flags; } /** @return true if m_handle is open, false if not */ - bool is_open() const - { - return(m_handle != OS_FILE_CLOSED); - } + bool is_open() const { return m_handle != OS_FILE_CLOSED; } /** Get Datafile::m_is_valid. @return m_is_valid */ @@ -324,9 +315,9 @@ public: @return the first data page */ const byte* get_first_page() const { return(m_first_page); } - void set_space_id(ulint space_id) { m_space_id= space_id; } + void set_space_id(uint32_t space_id) { m_space_id= space_id; } - void set_flags(ulint flags) { m_flags = flags; } + void set_flags(uint32_t flags) { m_flags = flags; } private: /** Free the filepath buffer. */ void free_filepath(); @@ -425,12 +416,12 @@ private: /** Tablespace ID. Contained in the datafile header. If this is a system tablespace, FSP_SPACE_ID is only valid in the first datafile. */ - ulint m_space_id; + uint32_t m_space_id; /** Tablespace flags. Contained in the datafile header. If this is a system tablespace, FSP_SPACE_FLAGS are only valid in the first datafile. */ - ulint m_flags; + uint32_t m_flags; /** true if file already existed on startup */ bool m_exists; diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index a519fa096b1..80ce122c603 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -595,14 +595,11 @@ fseg_print( /** Convert FSP_SPACE_FLAGS from the buggy MariaDB 10.1.0..10.1.20 format. @param[in] flags the contents of FSP_SPACE_FLAGS @return the flags corrected from the buggy MariaDB 10.1 format -@retval ULINT_UNDEFINED if the flags are not in the buggy 10.1 format */ +@retval UINT32_MAX if the flags are not in the buggy 10.1 format */ MY_ATTRIBUTE((warn_unused_result, const)) -UNIV_INLINE -ulint -fsp_flags_convert_from_101(ulint flags) +inline uint32_t fsp_flags_convert_from_101(uint32_t flags) { - DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", - return(ULINT_UNDEFINED);); + DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", return UINT32_MAX;); if (flags == 0 || fil_space_t::full_crc32(flags)) { return(flags); } @@ -611,7 +608,7 @@ fsp_flags_convert_from_101(ulint flags) /* The most significant FSP_SPACE_FLAGS bit that was ever set by MariaDB 10.1.0 to 10.1.20 was bit 17 (misplaced DATA_DIR flag). The flags must be less than 1<<18 in order to be valid. */ - return(ULINT_UNDEFINED); + return UINT32_MAX; } if ((flags & (FSP_FLAGS_MASK_POST_ANTELOPE | FSP_FLAGS_MASK_ATOMIC_BLOBS)) @@ -620,7 +617,7 @@ fsp_flags_convert_from_101(ulint flags) ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED) flag is set, then the "post Antelope" (ROW_FORMAT!=REDUNDANT) flag must also be set. */ - return(ULINT_UNDEFINED); + return UINT32_MAX; } /* Bits 6..10 denote compression in MariaDB 10.1.0 to 10.1.20. @@ -649,19 +646,19 @@ fsp_flags_convert_from_101(ulint flags) invalid (COMPRESSION_LEVEL=3 but COMPRESSION=0) +0b00000: innodb_page_size=16k (looks like COMPRESSION=0) ??? Could actually be compressed; see PAGE_SSIZE below */ - const ulint level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL_MARIADB101( + const uint32_t level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL_MARIADB101( flags); if (FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) != (level != 0) || level > 9) { /* The compression flags are not in the buggy MariaDB 10.1 format. */ - return(ULINT_UNDEFINED); + return UINT32_MAX; } if (!(~flags & FSP_FLAGS_MASK_ATOMIC_WRITES_MARIADB101)) { /* The ATOMIC_WRITES flags cannot be 0b11. (The bits 11..12 should actually never be 0b11, because in MySQL they would be SHARED|TEMPORARY.) */ - return(ULINT_UNDEFINED); + return UINT32_MAX; } /* Bits 13..16 are the wrong position for PAGE_SSIZE, and they @@ -676,23 +673,23 @@ fsp_flags_convert_from_101(ulint flags) will be properly rejected by older MariaDB 10.1.x because they would read as PAGE_SSIZE>=8 which is not valid. */ - const ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE_MARIADB101(flags); + const uint32_t ssize = FSP_FLAGS_GET_PAGE_SSIZE_MARIADB101(flags); if (ssize == 1 || ssize == 2 || ssize == 5 || ssize & 8) { /* the page_size is not between 4k and 64k; 16k should be encoded as 0, not 5 */ - return(ULINT_UNDEFINED); + return UINT32_MAX; } - const ulint zssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); + const uint32_t zssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); if (zssize == 0) { /* not ROW_FORMAT=COMPRESSED */ } else if (zssize > (ssize ? ssize : 5)) { /* invalid KEY_BLOCK_SIZE */ - return(ULINT_UNDEFINED); + return UINT32_MAX; } else if (~flags & (FSP_FLAGS_MASK_POST_ANTELOPE | FSP_FLAGS_MASK_ATOMIC_BLOBS)) { /* both these flags should be set for ROW_FORMAT=COMPRESSED */ - return(ULINT_UNDEFINED); + return UINT32_MAX; } flags = ((flags & 0x3f) | ssize << FSP_FLAGS_POS_PAGE_SSIZE @@ -707,19 +704,11 @@ fsp_flags_convert_from_101(ulint flags) @param[in] actual flags read from FSP_SPACE_FLAGS @return whether the flags match */ MY_ATTRIBUTE((warn_unused_result)) -UNIV_INLINE -bool -fsp_flags_match(ulint expected, ulint actual) +inline bool fsp_flags_match(uint32_t expected, uint32_t actual) { - expected &= ~FSP_FLAGS_MEM_MASK; - ut_ad(fil_space_t::is_valid_flags(expected, false)); - - if (actual == expected) { - return(true); - } - - actual = fsp_flags_convert_from_101(actual); - return(actual == expected); + expected&= ~FSP_FLAGS_MEM_MASK; + ut_ad(fil_space_t::is_valid_flags(expected, false)); + return actual == expected || fsp_flags_convert_from_101(actual) == expected; } /** Determine the descriptor index within a descriptor page. diff --git a/storage/innobase/include/fsp0space.h b/storage/innobase/include/fsp0space.h index ed65af52bc8..a2bb46d3125 100644 --- a/storage/innobase/include/fsp0space.h +++ b/storage/innobase/include/fsp0space.h @@ -47,22 +47,13 @@ public: /** Data file iterator */ typedef files_t::const_iterator const_iterator; - Tablespace() - : - m_files(), - m_space_id(ULINT_UNDEFINED), - m_path(), - m_flags(), - m_ignore_read_only(false) - { - /* No op */ - } + Tablespace() {} virtual ~Tablespace() { shutdown(); ut_ad(m_files.empty()); - ut_ad(m_space_id == ULINT_UNDEFINED); + ut_ad(m_space_id == UINT32_MAX); } // Disable copying @@ -104,22 +95,19 @@ public: /** Set the space id of the tablespace @param[in] space_id tablespace ID to set */ - void set_space_id(ulint space_id) + void set_space_id(uint32_t space_id) { - ut_ad(m_space_id == ULINT_UNDEFINED); + ut_ad(m_space_id == UINT32_MAX); m_space_id = space_id; } /** Get the space id of the tablespace @return m_space_id space id of the tablespace */ - ulint space_id() const - { - return(m_space_id); - } + uint32_t space_id() const { return m_space_id; } /** Set the tablespace flags @param[in] fsp_flags tablespace flags */ - void set_flags(ulint fsp_flags) + void set_flags(uint32_t fsp_flags) { ut_ad(fil_space_t::is_valid_flags(fsp_flags, false)); m_flags = fsp_flags; @@ -127,24 +115,15 @@ public: /** Get the tablespace flags @return m_flags tablespace flags */ - ulint flags() const - { - return(m_flags); - } + uint32_t flags() const { return m_flags; } /** Get the tablespace encryption mode @return m_mode tablespace encryption mode */ - fil_encryption_t encryption_mode() const - { - return (m_mode); - } + fil_encryption_t encryption_mode() const { return m_mode; } /** Get the tablespace encryption key_id @return m_key_id tablespace encryption key_id */ - uint32_t key_id() const - { - return (m_key_id); - } + uint32_t key_id() const { return m_key_id; } /** Set Ignore Read Only Status for tablespace. @param[in] read_only_status read only status indicator */ @@ -210,24 +189,21 @@ private: @param[in] file data file object */ void file_found(Datafile& file); - /* DATA MEMBERS */ - /** Tablespace ID */ - ulint m_space_id; + uint32_t m_space_id = UINT32_MAX; + /** Tablespace flags */ + uint32_t m_flags = UINT32_MAX; - /** Path where tablespace files will reside, not including a filename.*/ + /** Path where tablespace files will reside, excluding a filename */ char* m_path; - /** Tablespace flags */ - ulint m_flags; - /** Encryption mode and key_id */ fil_encryption_t m_mode; uint32_t m_key_id; protected: /** Ignore server read only configuration for this tablespace. */ - bool m_ignore_read_only; + bool m_ignore_read_only = false; }; #endif /* fsp0space_h */ diff --git a/storage/innobase/include/fsp0sysspace.h b/storage/innobase/include/fsp0sysspace.h index 2e0a395f71c..b6bdadd3501 100644 --- a/storage/innobase/include/fsp0sysspace.h +++ b/storage/innobase/include/fsp0sysspace.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2013, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -266,24 +267,15 @@ extern SysTablespace srv_tmp_space; /** Check if the space_id is for a system-tablespace (shared + temp). @param[in] id Space ID to check @return true if id is a system tablespace, false if not. */ -UNIV_INLINE -bool -is_system_tablespace(ulint id) +inline bool is_system_tablespace(uint32_t id) { - return(id == TRX_SYS_SPACE || id == SRV_TMP_SPACE_ID); + return id == TRX_SYS_SPACE || id == SRV_TMP_SPACE_ID; } /** Check if predefined shared tablespace. @return true if predefined shared tablespace */ -UNIV_INLINE -bool -is_predefined_tablespace( - ulint id) +inline bool is_predefined_tablespace(uint32_t id) { - ut_ad(srv_sys_space.space_id() == TRX_SYS_SPACE); - ut_ad(TRX_SYS_SPACE == 0); - return(id == TRX_SYS_SPACE - || id == SRV_TMP_SPACE_ID - || srv_is_undo_tablespace(id)); + return is_system_tablespace(id) || srv_is_undo_tablespace(id); } #endif /* fsp0sysspace_h */ diff --git a/storage/innobase/include/fsp0types.h b/storage/innobase/include/fsp0types.h index f8e4c06baae..e062ed189bd 100644 --- a/storage/innobase/include/fsp0types.h +++ b/storage/innobase/include/fsp0types.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2020, MariaDB Corporation. +Copyright (c) 2014, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -25,25 +25,24 @@ Created May 26, 2009 Vasil Dimov *******************************************************/ #pragma once -#include <cstddef> +#include "ut0byte.h" -/** The fil_space_t::id of the redo log. All persistent tablespaces -have a smaller fil_space_t::id. */ -static constexpr size_t SRV_SPACE_ID_UPPER_BOUND= 0xFFFFFFF0; +/** All persistent tablespaces have a smaller fil_space_t::id than this. */ +constexpr uint32_t SRV_SPACE_ID_UPPER_BOUND= 0xFFFFFFF0U; /** The fil_space_t::id of the innodb_temporary tablespace. */ -#define SRV_TMP_SPACE_ID 0xFFFFFFFEU - -#include "ut0byte.h" +constexpr uint32_t SRV_TMP_SPACE_ID= 0xFFFFFFFEU; /* Possible values of innodb_compression_algorithm */ -#define PAGE_UNCOMPRESSED 0 -#define PAGE_ZLIB_ALGORITHM 1 -#define PAGE_LZ4_ALGORITHM 2 -#define PAGE_LZO_ALGORITHM 3 -#define PAGE_LZMA_ALGORITHM 4 +#define PAGE_UNCOMPRESSED 0 +#define PAGE_ZLIB_ALGORITHM 1 +#define PAGE_LZ4_ALGORITHM 2 +#define PAGE_LZO_ALGORITHM 3 +#define PAGE_LZMA_ALGORITHM 4 #define PAGE_BZIP2_ALGORITHM 5 #define PAGE_SNAPPY_ALGORITHM 6 -#define PAGE_ALGORITHM_LAST PAGE_SNAPPY_ALGORITHM +#define PAGE_ALGORITHM_LAST PAGE_SNAPPY_ALGORITHM + +extern const char *page_compression_algorithms[]; /** @name Flags for inserting records in order If records are inserted in order, there are the following diff --git a/storage/innobase/include/fut0fut.h b/storage/innobase/include/fut0fut.h index 58c33c12a29..79d0bc0da0c 100644 --- a/storage/innobase/include/fut0fut.h +++ b/storage/innobase/include/fut0fut.h @@ -42,7 +42,7 @@ bufferfixed and latched */ inline byte* fut_get_ptr( - ulint space, + uint32_t space, ulint zip_size, fil_addr_t addr, rw_lock_type_t rw_latch, diff --git a/storage/innobase/include/ibuf0ibuf.h b/storage/innobase/include/ibuf0ibuf.h index b1abb90ff8e..0bc36029d2f 100644 --- a/storage/innobase/include/ibuf0ibuf.h +++ b/storage/innobase/include/ibuf0ibuf.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, 2020, MariaDB Corporation. +Copyright (c) 2016, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -334,7 +334,7 @@ void ibuf_merge_or_delete_for_page(buf_block_t *block, const page_id_t page_id, /** Delete all change buffer entries for a tablespace, in DISCARD TABLESPACE, IMPORT TABLESPACE, or crash recovery. @param[in] space missing or to-be-discarded tablespace */ -void ibuf_delete_for_discarded_space(ulint space); +void ibuf_delete_for_discarded_space(uint32_t space); /** Contract the change buffer by reading pages to the buffer pool. @return a lower limit for the combined size in bytes of entries which diff --git a/storage/innobase/include/log0log.inl b/storage/innobase/include/log0log.inl index c29c0bfa55f..73434737925 100644 --- a/storage/innobase/include/log0log.inl +++ b/storage/innobase/include/log0log.inl @@ -184,8 +184,8 @@ log_block_convert_lsn_to_no( lsn_t lsn) /*!< in: lsn of a byte within the block */ { return(((ulint) (lsn / OS_FILE_LOG_BLOCK_SIZE) & - DBUG_EVALUATE_IF("innodb_small_log_block_no_limit", - 0xFUL, 0x3FFFFFFFUL)) + 1); + (DBUG_IF("innodb_small_log_block_no_limit") + ? 0xFUL : 0x3FFFFFFFUL)) + 1); } /** Calculate the CRC-32C checksum of a log block. diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h index 9bca993b0f7..9b4c2488f55 100644 --- a/storage/innobase/include/log0recv.h +++ b/storage/innobase/include/log0recv.h @@ -87,13 +87,13 @@ void recv_sys_justify_left_parsing_buf(); @param[in] len length of name, in bytes @param[in] new_name new file name (NULL if not rename) @param[in] new_len length of new_name, in bytes (0 if NULL) */ -extern void (*log_file_op)(ulint space_id, int type, +extern void (*log_file_op)(uint32_t space_id, int type, const byte* name, ulint len, const byte* new_name, ulint new_len); /** Report an operation which does INIT_PAGE for page0 during backup. @param space_id tablespace identifier */ -extern void (*first_page_init)(ulint space_id); +extern void (*first_page_init)(uint32_t space_id); /** Stored redo log record */ struct log_rec_t diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index 326b21c6494..ec67b9fabe4 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -173,10 +173,10 @@ struct mtr_t { (needed for generating a FILE_MODIFY record) @param[in] space_id user or system tablespace ID @return the tablespace */ - fil_space_t* set_named_space_id(ulint space_id) + fil_space_t* set_named_space_id(uint32_t space_id) { ut_ad(!m_user_space_id); - ut_d(m_user_space_id = static_cast<uint32_t>(space_id)); + ut_d(m_user_space_id = space_id); if (!space_id) { return fil_system.sys_space; } else { @@ -194,7 +194,7 @@ struct mtr_t { void set_named_space(fil_space_t* space) { ut_ad(!m_user_space_id); - ut_d(m_user_space_id = static_cast<uint32_t>(space->id)); + ut_d(m_user_space_id = space->id); if (space->id) { m_user_space = space; } @@ -205,7 +205,7 @@ struct mtr_t { (needed for generating a FILE_MODIFY record) @param[in] space tablespace @return whether the mini-transaction is associated with the space */ - bool is_named_space(ulint space) const; + bool is_named_space(uint32_t space) const; /** Check the tablespace associated with the mini-transaction (needed for generating a FILE_MODIFY record) @param[in] space tablespace @@ -213,10 +213,10 @@ struct mtr_t { bool is_named_space(const fil_space_t* space) const; #endif /* UNIV_DEBUG */ - /** Acquire a tablespace X-latch. - @param[in] space_id tablespace ID - @return the tablespace object (never NULL) */ - fil_space_t* x_lock_space(ulint space_id); + /** Acquire a tablespace X-latch. + @param space_id tablespace ID + @return the tablespace object (never NULL) */ + fil_space_t *x_lock_space(uint32_t space_id); /** Acquire a shared rw-latch. */ void s_lock( @@ -562,7 +562,7 @@ public: @param space_id tablespace identifier @param path file path @param new_path new file path for type=FILE_RENAME */ - inline void log_file_op(mfile_type_t type, ulint space_id, + inline void log_file_op(mfile_type_t type, uint32_t space_id, const char *path, const char *new_path= nullptr); diff --git a/storage/innobase/include/row0merge.h b/storage/innobase/include/row0merge.h index e9e250435f0..64d97e6a777 100644 --- a/storage/innobase/include/row0merge.h +++ b/storage/innobase/include/row0merge.h @@ -266,15 +266,16 @@ row_merge_build_indexes( bool allow_non_null) MY_ATTRIBUTE((warn_unused_result)); -/********************************************************************//** -Write a buffer to a block. */ -void -row_merge_buf_write( -/*================*/ - const row_merge_buf_t* buf, /*!< in: sorted buffer */ - const merge_file_t* of, /*!< in: output file */ - row_merge_block_t* block) /*!< out: buffer for writing to file */ - MY_ATTRIBUTE((nonnull)); +/** Write a buffer to a block. +@param buf sorted buffer +@param block buffer for writing to file +@param blob_file blob file handle for doing bulk insert operation */ +dberr_t row_merge_buf_write(const row_merge_buf_t *buf, +#ifndef DBUG_OFF + const merge_file_t *of, /*!< output file */ +#endif + row_merge_block_t *block, + merge_file_t *blob_file= nullptr); /********************************************************************//** Sort a buffer. */ @@ -409,4 +410,77 @@ row_merge_read_rec( row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space) /*!< in: space id */ MY_ATTRIBUTE((warn_unused_result)); + +/** Buffer for bulk insert */ +class row_merge_bulk_t +{ + /** Buffer for each index in the table. main memory + buffer for sorting the index */ + row_merge_buf_t *m_merge_buf; + /** Block for IO operation */ + row_merge_block_t *m_block= nullptr; + /** File to store the buffer and used for merge sort */ + merge_file_t *m_merge_files= nullptr; + /** Temporary file to be used for merge sort */ + pfs_os_file_t m_tmpfd; + /** Allocate memory for merge file data structure */ + ut_allocator<row_merge_block_t> m_alloc; + /** Storage for description for the m_alloc */ + ut_new_pfx_t m_block_pfx; + /** Temporary file to store the blob */ + merge_file_t m_blob_file; +public: + /** Constructor. + Create all merge files, merge buffer for all the table indexes + expect fts indexes. + Create a merge block which is used to write IO operation + @param table table which undergoes bulk insert operation */ + row_merge_bulk_t(dict_table_t *table); + + /** Destructor. + Remove all merge files, merge buffer for all table indexes. */ + ~row_merge_bulk_t(); + + /** Remove all buffer for the table indexes */ + void remove_all_bulk_buffer(); + + /** Clean the merge buffer for the given index number */ + void clean_bulk_buffer(ulint index_no); + + /** Create the temporary file for the given index number + @retval true if temporary file creation went well */ + bool create_tmp_file(ulint index_no); + + /** Write the merge buffer to the tmp file for the given + index number. + @param index_no buffer to be written for the index */ + dberr_t write_to_tmp_file(ulint index_no); + + /** Add the tuple to the merge buffer for the given index. + If the buffer ran out of memory then write the buffer into + the temporary file and do insert the tuple again. + @param row tuple to be inserted + @param ind index to be buffered + @param trx bulk transaction */ + dberr_t bulk_insert_buffered(const dtuple_t &row, const dict_index_t &ind, + trx_t *trx); + + /** Do bulk insert operation into the index tree from + buffer or merge file if exists + @param index_no index to be inserted + @param trx bulk transaction */ + dberr_t write_to_index(ulint index_no, trx_t *trx); + + /** Do bulk insert for the buffered insert for the table. + @param table table which undergoes for bulk insert operation + @param trx bulk transaction */ + dberr_t write_to_table(dict_table_t *table, trx_t *trx); + + /** Allocate block for writing the buffer into disk */ + dberr_t alloc_block(); + + /** Init temporary files for each index */ + void init_tmp_file(); +}; + #endif /* row0merge.h */ diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 623684823e7..a3ef106b847 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -261,12 +261,12 @@ extern bool srv_was_started; extern char* srv_undo_dir; /** Number of undo tablespaces to use. */ -extern ulong srv_undo_tablespaces; +extern uint srv_undo_tablespaces; /** The number of UNDO tablespaces that are active (hosting some rollback segment). It is quite possible that some of the tablespaces doesn't host any of the rollback-segment based on configuration used. */ -extern ulint srv_undo_tablespaces_active; +extern uint32_t srv_undo_tablespaces_active; /** Maximum size of undo tablespace. */ extern unsigned long long srv_max_undo_log_size; @@ -452,8 +452,6 @@ extern ulint srv_log_writes_and_flush; extern my_bool innodb_evict_tables_on_commit_debug; extern my_bool srv_purge_view_update_only_debug; -/** Value of MySQL global used to disable master thread. */ -extern my_bool srv_master_thread_disabled_debug; /** InnoDB system tablespace to set during recovery */ extern uint srv_sys_space_size_debug; /** whether redo log file has been created at startup */ @@ -679,18 +677,6 @@ void srv_purge_shutdown(); /** Init purge tasks*/ void srv_init_purge_tasks(); -#ifdef UNIV_DEBUG -/** Disables master thread. It's used by: - SET GLOBAL innodb_master_thread_disabled_debug = 1 (0). -@param[in] save immediate result from check function */ -void -srv_master_thread_disabled_debug_update(THD*, st_mysql_sys_var*, void*, - const void* save); - -/** Enable the master thread on shutdown. */ -void srv_master_thread_enable(); -#endif /* UNIV_DEBUG */ - /** Status variables to be passed to MySQL */ struct export_var_t{ #ifdef BTR_CUR_HASH_ADAPT diff --git a/storage/innobase/include/trx0rec.h b/storage/innobase/include/trx0rec.h index 0afae491c78..86f305fe04b 100644 --- a/storage/innobase/include/trx0rec.h +++ b/storage/innobase/include/trx0rec.h @@ -194,7 +194,7 @@ trx_undo_report_row_operation( const rec_offs* offsets, /*!< in: rec_get_offsets(rec) */ roll_ptr_t* roll_ptr) /*!< out: DB_ROLL_PTR to the undo log record */ - MY_ATTRIBUTE((nonnull(1,2,8), warn_unused_result)); + MY_ATTRIBUTE((nonnull(1,2), warn_unused_result)); /** status bit used for trx_undo_prev_version_build() */ diff --git a/storage/innobase/include/trx0rseg.h b/storage/innobase/include/trx0rseg.h index 7de6d475b3a..a52d9779a51 100644 --- a/storage/innobase/include/trx0rseg.h +++ b/storage/innobase/include/trx0rseg.h @@ -40,18 +40,6 @@ UNIV_INLINE buf_block_t* trx_rsegf_get(fil_space_t* space, uint32_t page_no, mtr_t* mtr); -/** Gets a newly created rollback segment header. -@param[in] space space where placed -@param[in] page_no page number of the header -@param[in,out] mtr mini-transaction -@return rollback segment header, page x-latched */ -UNIV_INLINE -buf_block_t* -trx_rsegf_get_new( - ulint space, - uint32_t page_no, - mtr_t* mtr); - /** Create a rollback segment header. @param[in,out] space system, undo, or temporary tablespace @param[in] rseg_id rollback segment identifier diff --git a/storage/innobase/include/trx0rseg.inl b/storage/innobase/include/trx0rseg.inl index 8805a4c8ac3..af63d6acbf2 100644 --- a/storage/innobase/include/trx0rseg.inl +++ b/storage/innobase/include/trx0rseg.inl @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2013, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2020, MariaDB Corporation. +Copyright (c) 2017, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -43,22 +43,3 @@ trx_rsegf_get(fil_space_t* space, uint32_t page_no, mtr_t* mtr) return buf_page_get(page_id_t(space->id, page_no), 0, RW_X_LATCH, mtr); } - -/** Gets a newly created rollback segment header. -@param[in] space space where placed -@param[in] page_no page number of the header -@param[in,out] mtr mini-transaction -@return rollback segment header, page x-latched */ -UNIV_INLINE -buf_block_t* -trx_rsegf_get_new( - ulint space, - uint32_t page_no, - mtr_t* mtr) -{ - ut_ad(space <= srv_undo_tablespaces_active || space == SRV_TMP_SPACE_ID - || !srv_was_started); - ut_ad(space <= TRX_SYS_MAX_UNDO_SPACES || space == SRV_TMP_SPACE_ID); - - return buf_page_get(page_id_t(space, page_no), 0, RW_X_LATCH, mtr); -} diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index 498b6bb4571..24bafa61337 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -36,6 +36,7 @@ Created 3/26/1996 Heikki Tuuri #include "fts0fts.h" #include "read0types.h" #include "ilist.h" +#include "row0merge.h" #include <vector> @@ -435,6 +436,9 @@ class trx_mod_table_time_t /** First modification of a system versioned column (NONE= no versioning, BULK= the table was dropped) */ undo_no_t first_versioned= NONE; + + /** Buffer to store insert opertion */ + row_merge_bulk_t *bulk_store= nullptr; public: /** Constructor @param rows number of modified rows so far */ @@ -468,8 +472,14 @@ public: first_versioned= BULK; } - /** Notify the start of a bulk insert operation */ - void start_bulk_insert() { first|= BULK; } + /** Notify the start of a bulk insert operation + @param table table to do bulk operation */ + void start_bulk_insert(dict_table_t *table) + { + first|= BULK; + if (!table->is_temporary()) + bulk_store= new row_merge_bulk_t(table); + } /** Notify the end of a bulk insert operation */ void end_bulk_insert() { first&= ~BULK; } @@ -489,6 +499,36 @@ public: first_versioned= NONE; return false; } + + /** Add the tuple to the transaction bulk buffer for the given index. + @param entry tuple to be inserted + @param index bulk insert for the index + @param trx transaction */ + dberr_t bulk_insert_buffered(const dtuple_t &entry, + const dict_index_t &index, trx_t *trx) + { + return bulk_store->bulk_insert_buffered(entry, index, trx); + } + + /** Do bulk insert operation present in the buffered operation + @return DB_SUCCESS or error code */ + dberr_t write_bulk(dict_table_t *table, trx_t *trx) + { + if (!bulk_store) + return DB_SUCCESS; + dberr_t err= bulk_store->write_to_table(table, trx); + delete bulk_store; + bulk_store= nullptr; + return err; + } + + /** @return whether the buffer storage exist */ + bool bulk_buffer_exist() + { + if (is_bulk_insert() && bulk_store) + return true; + return false; + } }; /** Collection of persistent tables and their first modification @@ -1065,6 +1105,36 @@ public: return false; } + /** @return logical modification time of a table only + if the table has bulk buffer exist in the transaction */ + trx_mod_table_time_t *check_bulk_buffer(dict_table_t *table) + { + if (UNIV_LIKELY(!bulk_insert)) + return nullptr; + ut_ad(!check_unique_secondary); + ut_ad(!check_foreigns); + auto it= mod_tables.find(table); + if (it == mod_tables.end() || !it->second.bulk_buffer_exist()) + return nullptr; + return &it->second; + } + + /** Do the bulk insert for the buffered insert operation + for the transaction. + @return DB_SUCCESS or error code */ + dberr_t bulk_insert_apply() + { + if (UNIV_LIKELY(!bulk_insert)) + return DB_SUCCESS; + ut_ad(!check_unique_secondary); + ut_ad(!check_foreigns); + for (auto& t : mod_tables) + if (t.second.is_bulk_insert()) + if (dberr_t err= t.second.write_bulk(t.first, this)) + return err; + return DB_SUCCESS; + } + private: /** Assign a rollback segment for modifying temporary tables. @return the assigned rollback segment */ diff --git a/storage/innobase/include/trx0types.h b/storage/innobase/include/trx0types.h index 07c1c6a756b..d6ce902977c 100644 --- a/storage/innobase/include/trx0types.h +++ b/storage/innobase/include/trx0types.h @@ -38,7 +38,7 @@ the terminating NUL character. */ static const ulint TRX_ID_MAX_LEN = 17; /** Space id of the transaction system page (the system tablespace) */ -static const ulint TRX_SYS_SPACE = 0; +static constexpr uint32_t TRX_SYS_SPACE= 0; /** Page number of the transaction system page */ #define TRX_SYS_PAGE_NO FSP_TRX_SYS_PAGE_NO diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index 942dbd93a14..cb34f6c31f6 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -217,36 +217,6 @@ management to ensure correct alignment for doubles etc. */ ======================== */ -#ifdef HAVE_LZO -#define IF_LZO(A,B) A -#else -#define IF_LZO(A,B) B -#endif - -#ifdef HAVE_LZ4 -#define IF_LZ4(A,B) A -#else -#define IF_LZ4(A,B) B -#endif - -#ifdef HAVE_LZMA -#define IF_LZMA(A,B) A -#else -#define IF_LZMA(A,B) B -#endif - -#ifdef HAVE_BZIP2 -#define IF_BZIP2(A,B) A -#else -#define IF_BZIP2(A,B) B -#endif - -#ifdef HAVE_SNAPPY -#define IF_SNAPPY(A,B) A -#else -#define IF_SNAPPY(A,B) B -#endif - #if defined (HAVE_FALLOC_PUNCH_HOLE_AND_KEEP_SIZE) || defined(_WIN32) #define IF_PUNCH_HOLE(A,B) A #else @@ -524,7 +494,7 @@ it is read or written. */ #include "ut0lst.h" #include "ut0ut.h" -extern ulong srv_page_size_shift; +extern uint32_t srv_page_size_shift; extern ulong srv_page_size; /* Dimension of spatial object we support so far. It has its root in diff --git a/storage/innobase/include/ut0pool.h b/storage/innobase/include/ut0pool.h index 41b02e0a985..56ffbee48f3 100644 --- a/storage/innobase/include/ut0pool.h +++ b/storage/innobase/include/ut0pool.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2013, 2014, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2018, 2020, MariaDB Corporation. +Copyright (c) 2018, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -300,12 +300,9 @@ private: pool = UT_NEW_NOKEY(PoolType(m_size)); if (pool != NULL) { - - ut_ad(n_pools <= m_pools.size()); - m_pools.push_back(pool); - ib::info() << "Number of pools: " + ib::info() << "Number of transaction pools: " << m_pools.size(); added = true; diff --git a/storage/innobase/innodb.cmake b/storage/innobase/innodb.cmake index 0a276530637..5196637eed2 100644 --- a/storage/innobase/innodb.cmake +++ b/storage/innobase/innodb.cmake @@ -19,19 +19,9 @@ INCLUDE(CheckFunctionExists) INCLUDE(CheckCSourceCompiles) INCLUDE(CheckCSourceRuns) -INCLUDE(lz4.cmake) -INCLUDE(lzo.cmake) -INCLUDE(lzma.cmake) -INCLUDE(bzip2.cmake) -INCLUDE(snappy.cmake) INCLUDE(numa) INCLUDE(TestBigEndian) -MYSQL_CHECK_LZ4() -MYSQL_CHECK_LZO() -MYSQL_CHECK_LZMA() -MYSQL_CHECK_BZIP2() -MYSQL_CHECK_SNAPPY() MYSQL_CHECK_NUMA() INCLUDE(${MYSQL_CMAKE_SCRIPT_DIR}/compile_flags.cmake) diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 387a2a761ec..349ce4cd7d0 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -992,8 +992,6 @@ ATTRIBUTE_COLD void log_write_checkpoint_info(lsn_t end_lsn) MONITOR_INC(MONITOR_NUM_CHECKPOINT); - DBUG_EXECUTE_IF("crash_after_checkpoint", DBUG_SUICIDE();); - mysql_mutex_unlock(&log_sys.mutex); } @@ -1073,7 +1071,6 @@ ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown() dict_stats_shutdown(); btr_defragment_shutdown(); - ut_d(srv_master_thread_enable()); srv_shutdown_state = SRV_SHUTDOWN_CLEANUP; if (srv_buffer_pool_dump_at_shutdown && diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 2cfd2230608..8af6ab6521d 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -54,6 +54,7 @@ Created 9/20/1997 Heikki Tuuri #include "srv0srv.h" #include "srv0start.h" #include "fil0pagecompress.h" +#include "log.h" /** The recovery system */ recv_sys_t recv_sys; @@ -307,7 +308,8 @@ public: if (undo_append(block, ++l, --rlen) && !srv_force_recovery) { page_corrupted: - ib::error() << "Set innodb_force_recovery=1 to ignore corruption."; + sql_print_error("InnoDB: Set innodb_force_recovery=1" + " to ignore corruption."); recv_sys.set_corrupt_log(); return applied; } @@ -573,10 +575,10 @@ struct file_name_t { /** Map of dirty tablespaces during recovery */ typedef std::map< - ulint, + uint32_t, file_name_t, - std::less<ulint>, - ut_allocator<std::pair<const ulint, file_name_t> > > recv_spaces_t; + std::less<uint32_t>, + ut_allocator<std::pair<const uint32_t, file_name_t> > > recv_spaces_t; static recv_spaces_t recv_spaces; @@ -944,11 +946,11 @@ fail: @param[in] len length of name, in bytes @param[in] new_name new file name (NULL if not rename) @param[in] new_len length of new_name, in bytes (0 if NULL) */ -void (*log_file_op)(ulint space_id, int type, +void (*log_file_op)(uint32_t space_id, int type, const byte* name, ulint len, const byte* new_name, ulint new_len); -void (*first_page_init)(ulint space_id); +void (*first_page_init)(uint32_t space_id); /** Information about initializing page contents during redo log processing. FIXME: Rely on recv_sys.pages! */ @@ -1150,7 +1152,7 @@ inline size_t recv_sys_t::files_size() stored */ static void -fil_name_process(char* name, ulint len, ulint space_id, +fil_name_process(char* name, ulint len, uint32_t space_id, bool deleted, lsn_t lsn, store_t *store) { if (srv_operation == SRV_OPERATION_BACKUP @@ -1217,10 +1219,15 @@ same_space: f.name = fname.name; f.status = file_name_t::NORMAL; } else { - ib::error() << "Tablespace " << space_id - << " has been found in two places: '" - << f.name << "' and '" << name << "'." - " You must delete one of them."; + sql_print_error("InnoDB: Tablespace " UINT32PF + " has been found" + " in two places:" + " '%.*s' and '%s'." + " You must delete" + " one of them.", + space_id, + int(f.name.size()), + f.name.data(), name); recv_sys.set_corrupt_fs(); } break; @@ -1243,10 +1250,12 @@ same_space: Enable some more diagnostics when forcing recovery. */ - ib::info() - << "At LSN: " << recv_sys.recovered_lsn - << ": unable to open file " << name - << " for tablespace " << space_id; + sql_print_information( + "InnoDB: At LSN: " LSN_PF + ": unable to open file %s" + " for tablespace " UINT32PF, + recv_sys.recovered_lsn, + name, space_id); } break; @@ -1262,34 +1271,39 @@ same_space: case FIL_LOAD_INVALID: ut_ad(space == NULL); if (srv_force_recovery == 0) { - ib::warn() << "We do not continue the crash" + sql_print_warning( + "InnoDB: We do not continue the crash" " recovery, because the table may" " become corrupt if we cannot apply" " the log records in the InnoDB log to" " it. To fix the problem and start" - " mysqld:"; - ib::info() << "1) If there is a permission" + " mariadbd:"); + sql_print_information( + "InnoDB: 1) If there is a permission" " problem in the file and mysqld" " cannot open the file, you should" - " modify the permissions."; - ib::info() << "2) If the tablespace is not" + " modify the permissions."); + sql_print_information( + "InnoDB: 2) If the tablespace is not" " needed, or you can restore an older" " version from a backup, then you can" " remove the .ibd file, and use" " --innodb_force_recovery=1 to force" - " startup without this file."; - ib::info() << "3) If the file system or the" + " startup without this file."); + sql_print_information( + "InnoDB: 3) If the file system or the" " disk is broken, and you cannot" " remove the .ibd file, you can set" - " --innodb_force_recovery."; + " --innodb_force_recovery."); recv_sys.set_corrupt_fs(); break; } - ib::info() << "innodb_force_recovery was set to " - << srv_force_recovery << ". Continuing crash" - " recovery even though we cannot access the" - " files for tablespace " << space_id << "."; + sql_print_information( + "InnoDB: innodb_force_recovery was set to %lu." + " Continuing crash recovery even though" + " we cannot access the files for tablespace " + UINT32PF ".", srv_force_recovery, space_id); break; } } @@ -1564,10 +1578,11 @@ fail: } if (recv_sys.report(time(NULL))) { - ib::info() << "Read redo log up to LSN=" << *start_lsn; - service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL, - "Read redo log up to LSN=" LSN_PF, - *start_lsn); + sql_print_information("InnoDB: Read redo log up to LSN=" + LSN_PF, *start_lsn); + service_manager_extend_timeout( + INNODB_EXTEND_TIMEOUT_INTERVAL, + "Read redo log up to LSN=" LSN_PF, *start_lsn); } if (*start_lsn != end_lsn) { @@ -1638,9 +1653,9 @@ static bool redo_file_sizes_are_correct() if (it == paths.end()) return true; - ib::error() << "Log file " << *it << " is of different size " - << get_size(*it) << " bytes than other log files " << size - << " bytes!"; + sql_print_error("InnoDB: Log file %.*s is of different size " UINT64PF + " bytes than other log files " UINT64PF " bytes!", + int(it->size()), it->data(), get_size(*it), size); return false; } @@ -1707,7 +1722,7 @@ ATTRIBUTE_COLD static dberr_t recv_log_recover_pre_10_2() if (!log_crypt_101_read_checkpoint(buf)) { - ib::error() << "Decrypting checkpoint failed"; + sql_print_error("InnoDB: Decrypting checkpoint failed"); continue; } @@ -1729,11 +1744,11 @@ ATTRIBUTE_COLD static dberr_t recv_log_recover_pre_10_2() if (!lsn) { - ib::error() << "Upgrade after a crash is not supported." - " This redo log was created before MariaDB 10.2.2," - " and we did not find a valid checkpoint." - " Please follow the instructions at" - " https://mariadb.com/kb/en/library/upgrading/"; + sql_print_error("InnoDB: Upgrade after a crash is not supported." + " This redo log was created before MariaDB 10.2.2," + " and we did not find a valid checkpoint." + " Please follow the instructions at" + " https://mariadb.com/kb/en/library/upgrading/"); return DB_ERROR; } @@ -1741,7 +1756,7 @@ ATTRIBUTE_COLD static dberr_t recv_log_recover_pre_10_2() log_sys.set_flushed_lsn(lsn); const lsn_t source_offset= log_sys.log.calc_lsn_offset_old(lsn); - static constexpr char NO_UPGRADE_RECOVERY_MSG[]= + static const char NO_UPGRADE_RECOVERY_MSG[]= "Upgrade after a crash is not supported." " This redo log was created before MariaDB 10.2.2"; @@ -1750,7 +1765,8 @@ ATTRIBUTE_COLD static dberr_t recv_log_recover_pre_10_2() if (log_block_calc_checksum_format_0(buf) != log_block_get_checksum(buf) && !log_crypt_101_read_block(buf, lsn)) { - ib::error() << NO_UPGRADE_RECOVERY_MSG << ", and it appears corrupted."; + sql_print_error("InnoDB: %s, and it appears corrupted.", + NO_UPGRADE_RECOVERY_MSG); return DB_CORRUPTION; } @@ -1767,10 +1783,10 @@ ATTRIBUTE_COLD static dberr_t recv_log_recover_pre_10_2() } if (buf[20 + 32 * 9] == 2) - ib::error() << "Cannot decrypt log for upgrading." - " The encrypted log was created before MariaDB 10.2.2."; + sql_print_error("InnoDB: Cannot decrypt log for upgrading." + " The encrypted log was created before MariaDB 10.2.2."); else - ib::error() << NO_UPGRADE_RECOVERY_MSG << "."; + sql_print_error("InnoDB: %s.", NO_UPGRADE_RECOVERY_MSG); return DB_ERROR; } @@ -1816,13 +1832,11 @@ static dberr_t recv_log_recover_10_4() ulint cksum = log_block_get_checksum(buf); if (UNIV_UNLIKELY(crc != cksum)) { - ib::error() << "Invalid log block checksum." - << " block: " - << log_block_get_hdr_no(buf) - << " checkpoint no: " - << log_block_get_checkpoint_no(buf) - << " expected: " << crc - << " found: " << cksum; + sql_print_error("InnoDB: Invalid log block checksum." + " block: " ULINTPF " checkpoint no: " ULINTPF + " expected: " ULINTPF " found: " ULINTPF, + log_block_get_hdr_no(buf), + log_block_get_checkpoint_no(buf), crc, cksum); return DB_CORRUPTION; } @@ -1877,7 +1891,7 @@ recv_find_max_checkpoint(ulint* max_field) : 0; if (log_sys.log.format != log_t::FORMAT_3_23 && !recv_check_log_header_checksum(buf)) { - ib::error() << "Invalid redo log header checksum."; + sql_print_error("InnoDB: Invalid redo log header checksum."); return(DB_CORRUPTION); } @@ -1900,8 +1914,8 @@ recv_find_max_checkpoint(ulint* max_field) case log_t::FORMAT_10_5 | log_t::FORMAT_ENCRYPTED: break; default: - ib::error() << "Unsupported redo log format." - " The redo log was created with " << creator << "."; + sql_print_error("InnoDB: Unsupported redo log format." + " The redo log was created with %s.", creator); return(DB_ERROR); } @@ -1924,8 +1938,8 @@ recv_find_max_checkpoint(ulint* max_field) if (log_sys.is_encrypted() && !log_crypt_read_checkpoint_buf(buf)) { - ib::error() << "Reading checkpoint" - " encryption info failed."; + sql_print_error("InnoDB: Reading checkpoint" + " encryption info failed."); continue; } @@ -1950,14 +1964,14 @@ recv_find_max_checkpoint(ulint* max_field) if (*max_field == 0) { /* Before 10.2.2, we could get here during database - initialization if we created an LOG_FILE_NAME file that + initialization if we created an ib_logfile0 file that was filled with zeroes, and were killed. After 10.2.2, we would reject such a file already earlier, when checking the file header. */ - ib::error() << "No valid checkpoint found" - " (corrupted redo log)." - " You can try --innodb-force-recovery=6" - " as a last resort."; + sql_print_error("InnoDB: No valid checkpoint found" + " (corrupted redo log)." + " You can try --innodb-force-recovery=6" + " as a last resort."); return(DB_ERROR); } @@ -1967,11 +1981,12 @@ recv_find_max_checkpoint(ulint* max_field) break; default: if (dberr_t err = recv_log_recover_10_4()) { - ib::error() - << "Upgrade after a crash is not supported." - " The redo log was created with " << creator - << (err == DB_ERROR - ? "." : ", and it appears corrupted."); + sql_print_error("InnoDB: Upgrade after a crash" + " is not supported." + " The redo log was created with %s%s.", + creator, + (err == DB_ERROR + ? "" : ", and it appears corrupted")); return err; } } @@ -2158,17 +2173,19 @@ loop: goto eom_found; if (UNIV_LIKELY((*l & 0x70) != RESERVED)); else if (srv_force_recovery) - ib::warn() << "Ignoring unknown log record at LSN " << recovered_lsn; + sql_print_warning("InnoDB: Ignoring unknown log record at LSN " LSN_PF, + recovered_lsn); else { malformed: - ib::error() << "Malformed log record;" - " set innodb_force_recovery=1 to ignore."; + sql_print_error("InnoDB: Malformed log record;" + " set innodb_force_recovery=1 to ignore."); corrupted: const size_t trailing_bytes= std::min<size_t>(100, size_t(end - l)); - ib::info() << "Dump from the start of the mini-transaction (LSN=" - << start_lsn << ") to " - << trailing_bytes << " bytes after the record:"; + sql_print_information("InnoDB: Dump from the start of the" + " mini-transaction (LSN=" LSN_PF ") to %zu" + " bytes after the record:", + start_lsn, trailing_bytes); ut_print_buf(stderr, log, l - log + trailing_bytes); putc('\n', stderr); found_corrupt_log= true; @@ -2185,7 +2202,7 @@ corrupted: const uint32_t addlen= mlog_decode_varint(l); if (UNIV_UNLIKELY(addlen == MLOG_DECODE_ERROR)) { - ib::error() << "Corrupted record length"; + sql_print_error("InnoDB: Corrupted record length"); goto corrupted; } rlen= addlen + 15; @@ -2249,7 +2266,8 @@ record_corrupted: /* FREE_PAGE,INIT_PAGE cannot be with same_page flag */ if (!srv_force_recovery) goto malformed; - ib::warn() << "Ignoring malformed log record at LSN " << recovered_lsn; + sql_print_warning("InnoDB: Ignoring malformed log record at LSN " + LSN_PF, recovered_lsn); last_offset= 1; /* the next record must not be same_page */ continue; } @@ -2262,12 +2280,13 @@ record_corrupted: page_id_corrupted: if (!srv_force_recovery) { - ib::error() << "Corrupted page identifier at " << recovered_lsn - << "; set innodb_force_recovery=1 to ignore the record."; + sql_print_error("InnoDB: Corrupted page identifier at " LSN_PF + "; set innodb_force_recovery=1 to ignore the record.", + recovered_lsn); goto corrupted; } - ib::warn() << "Ignoring corrupted page identifier at LSN " - << recovered_lsn; + sql_print_warning("InnoDB: Ignoring corrupted page identifier at LSN " + LSN_PF, recovered_lsn); continue; } space_id= mlog_decode_varint(l); @@ -2524,7 +2543,8 @@ same_page: default: if (!srv_force_recovery) goto malformed; - ib::warn() << "Ignoring malformed log record at LSN " << recovered_lsn; + sql_print_warning("InnoDB: Ignoring malformed log record at LSN " + LSN_PF, recovered_lsn); continue; case FILE_DELETE: case FILE_MODIFY: @@ -2534,13 +2554,13 @@ same_page: file_rec_error: if (!srv_force_recovery) { - ib::error() << "Corrupted file-level record;" - " set innodb_force_recovery=1 to ignore."; + sql_print_error("InnoDB: Corrupted file-level record;" + " set innodb_force_recovery=1 to ignore."); goto corrupted; } - ib::warn() << "Ignoring corrupted file-level record at LSN " - << recovered_lsn; + sql_print_warning("InnoDB: Ignoring corrupted file-level record" + " at LSN " LSN_PF, recovered_lsn); continue; } /* fall through */ @@ -2694,12 +2714,13 @@ static void recv_recover_page(buf_block_t* block, mtr_t& mtr, if (skipped_after_init) { skipped_after_init = false; ut_ad(end_lsn == page_lsn); - if (end_lsn != page_lsn) - ib::warn() - << "The last skipped log record LSN " - << end_lsn - << " is not equal to page LSN " - << page_lsn; + if (end_lsn != page_lsn) { + sql_print_warning( + "InnoDB: The last skipped log record" + " LSN " LSN_PF + " is not equal to page LSN " LSN_PF, + end_lsn, page_lsn); + } } end_lsn = l->lsn; @@ -2829,10 +2850,12 @@ set_start_lsn: ut_ad(!recv_sys.pages.empty()); if (recv_sys.report(now)) { - const ulint n = recv_sys.pages.size(); - ib::info() << "To recover: " << n << " pages from log"; - service_manager_extend_timeout( - INNODB_EXTEND_TIMEOUT_INTERVAL, "To recover: " ULINTPF " pages from log", n); + const size_t n = recv_sys.pages.size(); + sql_print_information("InnoDB: To recover: %zu pages from log", + n); + service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL, + "To recover: %zu pages" + " from log", n); } } @@ -2942,7 +2965,7 @@ static void recv_read_in_area(page_id_t page_id, recv_sys_t::map::iterator i) if (p != page_nos) { mysql_mutex_unlock(&recv_sys.mutex); - buf_read_recv_pages(page_id.space(), page_nos, ulint(p - page_nos)); + buf_read_recv_pages(page_id.space(), {page_nos, p}); mysql_mutex_lock(&recv_sys.mutex); } } @@ -3128,8 +3151,8 @@ void recv_sys_t::apply(bool last_batch) const char *msg= last_batch ? "Starting final batch to recover " : "Starting a batch to recover "; - const ulint n= pages.size(); - ib::info() << msg << n << " pages from redo log."; + const size_t n= pages.size(); + sql_print_information("InnoDB: %s %zu pages from redo log.", msg, n); sd_notifyf(0, "STATUS=%s" ULINTPF " pages from redo log", msg, n); apply_log_recs= true; @@ -3241,7 +3264,8 @@ next_free_block: continue; } if (is_corrupt_fs() && !srv_force_recovery) - ib::info() << "Set innodb_force_recovery=1 to ignore corrupted pages."; + sql_print_information("InnoDB: Set innodb_force_recovery=1" + " to ignore corrupted pages."); mysql_mutex_unlock(&mutex); return; } @@ -3486,14 +3510,18 @@ static bool recv_scan_log_recs( recv_needed_recovery = true; if (srv_read_only_mode) { - ib::warn() << "innodb_read_only" - " prevents crash recovery"; + sql_print_warning( + "InnoDB: innodb_read_only" + " prevents crash recovery"); return(true); } - ib::info() << "Starting crash recovery from" - " checkpoint LSN=" << checkpoint_lsn - << "," << recv_sys.scanned_lsn; + sql_print_information("InnoDB: Starting" + " crash recovery from" + " checkpoint LSN=" LSN_PF + "," LSN_PF, + checkpoint_lsn, + recv_sys.scanned_lsn); } /* We were able to find more log data: add it to the @@ -3507,15 +3535,17 @@ static bool recv_scan_log_recs( if (recv_sys.len + 4 * OS_FILE_LOG_BLOCK_SIZE >= recv_parsing_buf_size) { - ib::error() << "Log parsing buffer overflow." - " Recovery may have failed!"; + sql_print_error("InnoDB: Log parsing buffer" + " overflow." + " Recovery may have failed!"); recv_sys.set_corrupt_log(); if (!srv_force_recovery) { - ib::error() - << "Set innodb_force_recovery" - " to ignore this error."; + sql_print_information( + "InnoDB: Set" + " innodb_force_recovery" + " to ignore this error."); return(true); } } else if (!recv_sys.is_corrupt_log()) { @@ -3662,29 +3692,38 @@ recv_init_missing_space(dberr_t err, const recv_spaces_t::const_iterator& i) case SRV_OPERATION_RESTORE: case SRV_OPERATION_RESTORE_EXPORT: if (i->second.name.find("/#sql") != std::string::npos) { - ib::warn() << "Tablespace " << i->first << " was not" - " found at " << i->second.name << " when" - " restoring a (partial?) backup. All redo log" - " for this file will be ignored!"; + sql_print_warning("InnoDB: Tablespace " UINT32PF + " was not found at %.*s when" + " restoring a (partial?) backup." + " All redo log" + " for this file will be ignored!", + i->first, int(i->second.name.size()), + i->second.name.data()); } return(err); } if (srv_force_recovery == 0) { - ib::error() << "Tablespace " << i->first << " was not" - " found at " << i->second.name << "."; + sql_print_error("InnoDB: Tablespace " UINT32PF " was not" + " found at %.*s.", i->first, + int(i->second.name.size()), + i->second.name.data()); if (err == DB_SUCCESS) { - ib::error() << "Set innodb_force_recovery=1 to" + sql_print_information( + "InnoDB: Set innodb_force_recovery=1 to" " ignore this and to permanently lose" - " all changes to the tablespace."; + " all changes to the tablespace."); err = DB_TABLESPACE_NOT_FOUND; } } else { - ib::warn() << "Tablespace " << i->first << " was not" - " found at " << i->second.name << ", and" - " innodb_force_recovery was set. All redo log" - " for this tablespace will be ignored!"; + sql_print_warning("InnoDB: Tablespace " UINT32PF + " was not found at %.*s" + ", and innodb_force_recovery was set." + " All redo log for this tablespace" + " will be ignored!", + i->first, int(i->second.name.size()), + i->second.name.data()); } return(err); @@ -3707,7 +3746,7 @@ recv_validate_tablespace(bool rescan, bool& missing_tablespace) for (recv_sys_t::map::iterator p = recv_sys.pages.begin(); p != recv_sys.pages.end();) { ut_ad(!p->second.log.empty()); - const ulint space = p->first.space(); + const uint32_t space = p->first.space(); if (is_predefined_tablespace(space)) { next: p++; @@ -3759,20 +3798,25 @@ func_exit: missing_tablespace = true; - if (srv_force_recovery > 0) { - ib::warn() << "Tablespace " << rs.first - <<" was not found at " << rs.second.name - <<", and innodb_force_recovery was set." - <<" All redo log for this tablespace" - <<" will be ignored!"; + if (srv_force_recovery) { + sql_print_warning("InnoDB: Tablespace " UINT32PF + " was not found at %.*s," + " and innodb_force_recovery was set." + " All redo log for this tablespace" + " will be ignored!", + rs.first, int(rs.second.name.size()), + rs.second.name.data()); continue; } if (!rescan) { - ib::info() << "Tablespace " << rs.first - << " was not found at '" - << rs.second.name << "', but there" - <<" were no modifications either."; + sql_print_information("InnoDB: Tablespace " UINT32PF + " was not found at '%.*s'," + " but there were" + " no modifications either.", + rs.first, + int(rs.second.name.size()), + rs.second.name.data()); } } @@ -3821,9 +3865,10 @@ recv_init_crash_recovery_spaces(bool rescan, bool& missing_tablespace) std::move(rs.second.freed_ranges)); } } else if (rs.second.name == "") { - ib::error() << "Missing FILE_CREATE, FILE_DELETE" - " or FILE_MODIFY before FILE_CHECKPOINT" - " for tablespace " << rs.first; + sql_print_error("InnoDB: Missing FILE_CREATE," + " FILE_DELETE or FILE_MODIFY" + " before FILE_CHECKPOINT" + " for tablespace " UINT32PF, rs.first); recv_sys.set_corrupt_log(); return(DB_CORRUPTION); } else { @@ -3880,18 +3925,19 @@ static dberr_t recv_rename_files() only be possible if the recovery of both files was deferred (no valid page 0 is contained in either file). We shall not rename the file, just rename the metadata. */ - ib::info() << "Renaming tablespace metadata " << id - << " from '" << old << "' to '" << r.second - << "' that is also associated with tablespace " - << other->id; + sql_print_information("InnoDB: Renaming tablespace metadata " UINT32PF + " from '%s' to '%s' that is also associated" + " with tablespace " UINT32PF, + id, old, new_name, other->id); space->chain.start->name= mem_strdup(new_name); ut_free(old); } else if (!os_file_status(new_name, &exists, &ftype) || exists) { - ib::error() << "Cannot replay rename of tablespace " << id - << " from '" << old << "' to '" << r.second << - (exists ? "' because the target file exists" : "'"); + sql_print_error("InnoDB: Cannot replay rename of tablespace " UINT32PF + " from '%s' to '%s'%s", + id, old, new_name, exists ? + " because the target file exists" : ""); err= DB_TABLESPACE_EXISTS; } else @@ -3899,8 +3945,8 @@ static dberr_t recv_rename_files() mysql_mutex_unlock(&fil_system.mutex); err= space->rename(new_name, false); if (err != DB_SUCCESS) - ib::error() << "Cannot replay rename of tablespace " << id - << " to '" << r.second << "': " << err; + sql_print_error("InnoDB: Cannot replay rename of tablespace " + UINT32PF " to '%s: %s", new_name, ut_strerr(err)); goto done; } mysql_mutex_unlock(&fil_system.mutex); @@ -3941,9 +3987,8 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn) ut_d(mysql_mutex_unlock(&buf_pool.flush_list_mutex)); if (srv_force_recovery >= SRV_FORCE_NO_LOG_REDO) { - - ib::info() << "innodb_force_recovery=6 skips redo log apply"; - + sql_print_information("InnoDB: innodb_force_recovery=6" + " skips redo log apply"); return(DB_SUCCESS); } @@ -3954,7 +3999,6 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn) err = recv_find_max_checkpoint(&max_cp_field); if (err != DB_SUCCESS) { - recv_sys.recovered_lsn = log_sys.get_lsn(); mysql_mutex_unlock(&log_sys.mutex); return(err); @@ -4015,7 +4059,8 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn) if (recv_sys.is_corrupt_log() && !srv_force_recovery) { mysql_mutex_unlock(&log_sys.mutex); - ib::warn() << "Log scan aborted at LSN " << contiguous_lsn; + sql_print_warning("InnoDB: Log scan aborted at LSN " LSN_PF, + contiguous_lsn); return(DB_ERROR); } @@ -4023,13 +4068,11 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn) lsn_t scan_lsn = log_sys.log.scanned_lsn; if (!srv_read_only_mode && scan_lsn != checkpoint_lsn) { mysql_mutex_unlock(&log_sys.mutex); - ib::error err; - err << "Missing FILE_CHECKPOINT"; - if (end_lsn) { - err << " at " << end_lsn; - } - err << " between the checkpoint " << checkpoint_lsn - << " and the end " << scan_lsn << "."; + sql_print_error("InnoDB: Missing FILE_CHECKPOINT" + " at " LSN_PF + " between the checkpoint " LSN_PF + " and the end " LSN_PF ".", + end_lsn, checkpoint_lsn, scan_lsn); return(DB_ERROR); } @@ -4059,29 +4102,26 @@ completed: ut_ad(!srv_log_file_created); if (checkpoint_lsn + sizeof_checkpoint < flush_lsn) { - ib::warn() - << "Are you sure you are using the right " - << LOG_FILE_NAME - << " to start up the database? Log sequence " - "number in the " - << LOG_FILE_NAME << " is " << checkpoint_lsn - << ", less than the log sequence number in " - "the first system tablespace file header, " - << flush_lsn << "."; + sql_print_warning("InnoDB: Are you sure you are using" + " the right ib_logfile0" + " to start up the database?" + " The checkpoint is " LSN_PF + ", less than the" + " log sequence number " LSN_PF + " in the system tablespace.", + checkpoint_lsn, flush_lsn); } if (!recv_needed_recovery) { - - ib::info() - << "The log sequence number " << flush_lsn - << " in the system tablespace does not match" - " the log sequence number " - << checkpoint_lsn << " in the " - << LOG_FILE_NAME << "!"; + sql_print_information( + "InnoDB: The log sequence number " LSN_PF + " in the system tablespace does not match" + " the log checkpoint " LSN_PF + " in ib_logfile0!", flush_lsn, checkpoint_lsn); if (srv_read_only_mode) { - ib::error() << "innodb_read_only" - " prevents crash recovery"; + sql_print_error("InnoDB: innodb_read_only" + " prevents crash recovery"); mysql_mutex_unlock(&log_sys.mutex); return(DB_READ_ONLY); } @@ -4171,21 +4211,20 @@ completed: && (log_sys.log.scanned_lsn < checkpoint_lsn || log_sys.log.scanned_lsn < recv_max_page_lsn)) { - ib::error() << "We scanned the log up to " - << log_sys.log.scanned_lsn - << ". A checkpoint was at " << checkpoint_lsn << " and" - " the maximum LSN on a database page was " - << recv_max_page_lsn << ". It is possible that the" - " database is now corrupt!"; + sql_print_error("InnoDB: We scanned the log up to " LSN_PF "." + " A checkpoint was at " LSN_PF + " and the maximum LSN on a database page was " + LSN_PF ". It is possible that the" + " database is now corrupt!", + log_sys.log.scanned_lsn, checkpoint_lsn, + recv_max_page_lsn); } if (recv_sys.recovered_lsn < checkpoint_lsn) { mysql_mutex_unlock(&log_sys.mutex); - - ib::error() << "Recovered only to lsn:" - << recv_sys.recovered_lsn - << " checkpoint_lsn: " << checkpoint_lsn; - + sql_print_error("InnoDB: Recovered only to lsn: " LSN_PF + " checkpoint_lsn: " LSN_PF, + recv_sys.recovered_lsn, checkpoint_lsn); return(DB_ERROR); } @@ -4246,11 +4285,11 @@ bool recv_dblwr_t::validate_page(const page_id_t page_id, { if (page_id.page_no() == 0) { - ulint flags= fsp_header_get_flags(page); + uint32_t flags= fsp_header_get_flags(page); if (!fil_space_t::is_valid_flags(flags, page_id.space())) { - ulint cflags= fsp_flags_convert_from_101(flags); - if (cflags == ULINT_UNDEFINED) + uint32_t cflags= fsp_flags_convert_from_101(flags); + if (cflags == UINT32_MAX) { ib::warn() << "Ignoring a doublewrite copy of page " << page_id << "due to invalid flags " << ib::hex(flags); diff --git a/storage/innobase/lz4.cmake b/storage/innobase/lz4.cmake deleted file mode 100644 index a908dd3b73e..00000000000 --- a/storage/innobase/lz4.cmake +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (C) 2014, SkySQL Ab. All Rights Reserved. -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free Software -# Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA - -SET(WITH_INNODB_LZ4 AUTO CACHE STRING - "Build with lz4. Possible values are 'ON', 'OFF', 'AUTO' and default is 'AUTO'") - -MACRO (MYSQL_CHECK_LZ4) - IF (WITH_INNODB_LZ4 STREQUAL "ON" OR WITH_INNODB_LZ4 STREQUAL "AUTO") - CHECK_INCLUDE_FILES(lz4.h HAVE_LZ4_H) - CHECK_LIBRARY_EXISTS(lz4 LZ4_compress_limitedOutput "" HAVE_LZ4_SHARED_LIB) - CHECK_LIBRARY_EXISTS(lz4 LZ4_compress_default "" HAVE_LZ4_COMPRESS_DEFAULT) - - IF (HAVE_LZ4_SHARED_LIB AND HAVE_LZ4_H) - SET(HAVE_INNODB_LZ4 TRUE) - ADD_DEFINITIONS(-DHAVE_LZ4=1) - IF (HAVE_LZ4_COMPRESS_DEFAULT) - ADD_DEFINITIONS(-DHAVE_LZ4_COMPRESS_DEFAULT=1) - ENDIF() - LINK_LIBRARIES(lz4) - ELSE() - IF (WITH_INNODB_LZ4 STREQUAL "ON") - MESSAGE(FATAL_ERROR "Required lz4 library is not found") - ENDIF() - ENDIF() - ENDIF() - ADD_FEATURE_INFO(INNODB_LZ4 HAVE_INNODB_LZ4 "LZ4 compression in the InnoDB storage engine") -ENDMACRO() diff --git a/storage/innobase/lzma.cmake b/storage/innobase/lzma.cmake deleted file mode 100644 index 3060139c27c..00000000000 --- a/storage/innobase/lzma.cmake +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (C) 2014, SkySQL Ab. All Rights Reserved. -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free Software -# Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA - -SET(WITH_INNODB_LZMA AUTO CACHE STRING - "Build with lzma. Possible values are 'ON', 'OFF', 'AUTO' and default is 'AUTO'") - -MACRO (MYSQL_CHECK_LZMA) - IF (WITH_INNODB_LZMA STREQUAL "ON" OR WITH_INNODB_LZMA STREQUAL "AUTO") - CHECK_INCLUDE_FILES(lzma.h HAVE_LZMA_H) - CHECK_LIBRARY_EXISTS(lzma lzma_stream_buffer_decode "" HAVE_LZMA_DECODE) - CHECK_LIBRARY_EXISTS(lzma lzma_easy_buffer_encode "" HAVE_LZMA_ENCODE) - - IF (HAVE_LZMA_DECODE AND HAVE_LZMA_ENCODE AND HAVE_LZMA_H) - SET(HAVE_INNODB_LZMA TRUE) - ADD_DEFINITIONS(-DHAVE_LZMA=1) - LINK_LIBRARIES(lzma) - ELSE() - IF (WITH_INNODB_LZMA STREQUAL "ON") - MESSAGE(FATAL_ERROR "Required lzma library is not found") - ENDIF() - ENDIF() - ENDIF() - ADD_FEATURE_INFO(INNODB_LZMA HAVE_INNODB_LZMA "LZMA compression in the InnoDB storage engine") -ENDMACRO() diff --git a/storage/innobase/lzo.cmake b/storage/innobase/lzo.cmake deleted file mode 100644 index ca2de6ab1c5..00000000000 --- a/storage/innobase/lzo.cmake +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (C) 2014, SkySQL Ab. All Rights Reserved. -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free Software -# Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA - -SET(WITH_INNODB_LZO AUTO CACHE STRING - "Build with lzo. Possible values are 'ON', 'OFF', 'AUTO' and default is 'AUTO'") - -MACRO (MYSQL_CHECK_LZO) - IF (WITH_INNODB_LZO STREQUAL "ON" OR WITH_INNODB_LZO STREQUAL "AUTO") - CHECK_INCLUDE_FILES(lzo/lzo1x.h HAVE_LZO_H) - CHECK_LIBRARY_EXISTS(lzo2 lzo1x_1_compress "" HAVE_LZO_SHARED_LIB) - - IF(HAVE_LZO_SHARED_LIB AND HAVE_LZO_H) - SET(HAVE_INNODB_LZO TRUE) - ADD_DEFINITIONS(-DHAVE_LZO=1) - LINK_LIBRARIES(lzo2) - ELSE() - IF (WITH_INNODB_LZO STREQUAL "ON") - MESSAGE(FATAL_ERROR "Required lzo library is not found") - ENDIF() - ENDIF() - ENDIF() - ADD_FEATURE_INFO(INNODB_LZO HAVE_INNODB_LZO "LZO compression in the InnoDB storage engine") -ENDMACRO() diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index fa72d95def3..03f133d9fc3 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -617,7 +617,7 @@ void mtr_t::commit_files(lsn_t checkpoint_lsn) @param[in] space tablespace @return whether the mini-transaction is associated with the space */ bool -mtr_t::is_named_space(ulint space) const +mtr_t::is_named_space(uint32_t space) const { ut_ad(!m_user_space || m_user_space->id != TRX_SYS_SPACE); @@ -657,8 +657,7 @@ bool mtr_t::is_named_space(const fil_space_t* space) const /** Acquire a tablespace X-latch. @param[in] space_id tablespace ID @return the tablespace object (never NULL) */ -fil_space_t* -mtr_t::x_lock_space(ulint space_id) +fil_space_t *mtr_t::x_lock_space(uint32_t space_id) { fil_space_t* space; diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 480cf2f2345..fea1eda17e9 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -4236,14 +4236,14 @@ corrupted: FSP_HEADER_OFFSET + FSP_SPACE_ID + page, 4) ? ULINT_UNDEFINED : mach_read_from_4(FIL_PAGE_SPACE_ID + page); - ulint flags= fsp_header_get_flags(page); + uint32_t flags= fsp_header_get_flags(page); const uint32_t size= fsp_header_get_field(page, FSP_SIZE); const uint32_t free_limit= fsp_header_get_field(page, FSP_FREE_LIMIT); const uint32_t free_len= flst_get_len(FSP_HEADER_OFFSET + FSP_FREE + page); if (!fil_space_t::is_valid_flags(flags, space->id)) { - ulint cflags= fsp_flags_convert_from_101(flags); - if (cflags == ULINT_UNDEFINED) + uint32_t cflags= fsp_flags_convert_from_101(flags); + if (cflags == UINT32_MAX) { invalid: ib::error() << "Expected tablespace flags " @@ -4253,8 +4253,8 @@ invalid: goto corrupted; } - ulint cf= cflags & ~FSP_FLAGS_MEM_MASK; - ulint sf= space->flags & ~FSP_FLAGS_MEM_MASK; + uint32_t cf= cflags & ~FSP_FLAGS_MEM_MASK; + uint32_t sf= space->flags & ~FSP_FLAGS_MEM_MASK; if (!fil_space_t::is_flags_equal(cf, sf) && !fil_space_t::is_flags_equal(sf, cf)) diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index 81cf283c5e0..6bf22efda9a 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -876,7 +876,9 @@ loop: if (t_ctx.rows_added[t_ctx.buf_used] && !processed) { row_merge_buf_sort(buf[t_ctx.buf_used], NULL); row_merge_buf_write(buf[t_ctx.buf_used], +#ifndef DBUG_OFF merge_file[t_ctx.buf_used], +#endif block[t_ctx.buf_used]); if (!row_merge_write(merge_file[t_ctx.buf_used]->fd, @@ -942,8 +944,11 @@ exit: for (i = 0; i < FTS_NUM_AUX_INDEX; i++) { if (t_ctx.rows_added[i]) { row_merge_buf_sort(buf[i], NULL); - row_merge_buf_write( - buf[i], merge_file[i], block[i]); + row_merge_buf_write(buf[i], +#ifndef DBUG_OFF + merge_file[i], +#endif + block[i]); /* Write to temp file, only if records have been flushed to temp file before (offset > 0): diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index 97f16bbce77..303f6265842 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -41,12 +41,8 @@ Created 2012-02-08 by Sunny Bains. #include "fil0pagecompress.h" #include "trx0undo.h" #include "lock0lock.h" -#ifdef HAVE_LZO #include "lzo/lzo1x.h" -#endif -#ifdef HAVE_SNAPPY #include "snappy-c.h" -#endif #include "scope.h" @@ -83,9 +79,9 @@ struct row_index_t { in the exporting server */ byte* m_name; /*!< Index name */ - ulint m_space; /*!< Space where it is placed */ + uint32_t m_space; /*!< Space where it is placed */ - ulint m_page_no; /*!< Root page number */ + uint32_t m_page_no; /*!< Root page number */ ulint m_type; /*!< Index type */ @@ -409,14 +405,14 @@ class AbstractCallback public: /** Constructor @param trx covering transaction */ - AbstractCallback(trx_t* trx, ulint space_id) + AbstractCallback(trx_t* trx, uint32_t space_id) : m_zip_size(0), m_trx(trx), m_space(space_id), m_xdes(), - m_xdes_page_no(ULINT_UNDEFINED), - m_space_flags(ULINT_UNDEFINED) UNIV_NOTHROW { } + m_xdes_page_no(UINT32_MAX), + m_space_flags(UINT32_MAX) UNIV_NOTHROW { } /** Free any extent descriptor instance */ virtual ~AbstractCallback() @@ -439,10 +435,7 @@ public: } /** @return the tablespace flags */ - ulint get_space_flags() const - { - return(m_space_flags); - } + uint32_t get_space_flags() const { return m_space_flags; } /** Set the name of the physical file and the file handle that is used @@ -473,7 +466,7 @@ public: virtual dberr_t operator()(buf_block_t* block) UNIV_NOTHROW = 0; /** @return the tablespace identifier */ - ulint get_space_id() const { return m_space; } + uint32_t get_space_id() const { return m_space; } bool is_interrupted() const { return trx_is_interrupted(m_trx); } @@ -516,7 +509,7 @@ protected: @param page page contents @return DB_SUCCESS or error code. */ dberr_t set_current_xdes( - ulint page_no, + uint32_t page_no, const page_t* page) UNIV_NOTHROW { m_xdes_page_no = page_no; @@ -581,19 +574,16 @@ protected: trx_t* m_trx; /** Space id of the file being iterated over. */ - ulint m_space; - - /** Current size of the space in pages */ - ulint m_size; + uint32_t m_space; /** Current extent descriptor page */ xdes_t* m_xdes; /** Physical page offset in the file of the extent descriptor */ - ulint m_xdes_page_no; + uint32_t m_xdes_page_no; /** Flags value read from the header page */ - ulint m_space_flags; + uint32_t m_space_flags; }; /** Determine the page size to use for traversing the tablespace @@ -609,8 +599,8 @@ AbstractCallback::init( m_space_flags = fsp_header_get_flags(page); if (!fil_space_t::is_valid_flags(m_space_flags, true)) { - ulint cflags = fsp_flags_convert_from_101(m_space_flags); - if (cflags == ULINT_UNDEFINED) { + uint32_t cflags = fsp_flags_convert_from_101(m_space_flags); + if (cflags == UINT32_MAX) { return(DB_CORRUPTION); } m_space_flags = cflags; @@ -639,8 +629,7 @@ AbstractCallback::init( return(DB_CORRUPTION); } - m_size = mach_read_from_4(page + FSP_SIZE); - if (m_space == ULINT_UNDEFINED) { + if (m_space == UINT32_MAX) { m_space = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page); } @@ -674,13 +663,13 @@ struct FetchIndexRootPages : public AbstractCallback { /** Index information gathered from the .ibd file. */ struct Index { - Index(index_id_t id, ulint page_no) + Index(index_id_t id, uint32_t page_no) : m_id(id), m_page_no(page_no) { } index_id_t m_id; /*!< Index id */ - ulint m_page_no; /*!< Root page number */ + uint32_t m_page_no; /*!< Root page number */ }; /** Constructor @@ -688,7 +677,7 @@ struct FetchIndexRootPages : public AbstractCallback { @param table table definition in server .*/ FetchIndexRootPages(const dict_table_t* table, trx_t* trx) : - AbstractCallback(trx, ULINT_UNDEFINED), + AbstractCallback(trx, UINT32_MAX), m_table(table), m_index(0, 0) UNIV_NOTHROW { } /** Destructor */ @@ -733,7 +722,7 @@ dberr_t FetchIndexRootPages::operator()(buf_block_t* block) UNIV_NOTHROW m_index.m_page_no = block->page.id().page_no(); /* Check that the tablespace flags match the table flags. */ - ulint expected = dict_tf_to_fsp_flags(m_table->flags); + const uint32_t expected = dict_tf_to_fsp_flags(m_table->flags); if (!fsp_flags_match(expected, m_space_flags)) { ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, ER_TABLE_SCHEMA_MISMATCH, @@ -850,7 +839,7 @@ public: @param cfg config of table being imported. @param space_id tablespace identifier @param trx transaction covering the import */ - PageConverter(row_import* cfg, ulint space_id, trx_t* trx) + PageConverter(row_import* cfg, uint32_t space_id, trx_t* trx) : AbstractCallback(trx, space_id), m_cfg(cfg), @@ -1422,7 +1411,7 @@ row_import::set_root_by_name() UNIV_NOTHROW /* We've already checked that it exists. */ ut_a(index != 0); - index->page = static_cast<uint32_t>(cfg_index->m_page_no); + index->page = cfg_index->m_page_no; } } @@ -1479,8 +1468,7 @@ row_import::set_root_by_heuristic() UNIV_NOTHROW cfg_index[i].m_srv_index = index; - index->page = static_cast<uint32_t>( - cfg_index[i++].m_page_no); + index->page = cfg_index[i++].m_page_no; } } @@ -3070,8 +3058,8 @@ row_import_read_meta_data( /* decrypt and decompress page if needed */ static dberr_t decrypt_decompress(fil_space_crypt_t *space_crypt, - size_t space_flags, span<byte> page, - size_t space_id, byte *page_compress_buf) + uint32_t space_flags, span<byte> page, + uint32_t space_id, byte *page_compress_buf) { auto *data= page.data(); @@ -3080,8 +3068,8 @@ static dberr_t decrypt_decompress(fil_space_crypt_t *space_crypt, if (!buf_page_verify_crypt_checksum(data, space_flags)) return DB_CORRUPTION; - if (dberr_t err= fil_space_decrypt(space_id, space_crypt, data, - page.size(), space_flags, data)) + if (dberr_t err= fil_space_decrypt(space_id, space_flags, space_crypt, + data, page.size(), data)) return err; } @@ -3114,13 +3102,11 @@ static dberr_t decrypt_decompress(fil_space_crypt_t *space_crypt, static size_t get_buf_size() { - return srv_page_size -#ifdef HAVE_LZO - + LZO1X_1_15_MEM_COMPRESS -#elif defined HAVE_SNAPPY - + snappy_max_compressed_length(srv_page_size) -#endif - ; + return srv_page_size + ( + provider_service_lzo->is_loaded ? LZO1X_1_15_MEM_COMPRESS : + provider_service_snappy->is_loaded ? snappy_max_compressed_length(srv_page_size) : + 0 + ); } /* find, parse instant metadata, performing variaous checks, @@ -3171,7 +3157,7 @@ static dberr_t handle_instant_metadata(dict_table_t *table, if (!fil_space_t::is_valid_flags(space_flags, true)) { auto cflags= fsp_flags_convert_from_101(space_flags); - if (cflags == ULINT_UNDEFINED) + if (cflags == UINT32_MAX) { ib::error() << "Invalid FSP_SPACE_FLAGS=" << ib::hex(space_flags); return DB_CORRUPTION; @@ -3794,12 +3780,8 @@ page_corrupted: if (!buf_page_verify_crypt_checksum(readptr, m_space_flags)) goto page_corrupted; - if (ENCRYPTION_KEY_NOT_ENCRYPTED == - buf_page_get_key_version(readptr, m_space_flags)) - goto page_corrupted; - - if ((err= fil_space_decrypt(get_space_id(), iter.crypt_data, readptr, size, - m_space_flags, readptr))) + if ((err= fil_space_decrypt(get_space_id(), m_space_flags, iter.crypt_data, + readptr, size, readptr))) goto func_exit; } @@ -3842,7 +3824,7 @@ static dberr_t fil_iterate( return DB_OUT_OF_MEMORY; } - ulint actual_space_id = 0; + uint32_t actual_space_id = 0; const bool full_crc32 = fil_space_t::full_crc32( callback.get_space_flags()); @@ -3970,9 +3952,9 @@ page_corrupted: if ((err = fil_space_decrypt( actual_space_id, + callback.get_space_flags(), iter.crypt_data, dst, callback.physical_size(), - callback.get_space_flags(), src))) { goto func_exit; } @@ -4515,11 +4497,9 @@ row_import_for_mysql( we will not be writing any redo log for it before we have invoked fil_space_t::set_imported() to declare it a persistent tablespace. */ - ulint fsp_flags = dict_tf_to_fsp_flags(table->flags); - table->space = fil_ibd_open( true, FIL_TYPE_IMPORT, table->space_id, - fsp_flags, name, filepath, &err); + dict_tf_to_fsp_flags(table->flags), name, filepath, &err); ut_ad((table->space == NULL) == (err != DB_SUCCESS)); DBUG_EXECUTE_IF("ib_import_open_tablespace_failure", diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index bc31a5d1329..b7688c7bfe4 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -2641,15 +2641,20 @@ commit_exit: && !thd_is_slave(trx->mysql_thd) /* FIXME: MDEV-24622 */) { DEBUG_SYNC_C("empty_root_page_insert"); + trx->bulk_insert = true; + if (!index->table->is_temporary()) { err = lock_table(index->table, LOCK_X, thr); if (err != DB_SUCCESS) { trx->error_state = err; + trx->bulk_insert = false; goto commit_exit; } if (index->table->n_rec_locks) { +avoid_bulk: + trx->bulk_insert = false; goto skip_bulk_insert; } @@ -2664,9 +2669,20 @@ commit_exit: #else /* BTR_CUR_HASH_ADAPT */ index->table->bulk_trx_id = trx->id; #endif /* BTR_CUR_HASH_ADAPT */ - } - trx->bulk_insert = true; + /* Write TRX_UNDO_EMPTY undo log and + start buffering the insert operation */ + err = trx_undo_report_row_operation( + thr, index, entry, + nullptr, 0, nullptr, nullptr, + nullptr); + + if (err != DB_SUCCESS) { + goto avoid_bulk; + } + + goto commit_exit; + } } skip_bulk_insert: @@ -3269,7 +3285,7 @@ row_ins_sec_index_entry( bool check_foreign) /*!< in: true if check foreign table is needed, false otherwise */ { - dberr_t err; + dberr_t err = DB_SUCCESS; mem_heap_t* offsets_heap; mem_heap_t* heap; trx_id_t trx_id = 0; @@ -3346,13 +3362,24 @@ row_ins_index_entry( dtuple_t* entry, /*!< in/out: index entry to insert */ que_thr_t* thr) /*!< in: query thread */ { - ut_ad(thr_get_trx(thr)->id || index->table->no_rollback() + trx_t* trx = thr_get_trx(thr); + + ut_ad(trx->id || index->table->no_rollback() || index->table->is_temporary()); DBUG_EXECUTE_IF("row_ins_index_entry_timeout", { DBUG_SET("-d,row_ins_index_entry_timeout"); return(DB_LOCK_WAIT);}); + if (index->is_btree()) { + if (auto t= trx->check_bulk_buffer(index->table)) { + /* MDEV-25036 FIXME: check also foreign key + constraints */ + ut_ad(!trx->check_foreigns); + return t->bulk_insert_buffered(*entry, *index, trx); + } + } + if (index->is_primary()) { return row_ins_clust_index_entry(index, entry, thr, 0); } else { diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index a4209fb1a9c..44ab435c8e8 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2021, MariaDB Corporation. +Copyright (c) 2014, 2022, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -261,9 +261,18 @@ private: @param[in] row_buf row_buf the sorted data tuples, or NULL if fd, block will be used instead @param[in,out] btr_bulk btr bulk instance +@param[in] table_total_rows total rows of old table +@param[in] pct_progress total progress percent untill now +@param[in] pct_cost current progress percent +@param[in] crypt_block buffer for encryption or NULL +@param[in] space space id @param[in,out] stage performance schema accounting object, used by ALTER TABLE. If not NULL stage->begin_phase_insert() will be called initially and then stage->inc() will be called for each record that is processed. +@param[in] blob_file To read big column field data from + the given blob file. It is + applicable only for bulk insert + operation @return DB_SUCCESS or error number */ static MY_ATTRIBUTE((warn_unused_result)) dberr_t @@ -274,14 +283,13 @@ row_merge_insert_index_tuples( row_merge_block_t* block, const row_merge_buf_t* row_buf, BtrBulk* btr_bulk, - const ib_uint64_t table_total_rows, /*!< in: total rows of old table */ - const double pct_progress, /*!< in: total progress - percent until now */ - const double pct_cost, /*!< in: current progress percent - */ - row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space, /*!< in: space id */ - ut_stage_alter_t* stage = NULL); + const ib_uint64_t table_total_rows, + double pct_progress, + double pct_cost, + row_merge_block_t* crypt_block, + ulint space, + ut_stage_alter_t* stage= nullptr, + merge_file_t* blob_file= nullptr); /******************************************************//** Encode an index record. */ @@ -319,35 +327,23 @@ row_merge_buf_encode( *b += size; } -/******************************************************//** -Allocate a sort buffer. -@return own: sort buffer */ static MY_ATTRIBUTE((malloc, nonnull)) row_merge_buf_t* row_merge_buf_create_low( -/*=====================*/ - mem_heap_t* heap, /*!< in: heap where allocated */ - dict_index_t* index, /*!< in: secondary index */ - ulint max_tuples, /*!< in: maximum number of - data tuples */ - ulint buf_size) /*!< in: size of the buffer, - in bytes */ + row_merge_buf_t *buf, mem_heap_t *heap, dict_index_t *index) { - row_merge_buf_t* buf; - - ut_ad(max_tuples > 0); - - ut_ad(max_tuples <= srv_sort_buf_size); - - buf = static_cast<row_merge_buf_t*>(mem_heap_zalloc(heap, buf_size)); - buf->heap = heap; - buf->index = index; - buf->max_tuples = max_tuples; - buf->tuples = static_cast<mtuple_t*>( - ut_malloc_nokey(2 * max_tuples * sizeof *buf->tuples)); - buf->tmp_tuples = buf->tuples + max_tuples; - - return(buf); + ulint max_tuples = srv_sort_buf_size + / std::max<ulint>(1, dict_index_get_min_size(index)); + ut_ad(max_tuples > 0); + ut_ad(max_tuples <= srv_sort_buf_size); + + buf->heap = heap; + buf->index = index; + buf->max_tuples = max_tuples; + buf->tuples = static_cast<mtuple_t*>( + ut_malloc_nokey(2 * max_tuples * sizeof *buf->tuples)); + buf->tmp_tuples = buf->tuples + max_tuples; + return(buf); } /******************************************************//** @@ -359,18 +355,16 @@ row_merge_buf_create( dict_index_t* index) /*!< in: secondary index */ { row_merge_buf_t* buf; - ulint max_tuples; ulint buf_size; mem_heap_t* heap; - max_tuples = srv_sort_buf_size - / std::max<ulint>(1, dict_index_get_min_size(index)); - buf_size = (sizeof *buf); heap = mem_heap_create(buf_size); - buf = row_merge_buf_create_low(heap, index, max_tuples, buf_size); + buf = static_cast<row_merge_buf_t*>( + mem_heap_zalloc(heap, buf_size)); + row_merge_buf_create_low(buf, heap, index); return(buf); } @@ -463,6 +457,68 @@ row_merge_buf_redundant_convert( dfield_set_data(field, buf, len); } +/** Insert the tuple into bulk buffer insert operation +@param buf merge buffer for the index operation +@param table bulk insert operation for the table +@param row tuple to be inserted +@return number of rows inserted */ +static ulint row_merge_bulk_buf_add(row_merge_buf_t* buf, + const dict_table_t &table, + const dtuple_t &row) +{ + if (buf->n_tuples >= buf->max_tuples) + return 0; + + const dict_index_t *index= buf->index; + ulint n_fields= dict_index_get_n_fields(index); + mtuple_t *entry= &buf->tuples[buf->n_tuples]; + ulint data_size= 0; + ulint extra_size= UT_BITS_IN_BYTES(unsigned(index->n_nullable)); + dfield_t *field= entry->fields= static_cast<dfield_t*>( + mem_heap_alloc(buf->heap, n_fields * sizeof *entry->fields)); + const dict_field_t *ifield= dict_index_get_nth_field(index, 0); + + for (ulint i = 0; i < n_fields; i++, field++, ifield++) + { + dfield_copy(field, &row.fields[i]); + ulint len= dfield_get_len(field); + const dict_col_t* const col= ifield->col; + + if (dfield_is_null(field)) + continue; + + ulint fixed_len= ifield->fixed_len; + + if (fixed_len); + else if (len < 128 || (!DATA_BIG_COL(col))) + extra_size++; + else + extra_size += 2; + data_size += len; + } + + /* Add to the total size of the record in row_merge_block_t + the encoded length of extra_size and the extra bytes (extra_size). + See row_merge_buf_write() for the variable-length encoding + of extra_size. */ + data_size += (extra_size + 1) + ((extra_size + 1) >= 0x80); + + /* Reserve bytes for the end marker of row_merge_block_t. */ + if (buf->total_size + data_size >= srv_sort_buf_size) + return 0; + + buf->total_size += data_size; + buf->n_tuples++; + + field= entry->fields; + + do + dfield_dup(field++, buf->heap); + while (--n_fields); + + return 1; +} + /** Insert a data tuple into a sort buffer. @param[in,out] buf sort buffer @param[in] fts_index fts index to be created @@ -854,6 +910,10 @@ row_merge_dup_report( const dfield_t* entry) /*!< in: duplicate index entry */ { if (!dup->n_dup++) { + if (!dup->table) { + /* bulk insert */ + return; + } /* Only report the first duplicate record, but count all duplicate records. */ innobase_fields_to_mysql(dup->table, dup->index, entry); @@ -983,25 +1043,135 @@ row_merge_buf_sort( buf->tuples, buf->tmp_tuples, 0, buf->n_tuples); } -/******************************************************//** -Write a buffer to a block. */ -void -row_merge_buf_write( -/*================*/ - const row_merge_buf_t* buf, /*!< in: sorted buffer */ - const merge_file_t* of UNIV_UNUSED, - /*!< in: output file */ - row_merge_block_t* block) /*!< out: buffer for writing to file */ +/** Write the blob field data to temporary file and fill the offset, +length in the field data +@param field tuple field +@param blob_file file to store the blob data +@param heap heap to store the blob offset and length +@return DB_SUCCESS if successful */ +static dberr_t row_merge_write_blob_to_tmp_file( + dfield_t *field, merge_file_t *blob_file,mem_heap_t **heap) +{ + if (blob_file->fd == OS_FILE_CLOSED) + { + blob_file->fd= row_merge_file_create_low(nullptr); + if (blob_file->fd == OS_FILE_CLOSED) + return DB_OUT_OF_MEMORY; + } + uint64_t val= blob_file->offset; + uint32_t len= field->len; + dberr_t err= os_file_write( + IORequestWrite, "(bulk insert)", blob_file->fd, + field->data, blob_file->offset * srv_page_size, len); + + if (err != DB_SUCCESS) + return err; + + byte *data= static_cast<byte*> + (mem_heap_alloc(*heap, BTR_EXTERN_FIELD_REF_SIZE)); + + /* Write zeroes for first 8 bytes */ + memset(data, 0, 8); + /* Write offset for next 8 bytes */ + mach_write_to_8(data + 8, val); + /* Write length of the blob in 4 bytes */ + mach_write_to_4(data + 16, len); + blob_file->offset+= field->len; + blob_file->n_rec++; + dfield_set_data(field, data, BTR_EXTERN_FIELD_REF_SIZE); + dfield_set_ext(field); + return err; +} + +/** This function is invoked when tuple size is greater than +innodb_sort_buffer_size. Basically it recreates the tuple +by writing the blob field to the temporary file. +@param entry index fields to be encode the blob +@param blob_file file to store the blob data +@param heap heap to store the blob offset and blob length +@return tuple which fits into sort_buffer_size */ +static dtuple_t* row_merge_buf_large_tuple(const dtuple_t &entry, + merge_file_t *blob_file, + mem_heap_t **heap) +{ + if (!*heap) + *heap= mem_heap_create(DTUPLE_EST_ALLOC(entry.n_fields)); + + dtuple_t *tuple= dtuple_copy(&entry, *heap); + for (ulint i= 0; i < tuple->n_fields; i++) + { + dfield_t *field= &tuple->fields[i]; + if (dfield_is_null(field) || field->len <= 2000) + continue; + + dberr_t err= row_merge_write_blob_to_tmp_file(field, blob_file, heap); + if (err != DB_SUCCESS) + return nullptr; + } + + return tuple; +} + + +/** Write the field data whose length is more than 2000 bytes +into blob temporary file and write offset, length into the +tuple field +@param entry index fields to be encode the blob +@param n_fields number of fields in the entry +@param heap heap to store the blob offset and blob length +@param blob_file file to store the blob data */ +static dberr_t row_merge_buf_blob(const mtuple_t *entry, ulint n_fields, + mem_heap_t **heap, merge_file_t *blob_file) +{ + + if (!*heap) + *heap= mem_heap_create(100); + + for (ulint i= 0; i < n_fields; i++) + { + dfield_t *field= &entry->fields[i]; + if (dfield_is_null(field) || field->len <= 2000) + continue; + + dberr_t err= row_merge_write_blob_to_tmp_file(field, blob_file, heap); + if (err != DB_SUCCESS) + return err; + } + + return DB_SUCCESS; +} + +/** Write a buffer to a block. +@param buf sorted buffer +@param block buffer for writing to file +@param blob_file blob file handle for doing bulk insert operation */ +dberr_t row_merge_buf_write(const row_merge_buf_t *buf, +#ifndef DBUG_OFF + const merge_file_t *of, /*!< output file */ +#endif + row_merge_block_t *block, + merge_file_t *blob_file) { const dict_index_t* index = buf->index; ulint n_fields= dict_index_get_n_fields(index); byte* b = &block[0]; + mem_heap_t* blob_heap = nullptr; + dberr_t err = DB_SUCCESS; DBUG_ENTER("row_merge_buf_write"); for (ulint i = 0; i < buf->n_tuples; i++) { const mtuple_t* entry = &buf->tuples[i]; + if (blob_file) { + ut_ad(buf->index->is_primary()); + err = row_merge_buf_blob( + entry, n_fields, &blob_heap, blob_file); + if (err != DB_SUCCESS) { + goto func_exit; + } + } + row_merge_buf_encode(&b, index, entry, n_fields); ut_ad(b < &block[srv_sort_buf_size]); @@ -1014,7 +1184,7 @@ row_merge_buf_write( /* Write an "end-of-chunk" marker. */ ut_a(b < &block[srv_sort_buf_size]); - ut_a(b == &block[0] + buf->total_size); + ut_a(b == &block[0] + buf->total_size || blob_file); *b++ = 0; #ifdef HAVE_valgrind /* The rest of the block is uninitialized. Initialize it @@ -1024,7 +1194,12 @@ row_merge_buf_write( DBUG_LOG("ib_merge_sort", "write " << reinterpret_cast<const void*>(b) << ',' << of->fd << ',' << of->offset << " EOF"); - DBUG_VOID_RETURN; +func_exit: + if (blob_heap) { + mem_heap_free(blob_heap); + } + + DBUG_RETURN(err); } /******************************************************//** @@ -2656,7 +2831,11 @@ write_buffers: ut_ad(file->n_rec > 0); - row_merge_buf_write(buf, file, block); + row_merge_buf_write(buf, +#ifndef DBUG_OFF + file, +#endif + block); if (!row_merge_write( file->fd, file->offset++, @@ -3322,7 +3501,7 @@ row_merge_sort( */ #ifndef UNIV_SOLARIS /* Progress report only for "normal" indexes. */ - if (!(dup->index->type & DICT_FTS)) { + if (dup && !(dup->index->type & DICT_FTS)) { thd_progress_init(trx->mysql_thd, 1); } #endif /* UNIV_SOLARIS */ @@ -3339,7 +3518,7 @@ row_merge_sort( show processlist progress field */ /* Progress report only for "normal" indexes. */ #ifndef UNIV_SOLARIS - if (!(dup->index->type & DICT_FTS)) { + if (dup && !(dup->index->type & DICT_FTS)) { thd_progress_report(trx->mysql_thd, file->offset - num_runs, file->offset); } #endif /* UNIV_SOLARIS */ @@ -3369,7 +3548,7 @@ row_merge_sort( /* Progress report only for "normal" indexes. */ #ifndef UNIV_SOLARIS - if (!(dup->index->type & DICT_FTS)) { + if (dup && !(dup->index->type & DICT_FTS)) { thd_progress_end(trx->mysql_thd); } #endif /* UNIV_SOLARIS */ @@ -3377,6 +3556,39 @@ row_merge_sort( DBUG_RETURN(error); } +/** Copy the blob from the given blob file and store it +in field data for the tuple +@param tuple tuple to be inserted +@param heap heap to allocate the memory for the blob storage +@param blob_file file to handle blob data */ +static dberr_t row_merge_copy_blob_from_file(dtuple_t *tuple, mem_heap_t *heap, + merge_file_t *blob_file) +{ + for (ulint i = 0; i < dtuple_get_n_fields(tuple); i++) + { + dfield_t *field= dtuple_get_nth_field(tuple, i); + const byte *field_data= static_cast<byte*>(dfield_get_data(field)); + ulint field_len= dfield_get_len(field); + if (!dfield_is_ext(field)) + continue; + + ut_a(field_len >= BTR_EXTERN_FIELD_REF_SIZE); + ut_ad(!dfield_is_null(field)); + + ut_ad(mach_read_from_8(field_data) == 0); + uint64_t offset= mach_read_from_8(field_data + 8); + uint32_t len= mach_read_from_4(field_data + 16); + + byte *data= (byte*) mem_heap_alloc(heap, len); + if (dberr_t err= os_file_read(IORequestRead, blob_file->fd, data, + offset, len)) + return err; + dfield_set_data(field, data, len); + } + + return DB_SUCCESS; +} + /** Copy externally stored columns to the data tuple. @param[in] mrec record containing BLOB pointers, or NULL to use tuple instead @@ -3462,18 +3674,6 @@ row_merge_mtuple_to_dtuple( dtuple->n_fields * sizeof *mtuple->fields); } -/** Insert sorted data tuples to the index. -@param[in] index index to be inserted -@param[in] old_table old table -@param[in] fd file descriptor -@param[in,out] block file buffer -@param[in] row_buf row_buf the sorted data tuples, -or NULL if fd, block will be used instead -@param[in,out] btr_bulk btr bulk instance -@param[in,out] stage performance schema accounting object, used by -ALTER TABLE. If not NULL stage->begin_phase_insert() will be called initially -and then stage->inc() will be called for each record that is processed. -@return DB_SUCCESS or error number */ static MY_ATTRIBUTE((warn_unused_result)) dberr_t row_merge_insert_index_tuples( @@ -3483,14 +3683,13 @@ row_merge_insert_index_tuples( row_merge_block_t* block, const row_merge_buf_t* row_buf, BtrBulk* btr_bulk, - const ib_uint64_t table_total_rows, /*!< in: total rows of old table */ - const double pct_progress, /*!< in: total progress - percent until now */ - const double pct_cost, /*!< in: current progress percent - */ - row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ - ulint space, /*!< in: space id */ - ut_stage_alter_t* stage) + const ib_uint64_t table_total_rows, + double pct_progress, + double pct_cost, + row_merge_block_t* crypt_block, + ulint space, + ut_stage_alter_t* stage, + merge_file_t* blob_file) { const byte* b; mem_heap_t* heap; @@ -3601,7 +3800,16 @@ row_merge_insert_index_tuples( } } - if (dict_index_is_clust(index) && dtuple_get_n_ext(dtuple)) { + ut_ad(!dtuple_get_n_ext(dtuple) || index->is_primary()); + + if (!dtuple_get_n_ext(dtuple)) { + } else if (blob_file) { + error = row_merge_copy_blob_from_file( + dtuple, tuple_heap, blob_file); + if (error != DB_SUCCESS) { + break; + } + } else { /* Off-page columns can be fetched safely when concurrent modifications to the table are disabled. (Purge can process delete-marked @@ -3622,7 +3830,8 @@ row_merge_insert_index_tuples( row_log_table_blob_alloc() and row_log_table_blob_free(). */ row_merge_copy_blobs( - mrec, offsets, old_table->space->zip_size(), + mrec, offsets, + old_table->space->zip_size(), dtuple, tuple_heap); } @@ -4803,6 +5012,275 @@ func_exit: } } - DBUG_EXECUTE_IF("ib_index_crash_after_bulk_load", DBUG_SUICIDE();); DBUG_RETURN(error); } + +dberr_t row_merge_bulk_t::alloc_block() +{ + if (m_block) + return DB_SUCCESS; + m_block= m_alloc.allocate_large_dontdump( + 3 * srv_sort_buf_size, &m_block_pfx); + if (m_block == nullptr) + return DB_OUT_OF_MEMORY; + return DB_SUCCESS; +} + +row_merge_bulk_t::row_merge_bulk_t(dict_table_t *table) +{ + ulint n_index= 0; + for (dict_index_t *index= UT_LIST_GET_FIRST(table->indexes); + index; index= UT_LIST_GET_NEXT(indexes, index)) + { + if (!index->is_btree()) + continue; + n_index++; + } + + m_merge_buf= static_cast<row_merge_buf_t*>( + ut_zalloc_nokey(n_index * sizeof *m_merge_buf)); + + ulint i= 0; + for (dict_index_t *index= UT_LIST_GET_FIRST(table->indexes); + index; index= UT_LIST_GET_NEXT(indexes, index)) + { + if (!index->is_btree()) + continue; + + mem_heap_t *heap= mem_heap_create(100); + row_merge_buf_create_low(&m_merge_buf[i], heap, index); + i++; + } + + m_tmpfd= OS_FILE_CLOSED; + m_blob_file.fd= OS_FILE_CLOSED; + m_blob_file.offset= 0; + m_blob_file.n_rec= 0; +} + +row_merge_bulk_t::~row_merge_bulk_t() +{ + ulint i= 0; + dict_table_t *table= m_merge_buf[0].index->table; + for (dict_index_t *index= UT_LIST_GET_FIRST(table->indexes); + index; index= UT_LIST_GET_NEXT(indexes, index)) + { + if (!index->is_btree()) + continue; + row_merge_buf_free(&m_merge_buf[i]); + if (m_merge_files) + row_merge_file_destroy(&m_merge_files[i]); + i++; + } + + row_merge_file_destroy_low(m_tmpfd); + + row_merge_file_destroy(&m_blob_file); + + ut_free(m_merge_buf); + + ut_free(m_merge_files); + + if (m_block) + m_alloc.deallocate_large(m_block, &m_block_pfx); +} + +void row_merge_bulk_t::init_tmp_file() +{ + if (m_merge_files) + return; + + ulint n_index= 0; + dict_table_t *table= m_merge_buf[0].index->table; + for (dict_index_t *index= UT_LIST_GET_FIRST(table->indexes); + index; index= UT_LIST_GET_NEXT(indexes, index)) + { + if (!index->is_btree()) + continue; + n_index++; + } + + m_merge_files= static_cast<merge_file_t*>( + ut_malloc_nokey(n_index * sizeof *m_merge_files)); + + for (ulint i= 0; i < n_index; i++) + { + m_merge_files[i].fd= OS_FILE_CLOSED; + m_merge_files[i].offset= 0; + m_merge_files[i].n_rec= 0; + } +} + +void row_merge_bulk_t::clean_bulk_buffer(ulint index_no) +{ + mem_heap_empty(m_merge_buf[index_no].heap); + m_merge_buf[index_no].total_size = m_merge_buf[index_no].n_tuples = 0; +} + +bool row_merge_bulk_t::create_tmp_file(ulint index_no) +{ + return row_merge_file_create_if_needed( + &m_merge_files[index_no], &m_tmpfd, + m_merge_buf[index_no].n_tuples, NULL); +} + +dberr_t row_merge_bulk_t::write_to_tmp_file(ulint index_no) +{ + if (!create_tmp_file(index_no)) + return DB_OUT_OF_MEMORY; + merge_file_t *file= &m_merge_files[index_no]; + row_merge_buf_t *buf= &m_merge_buf[index_no]; + + alloc_block(); + + if (dberr_t err= row_merge_buf_write(buf, +#ifndef DBUG_OFF + file, +#endif + m_block, + index_no == 0 ? &m_blob_file : nullptr)) + return err; + + if (!row_merge_write(file->fd, file->offset++, + m_block, nullptr, + buf->index->table->space->id)) + return DB_TEMP_FILE_WRITE_FAIL; + MEM_UNDEFINED(&m_block[0], srv_sort_buf_size); + return DB_SUCCESS; +} + +dberr_t row_merge_bulk_t::bulk_insert_buffered(const dtuple_t &row, + const dict_index_t &ind, + trx_t *trx) +{ + dberr_t err= DB_SUCCESS; + ulint i= 0; + mem_heap_t *large_tuple_heap= nullptr; + for (dict_index_t *index= UT_LIST_GET_FIRST(ind.table->indexes); + index; index= UT_LIST_GET_NEXT(indexes, index)) + { + if (!index->is_btree()) + continue; + + if (index != &ind) + { + i++; + continue; + } + row_merge_buf_t *buf= &m_merge_buf[i]; +add_to_buf: + if (row_merge_bulk_buf_add(buf, *ind.table, row)) + { + i++; + goto func_exit; + } + + if (buf->n_tuples == 0) + { + /* Tuple data size is greater than srv_sort_buf_size */ + dtuple_t *big_tuple= row_merge_buf_large_tuple( + row, &m_blob_file, &large_tuple_heap); + if (row_merge_bulk_buf_add(buf, *ind.table, *big_tuple)) + { + i++; + goto func_exit; + } + } + + if (index->is_unique()) + { + row_merge_dup_t dup{index, nullptr, nullptr, 0}; + row_merge_buf_sort(buf, &dup); + if (dup.n_dup) + return DB_DUPLICATE_KEY; + } + else + row_merge_buf_sort(buf, NULL); + init_tmp_file(); + merge_file_t *file= &m_merge_files[i]; + file->n_rec+= buf->n_tuples; + err= write_to_tmp_file(i); + if (err != DB_SUCCESS) + return err; + clean_bulk_buffer(i); + buf= &m_merge_buf[i]; + goto add_to_buf; + } + +func_exit: + if (large_tuple_heap) + mem_heap_free(large_tuple_heap); + return err; +} + +dberr_t row_merge_bulk_t::write_to_index(ulint index_no, trx_t *trx) +{ + dberr_t err= DB_SUCCESS; + row_merge_buf_t buf= m_merge_buf[index_no]; + merge_file_t *file= m_merge_files ? + &m_merge_files[index_no] : nullptr; + dict_index_t *index= buf.index; + dict_table_t *table= index->table; + BtrBulk btr_bulk(index, trx); + row_merge_dup_t dup = {index, nullptr, nullptr, 0}; + + if (buf.n_tuples) + { + if (dict_index_is_unique(index)) + { + row_merge_buf_sort(&buf, &dup); + if (dup.n_dup) + return DB_DUPLICATE_KEY; + } + else row_merge_buf_sort(&buf, NULL); + if (file && file->fd != OS_FILE_CLOSED) + { + file->n_rec+= buf.n_tuples; + err= write_to_tmp_file(index_no); + if (err!= DB_SUCCESS) + goto func_exit; + } + else + { + /* Data got fit in merge buffer. */ + err= row_merge_insert_index_tuples( + index, table, OS_FILE_CLOSED, nullptr, + &buf, &btr_bulk, 0, 0, 0, nullptr, table->space_id, nullptr, + m_blob_file.fd == OS_FILE_CLOSED ? nullptr : &m_blob_file); + goto func_exit; + } + } + + err= row_merge_sort(trx, &dup, file, + m_block, &m_tmpfd, true, 0, 0, + nullptr, table->space_id, nullptr); + if (err != DB_SUCCESS) + goto func_exit; + + err= row_merge_insert_index_tuples( + index, table, file->fd, m_block, nullptr, + &btr_bulk, 0, 0, 0, nullptr, table->space_id, + nullptr, &m_blob_file); + +func_exit: + err= btr_bulk.finish(err); + return err; +} + +dberr_t row_merge_bulk_t::write_to_table(dict_table_t *table, trx_t *trx) +{ + ulint i= 0; + for (dict_index_t *index= UT_LIST_GET_FIRST(table->indexes); + index; index= UT_LIST_GET_NEXT(indexes, index)) + { + if (!index->is_btree()) + continue; + + dberr_t err= write_to_index(i, trx); + if (err != DB_SUCCESS) + return err; + i++; + } + + return DB_SUCCESS; +} diff --git a/storage/innobase/snappy.cmake b/storage/innobase/snappy.cmake deleted file mode 100644 index 3a2d828ee5c..00000000000 --- a/storage/innobase/snappy.cmake +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (C) 2015, MariaDB Corporation. All Rights Reserved. -# -# This program is free software; you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free Software -# Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with -# this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA - -SET(WITH_INNODB_SNAPPY AUTO CACHE STRING - "Build with snappy. Possible values are 'ON', 'OFF', 'AUTO' and default is 'AUTO'") - -MACRO (MYSQL_CHECK_SNAPPY) - IF (WITH_INNODB_SNAPPY STREQUAL "ON" OR WITH_INNODB_SNAPPY STREQUAL "AUTO") - CHECK_INCLUDE_FILES(snappy-c.h HAVE_SNAPPY_H) - CHECK_LIBRARY_EXISTS(snappy snappy_uncompress "" HAVE_SNAPPY_SHARED_LIB) - - IF(HAVE_SNAPPY_SHARED_LIB AND HAVE_SNAPPY_H) - SET(HAVE_INNODB_SNAPPY TRUE) - ADD_DEFINITIONS(-DHAVE_SNAPPY=1) - LINK_LIBRARIES(snappy) - ELSE() - IF (WITH_INNODB_SNAPPY STREQUAL "ON") - MESSAGE(FATAL_ERROR "Required snappy library is not found") - ENDIF() - ENDIF() - ENDIF() - ADD_FEATURE_INFO(INNODB_SNAPPY HAVE_INNODB_SNAPPY "Snappy compression in the InnoDB storage engine") -ENDMACRO() diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 818beb3cc25..fd30c70c313 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -97,15 +97,15 @@ char* srv_data_home; char* srv_undo_dir; /** The number of tablespaces to use for rollback segments. */ -ulong srv_undo_tablespaces; +uint srv_undo_tablespaces; /** The number of UNDO tablespaces that are open and ready to use. */ -ulint srv_undo_tablespaces_open; +uint32_t srv_undo_tablespaces_open; /** The number of UNDO tablespaces that are active (hosting some rollback segment). It is quite possible that some of the tablespaces doesn't host any of the rollback-segment based on configuration used. */ -ulint srv_undo_tablespaces_active; +uint32_t srv_undo_tablespaces_active; /** Rate at which UNDO records should be purged. */ ulong srv_purge_rseg_truncate_frequency; @@ -147,13 +147,6 @@ my_bool srv_use_atomic_writes; /** innodb_compression_algorithm; used with page compression */ ulong innodb_compression_algorithm; -#ifdef UNIV_DEBUG -/** Used by SET GLOBAL innodb_master_thread_disabled_debug = X. */ -my_bool srv_master_thread_disabled_debug; -/** Event used to inform that master thread is disabled. */ -static pthread_cond_t srv_master_thread_disabled_cond; -#endif /* UNIV_DEBUG */ - /*------------------------- LOG FILES ------------------------ */ char* srv_log_group_home_dir; @@ -169,7 +162,7 @@ uint srv_flush_log_at_timeout; /** innodb_page_size */ ulong srv_page_size; /** log2 of innodb_page_size; @see innodb_init_params() */ -ulong srv_page_size_shift; +uint32_t srv_page_size_shift; /** innodb_log_write_ahead_size */ ulong srv_log_write_ahead_size; @@ -662,7 +655,6 @@ static void srv_init() UT_LIST_INIT(srv_sys.tasks, &que_thr_t::queue); need_srv_free = true; - ut_d(pthread_cond_init(&srv_master_thread_disabled_cond, nullptr)); mysql_mutex_init(page_zip_stat_per_index_mutex_key, &page_zip_stat_per_index_mutex, nullptr); @@ -686,8 +678,6 @@ srv_free(void) mysql_mutex_destroy(&page_zip_stat_per_index_mutex); mysql_mutex_destroy(&srv_sys.tasks_mutex); - ut_d(pthread_cond_destroy(&srv_master_thread_disabled_cond)); - trx_i_s_cache_free(trx_i_s_cache); srv_thread_pool_end(); } @@ -1529,48 +1519,6 @@ srv_shutdown_print_master_pending( } } -#ifdef UNIV_DEBUG -/** Waits in loop as long as master thread is disabled (debug) */ -static void srv_master_do_disabled_loop() -{ - if (!srv_master_thread_disabled_debug) - return; - srv_main_thread_op_info = "disabled"; - mysql_mutex_lock(&LOCK_global_system_variables); - while (srv_master_thread_disabled_debug) - my_cond_wait(&srv_master_thread_disabled_cond, - &LOCK_global_system_variables.m_mutex); - mysql_mutex_unlock(&LOCK_global_system_variables); - srv_main_thread_op_info = ""; -} - -/** Disables master thread. It's used by: - SET GLOBAL innodb_master_thread_disabled_debug = 1 (0). -@param[in] save immediate result from check function */ -void -srv_master_thread_disabled_debug_update(THD*, st_mysql_sys_var*, void*, - const void* save) -{ - mysql_mutex_assert_owner(&LOCK_global_system_variables); - const bool disable= *static_cast<const my_bool*>(save); - srv_master_thread_disabled_debug= disable; - if (!disable) - pthread_cond_signal(&srv_master_thread_disabled_cond); -} - -/** Enable the master thread on shutdown. */ -void srv_master_thread_enable() -{ - if (srv_master_thread_disabled_debug) - { - mysql_mutex_lock(&LOCK_global_system_variables); - srv_master_thread_disabled_debug= FALSE; - pthread_cond_signal(&srv_master_thread_disabled_cond); - mysql_mutex_unlock(&LOCK_global_system_variables); - } -} -#endif /* UNIV_DEBUG */ - /** Perform periodic tasks whenever the server is active. @param counter_time microsecond_interval_timer() */ static void srv_master_do_active_tasks(ulonglong counter_time) @@ -1647,7 +1595,6 @@ void srv_master_callback(void*) ut_a(srv_shutdown_state <= SRV_SHUTDOWN_INITIATED); MONITOR_INC(MONITOR_MASTER_THREAD_SLEEP); - ut_d(srv_master_do_disabled_loop()); if (!purge_state.m_running) srv_wake_purge_thread_if_not_active(); ulonglong counter_time= microsecond_interval_timer(); @@ -1726,7 +1673,8 @@ std::mutex purge_thread_count_mtx; void srv_update_purge_thread_count(uint n) { std::lock_guard<std::mutex> lk(purge_thread_count_mtx); - purge_create_background_thds(n); + ut_ad(n > 0); + ut_ad(n <= innodb_purge_threads_MAX); srv_n_purge_threads = n; srv_purge_thread_count_changed = 1; } @@ -1969,7 +1917,7 @@ static void purge_coordinator_callback(void*) void srv_init_purge_tasks() { - purge_create_background_thds(srv_n_purge_threads); + purge_create_background_thds(innodb_purge_threads_MAX); purge_coordinator_timer= srv_thread_pool->create_timer (purge_coordinator_callback, nullptr); } diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 830ccbcaa56..ff7fc3e28db 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -113,7 +113,7 @@ ibool srv_start_raw_disk_in_use; uint srv_n_file_io_threads; /** UNDO tablespaces starts with space id. */ -ulint srv_undo_space_id_start; +uint32_t srv_undo_space_id_start; /** TRUE if the server is being started, before rolling back any incomplete transactions */ @@ -463,7 +463,7 @@ static dberr_t srv_validate_undo_tablespaces() } /** @return the number of active undo tablespaces (except system tablespace) */ -static ulint trx_rseg_get_n_undo_tablespaces() +static uint32_t trx_rseg_get_n_undo_tablespaces() { std::set<uint32_t> space_ids; mtr_t mtr; @@ -475,7 +475,7 @@ static ulint trx_rseg_get_n_undo_tablespaces() if (uint32_t space= trx_sysf_rseg_get_space(sys_header, rseg_id)) space_ids.insert(space); mtr.commit(); - return space_ids.size(); + return static_cast<uint32_t>(space_ids.size()); } /** Open an undo tablespace. @@ -484,11 +484,12 @@ static ulint trx_rseg_get_n_undo_tablespaces() @param[in] i undo tablespace count @return undo tablespace identifier @retval 0 on failure */ -static ulint srv_undo_tablespace_open(bool create, const char* name, ulint i) +static uint32_t srv_undo_tablespace_open(bool create, const char *name, + uint32_t i) { bool success; - ulint space_id= 0; - ulint fsp_flags= 0; + uint32_t space_id= 0; + uint32_t fsp_flags= 0; if (create) { @@ -644,20 +645,21 @@ srv_check_undo_redo_logs_exists() return(DB_SUCCESS); } -static dberr_t srv_all_undo_tablespaces_open(bool create_new_db, ulint n_undo) +static dberr_t srv_all_undo_tablespaces_open(bool create_new_db, + uint32_t n_undo) { /* Open all the undo tablespaces that are currently in use. If we fail to open any of these it is a fatal error. The tablespace ids should be contiguous. It is a fatal error because they are required for recovery and are referenced by the UNDO logs (a.k.a RBS). */ - ulint prev_id= create_new_db ? srv_undo_space_id_start - 1 : 0; + uint32_t prev_id= create_new_db ? srv_undo_space_id_start - 1 : 0; - for (ulint i= 0; i < n_undo; ++i) + for (uint32_t i= 0; i < n_undo; ++i) { char name[OS_FILE_MAX_PATH]; - snprintf(name, sizeof name, "%s/undo%03zu", srv_undo_dir, i + 1); - ulint space_id= srv_undo_tablespace_open(create_new_db, name, i); + snprintf(name, sizeof name, "%s/undo%03u", srv_undo_dir, i + 1); + uint32_t space_id= srv_undo_tablespace_open(create_new_db, name, i); if (!space_id) { if (!create_new_db) @@ -682,11 +684,11 @@ static dberr_t srv_all_undo_tablespaces_open(bool create_new_db, ulint n_undo) not in use and therefore not required by recovery. We only check that there are no gaps. */ - for (ulint i= prev_id + 1; i < srv_undo_space_id_start + TRX_SYS_N_RSEGS; + for (uint32_t i= prev_id + 1; i < srv_undo_space_id_start + TRX_SYS_N_RSEGS; ++i) { char name[OS_FILE_MAX_PATH]; - snprintf(name, sizeof name, "%s/undo%03zu", srv_undo_dir, i); + snprintf(name, sizeof name, "%s/undo%03u", srv_undo_dir, i); if (!srv_undo_tablespace_open(create_new_db, name, i)) break; ++srv_undo_tablespaces_open; @@ -698,8 +700,7 @@ static dberr_t srv_all_undo_tablespaces_open(bool create_new_db, ulint n_undo) /** Open the configured number of dedicated undo tablespaces. @param[in] create_new_db whether the database is being initialized @return DB_SUCCESS or error code */ -dberr_t -srv_undo_tablespaces_init(bool create_new_db) +dberr_t srv_undo_tablespaces_init(bool create_new_db) { srv_undo_tablespaces_open= 0; @@ -735,8 +736,8 @@ srv_undo_tablespaces_init(bool create_new_db) already exist. */ srv_undo_tablespaces_active= srv_undo_tablespaces; - ulint n_undo= (create_new_db || srv_operation == SRV_OPERATION_BACKUP || - srv_operation == SRV_OPERATION_RESTORE_DELTA) + uint32_t n_undo= (create_new_db || srv_operation == SRV_OPERATION_BACKUP || + srv_operation == SRV_OPERATION_RESTORE_DELTA) ? srv_undo_tablespaces : TRX_SYS_N_RSEGS; if (dberr_t err= srv_all_undo_tablespaces_open(create_new_db, n_undo)) @@ -750,7 +751,7 @@ srv_undo_tablespaces_init(bool create_new_db) if (create_new_db) { mtr_t mtr; - for (ulint i= 0; i < srv_undo_tablespaces; ++i) + for (uint32_t i= 0; i < srv_undo_tablespaces; ++i) { mtr.start(); fsp_header_init(fil_space_get(srv_undo_space_id_start + i), @@ -820,7 +821,6 @@ srv_open_tmp_tablespace(bool create_new_db) static void srv_shutdown_threads() { ut_ad(!srv_undo_sources); - ut_d(srv_master_thread_enable()); srv_master_timer.reset(); srv_shutdown_state = SRV_SHUTDOWN_EXIT_THREADS; @@ -1499,8 +1499,9 @@ file_checked: fil_system.space_id_reuse_warned = false; if (!srv_read_only_mode) { - const ulint flags = FSP_FLAGS_PAGE_SSIZE(); - for (ulint id = 0; id <= srv_undo_tablespaces; id++) { + const uint32_t flags = FSP_FLAGS_PAGE_SSIZE(); + for (uint32_t id = 0; id <= srv_undo_tablespaces; + id++) { if (fil_space_t* space = fil_space_get(id)) { fsp_flags_try_adjust(space, flags); } @@ -1899,8 +1900,6 @@ void srv_shutdown_bg_undo_sources() { srv_shutdown_state = SRV_SHUTDOWN_INITIATED; - ut_d(srv_master_thread_enable()); - if (srv_undo_sources) { ut_ad(!srv_read_only_mode); fts_optimize_shutdown(); @@ -2046,6 +2045,10 @@ void innodb_shutdown() } srv_tmp_space.shutdown(); + if (srv_stats.pages_page_compression_error) + ib::warn() << "Page compression errors: " + << srv_stats.pages_page_compression_error; + if (srv_was_started && srv_print_verbose_log) { ib::info() << "Shutdown completed; log sequence number " << srv_shutdown_lsn diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 2564049a2e1..07019b8a1e8 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -594,11 +594,11 @@ TRANSACTIONAL_TARGET static void trx_purge_truncate_history() { const ulint threshold= ulint(srv_max_undo_log_size >> srv_page_size_shift); - for (ulint i= purge_sys.truncate.last + for (uint32_t i= purge_sys.truncate.last ? purge_sys.truncate.last->id - srv_undo_space_id_start : 0, j= i;; ) { - const auto space_id= srv_undo_space_id_start + i; + const uint32_t space_id= srv_undo_space_id_start + i; ut_ad(srv_is_undo_tablespace(space_id)); fil_space_t *space= fil_space_get(space_id); ut_a(UT_LIST_GET_LEN(space->chain) == 1); @@ -999,8 +999,7 @@ trx_purge_get_next_rec( ut_ad(purge_sys.next_stored); ut_ad(purge_sys.tail.trx_no < purge_sys.low_limit_no()); - const ulint space = purge_sys.rseg->space->id; - const uint32_t page_no = purge_sys.page_no; + const page_id_t page_id{purge_sys.rseg->space->id, purge_sys.page_no}; const uint16_t offset = purge_sys.offset; if (offset == 0) { @@ -1018,8 +1017,7 @@ trx_purge_get_next_rec( mtr_start(&mtr); - buf_block_t* undo_page = trx_undo_page_get_s_latched( - page_id_t(space, page_no), &mtr); + buf_block_t* undo_page = trx_undo_page_get_s_latched(page_id, &mtr); buf_block_t* rec2_page = undo_page; const trx_undo_rec_t* rec2 = trx_undo_page_get_next_rec( @@ -1042,8 +1040,7 @@ trx_purge_get_next_rec( mtr_start(&mtr); - undo_page = trx_undo_page_get_s_latched( - page_id_t(space, page_no), &mtr); + undo_page = trx_undo_page_get_s_latched(page_id, &mtr); } else { purge_sys.offset = page_offset(rec2); purge_sys.page_no = rec2_page->page.id().page_no(); diff --git a/storage/innobase/trx/trx0rec.cc b/storage/innobase/trx/trx0rec.cc index 9447461e944..9108e547336 100644 --- a/storage/innobase/trx/trx0rec.cc +++ b/storage/innobase/trx/trx0rec.cc @@ -1955,8 +1955,7 @@ TRANSACTIONAL_TARGET ATTRIBUTE_COLD ATTRIBUTE_NOINLINE /** @return whether the transaction holds an exclusive lock on a table */ static bool trx_has_lock_x(const trx_t &trx, dict_table_t& table) { - if (table.is_temporary()) - return true; + ut_ad(!table.is_temporary()); uint32_t n; @@ -2053,9 +2052,16 @@ trx_undo_report_row_operation( ut_ad(que_node_get_type(thr->run_node) == QUE_NODE_INSERT); ut_ad(trx->bulk_insert); return DB_SUCCESS; - } else if (m.second && trx->bulk_insert - && trx_has_lock_x(*trx, *index->table)) { - m.first->second.start_bulk_insert(); + } else if (!m.second || !trx->bulk_insert) { + bulk = false; + } else if (index->table->is_temporary()) { + } else if (trx_has_lock_x(*trx, *index->table)) { + m.first->second.start_bulk_insert(index->table); + + if (dberr_t err = m.first->second.bulk_insert_buffered( + *clust_entry, *index, trx)) { + return err; + } } else { bulk = false; } diff --git a/storage/innobase/trx/trx0rseg.cc b/storage/innobase/trx/trx0rseg.cc index 23999a4208b..eaf475f6b53 100644 --- a/storage/innobase/trx/trx0rseg.cc +++ b/storage/innobase/trx/trx0rseg.cc @@ -31,6 +31,22 @@ Created 3/26/1996 Heikki Tuuri #include "trx0purge.h" #include "srv0mon.h" +/** Get a newly created rollback segment header. +@param page_id header page location +@param mtr mini-transaction +@return rollback segment header, page x-latched */ +static buf_block_t *trx_rsegf_get_new(page_id_t page_id, mtr_t *mtr) +{ +#ifdef UNIV_DEBUG + if (page_id.space() != SRV_TMP_SPACE_ID) + { + ut_ad(page_id.space() <= srv_undo_tablespaces_active || !srv_was_started); + ut_ad(page_id.space() <= TRX_SYS_MAX_UNDO_SPACES); + } +#endif + return buf_page_get(page_id, 0, RW_X_LATCH, mtr); +} + #ifdef WITH_WSREP #include <mysql/service_wsrep.h> @@ -258,7 +274,8 @@ bool trx_rseg_read_wsrep_checkpoint(XID& xid) } const buf_block_t* rseg_header = trx_rsegf_get_new( - trx_sysf_rseg_get_space(sys, rseg_id), page_no, &mtr); + page_id_t(trx_sysf_rseg_get_space(sys, rseg_id), + page_no), &mtr); if (mach_read_from_4(TRX_RSEG + TRX_RSEG_FORMAT + rseg_header->page.frame)) { @@ -453,7 +470,7 @@ static dberr_t trx_rseg_mem_restore(trx_rseg_t *rseg, trx_id_t &max_trx_id, mtr_t *mtr) { buf_block_t* rseg_hdr = trx_rsegf_get_new( - rseg->space->id, rseg->page_no, mtr); + page_id_t(rseg->space->id, rseg->page_no), mtr); if (!mach_read_from_4(TRX_RSEG + TRX_RSEG_FORMAT + rseg_hdr->page.frame)) { diff --git a/storage/innobase/trx/trx0sys.cc b/storage/innobase/trx/trx0sys.cc index abe97370c03..bb6c9247fc7 100644 --- a/storage/innobase/trx/trx0sys.cc +++ b/storage/innobase/trx/trx0sys.cc @@ -279,7 +279,7 @@ trx_sys_create_sys_pages(void) @param space_id system or undo tablespace id @return pointer to new rollback segment @retval nullptr on failure */ -static trx_rseg_t *trx_rseg_create(ulint space_id) +static trx_rseg_t *trx_rseg_create(uint32_t space_id) { trx_rseg_t *rseg= nullptr; mtr_t mtr; @@ -333,11 +333,11 @@ bool trx_sys_create_rsegs() in the system tablespace. */ ut_a(srv_available_undo_logs > 0); - for (ulint i = 0; srv_available_undo_logs < TRX_SYS_N_RSEGS; + for (uint32_t i = 0; srv_available_undo_logs < TRX_SYS_N_RSEGS; i++, srv_available_undo_logs++) { /* Tablespace 0 is the system tablespace. Dedicated undo log tablespaces start from 1. */ - ulint space = srv_undo_tablespaces > 0 + uint32_t space = srv_undo_tablespaces > 0 ? (i % srv_undo_tablespaces) + srv_undo_space_id_start : TRX_SYS_SPACE; diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 0f57e7e2baf..350551a88b1 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -1634,6 +1634,9 @@ trx_mark_sql_stat_end( } if (trx->is_bulk_insert()) { + /* MDEV-25036 FIXME: we support buffered + insert only for the first insert statement */ + trx->error_state = trx->bulk_insert_apply(); /* Allow a subsequent INSERT into an empty table if !unique_checks && !foreign_key_checks. */ return; @@ -1845,8 +1848,6 @@ trx_prepare( lsn_t lsn = trx_prepare_low(trx); - DBUG_EXECUTE_IF("ib_trx_crash_during_xa_prepare_step", DBUG_SUICIDE();); - ut_a(trx->state == TRX_STATE_ACTIVE); { TMTrxGuard tg{*trx}; diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc index 23b27656dfb..8b41da56e8d 100644 --- a/storage/innobase/trx/trx0undo.cc +++ b/storage/innobase/trx/trx0undo.cc @@ -611,7 +611,7 @@ trx_undo_free_page( undo log page; the caller must have reserved the rollback segment mutex */ { - const ulint space = rseg->space->id; + const uint32_t space = rseg->space->id; ut_a(hdr_page_no != page_no); diff --git a/storage/innobase/ut/ut0ut.cc b/storage/innobase/ut/ut0ut.cc index d49fa9fb16c..d127f2138c9 100644 --- a/storage/innobase/ut/ut0ut.cc +++ b/storage/innobase/ut/ut0ut.cc @@ -447,7 +447,7 @@ ut_strerr( case DB_FTS_TOO_MANY_WORDS_IN_PHRASE: return("Too many words in a FTS phrase or proximity search"); case DB_DECRYPTION_FAILED: - return("Table is encrypted but decrypt failed."); + return("Table is compressed or encrypted but uncompress or decrypt failed."); case DB_IO_PARTIAL_FAILED: return("Partial IO failed"); case DB_FORCED_ABORT: |