diff options
Diffstat (limited to 'storage')
34 files changed, 511 insertions, 708 deletions
diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index 3c79fe6a72d..c7ee80cca5f 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -272,11 +272,6 @@ btr_root_get( And block the segment list access by others.*/ buf_block_t* root = btr_root_block_get(index, RW_SX_LATCH, mtr); - - if (root && root->page.encrypted == true) { - root = NULL; - } - return(root ? buf_block_get_frame(root) : NULL); } @@ -730,159 +725,54 @@ btr_page_free_for_ibuf( mtr)); } -/**************************************************************//** -Frees a file page used in an index tree. Can be used also to (BLOB) -external storage pages. */ -void -btr_page_free_low( -/*==============*/ - dict_index_t* index, /*!< in: index tree */ - buf_block_t* block, /*!< in: block to be freed, x-latched */ - ulint level, /*!< in: page level (ULINT_UNDEFINED=BLOB) */ - bool blob, /*!< in: blob page */ - mtr_t* mtr) /*!< in: mtr */ +/** Free an index page. +@param[in,out] index index tree +@param[in,out] block block to be freed +@param[in,out] mtr mini-transaction +@param[in] blob whether this is freeing a BLOB page */ +void btr_page_free(dict_index_t* index, buf_block_t* block, mtr_t* mtr, + bool blob) { - fseg_header_t* seg_header; - page_t* root; - ut_ad(mtr_memo_contains(mtr, block, MTR_MEMO_PAGE_X_FIX)); +#ifdef BTR_CUR_HASH_ADAPT + ut_ad(!block->index || !blob); + ut_ad(!block->index || page_is_leaf(block->frame)); +#endif + ut_ad(index->table->space_id == block->page.id.space()); + /* The root page is freed by btr_free_root(). */ + ut_ad(block->page.id.page_no() != index->page); + ut_ad(mtr->is_named_space(index->table->space)); + /* The page gets invalid for optimistic searches: increment the frame modify clock */ buf_block_modify_clock_inc(block); - if (blob) { - ut_a(level == 0); - } - - bool scrub = srv_immediate_scrub_data_uncompressed; - /* scrub page */ - if (scrub && blob) { - /* blob page: scrub entire page */ - // TODO(jonaso): scrub only what is actually needed - page_t* page = buf_block_get_frame(block); - memset(page + PAGE_HEADER, 0, - srv_page_size - PAGE_HEADER); -#ifdef UNIV_DEBUG_SCRUBBING - fprintf(stderr, - "btr_page_free_low: scrub blob page %lu/%lu\n", - buf_block_get_space(block), - buf_block_get_page_no(block)); -#endif /* UNIV_DEBUG_SCRUBBING */ - } else if (scrub) { - /* scrub records on page */ - - /* TODO(jonaso): in theory we could clear full page - * but, since page still remains in buffer pool, and - * gets flushed etc. Lots of routines validates consistency - * of it. And in order to remain structurally consistent - * we clear each record by it own - * - * NOTE: The TODO below mentions removing page from buffer pool - * and removing redo entries, once that is done, clearing full - * pages should be possible - */ - uint cnt = 0; - ulint bytes = 0; - page_t* page = buf_block_get_frame(block); - mem_heap_t* heap = NULL; - ulint* offsets = NULL; - rec_t* rec = page_rec_get_next(page_get_infimum_rec(page)); - while (!page_rec_is_supremum(rec)) { - offsets = rec_get_offsets(rec, index, offsets, - page_is_leaf(page), - ULINT_UNDEFINED, - &heap); - ulint size = rec_offs_data_size(offsets); - memset(rec, 0, size); - rec = page_rec_get_next(rec); - cnt++; - bytes += size; - } -#ifdef UNIV_DEBUG_SCRUBBING - fprintf(stderr, - "btr_page_free_low: scrub %lu/%lu - " - "%u records " ULINTPF " bytes\n", - buf_block_get_space(block), - buf_block_get_page_no(block), - cnt, bytes); -#endif /* UNIV_DEBUG_SCRUBBING */ - if (heap) { - mem_heap_free(heap); - } - } - -#ifdef UNIV_DEBUG_SCRUBBING - if (scrub == false) { - fprintf(stderr, - "btr_page_free_low %lu/%lu blob: %u\n", - buf_block_get_space(block), - buf_block_get_page_no(block), - blob); - } -#endif /* UNIV_DEBUG_SCRUBBING */ - if (dict_index_is_ibuf(index)) { - btr_page_free_for_ibuf(index, block, mtr); - return; } - root = btr_root_get(index, mtr); - - if (level == 0 || level == ULINT_UNDEFINED) { - seg_header = root + PAGE_HEADER + PAGE_BTR_SEG_LEAF; - } else { - seg_header = root + PAGE_HEADER + PAGE_BTR_SEG_TOP; - } - -#ifdef UNIV_GIS_DEBUG - if (dict_index_is_spatial(index)) { - fprintf(stderr, "GIS_DIAG: Freed %ld\n", - (long) block->page.id.page_no()); - } -#endif - - if (scrub) { - /** - * Reset page type so that scrub thread won't try to scrub it - */ - mlog_write_ulint(buf_block_get_frame(block) + FIL_PAGE_TYPE, - FIL_PAGE_TYPE_ALLOCATED, MLOG_2BYTES, mtr); - } - + /* TODO: Discard any operations for block from mtr->log. + The page will be freed, so previous changes to it by this + mini-transaction should not matter. */ + page_t* root = btr_root_get(index, mtr); + fseg_header_t* seg_header = &root[blob || page_is_leaf(block->frame) + ? PAGE_HEADER + PAGE_BTR_SEG_LEAF + : PAGE_HEADER + PAGE_BTR_SEG_TOP]; fseg_free_page(seg_header, - block->page.id.space(), - block->page.id.page_no(), - level != ULINT_UNDEFINED, mtr); + index->table->space, block->page.id.page_no(), + block->index != NULL, !block->page.flush_observer, mtr); /* The page was marked free in the allocation bitmap, but it - should remain buffer-fixed until mtr_commit(mtr) or until it + should remain exclusively latched until mtr_t::commit() or until it is explicitly freed from the mini-transaction. */ ut_ad(mtr_memo_contains(mtr, block, MTR_MEMO_PAGE_X_FIX)); - /* TODO: Discard any operations on the page from the redo log - and remove the block from the flush list and the buffer pool. - This would free up buffer pool earlier and reduce writes to - both the tablespace and the redo log. */ -} -/**************************************************************//** -Frees a file page used in an index tree. NOTE: cannot free field external -storage pages because the page must contain info on its level. */ -void -btr_page_free( -/*==========*/ - dict_index_t* index, /*!< in: index tree */ - buf_block_t* block, /*!< in: block to be freed, x-latched */ - mtr_t* mtr) /*!< in: mtr */ -{ - const page_t* page = buf_block_get_frame(block); - ulint level = btr_page_get_level(page); - - ut_ad(fil_page_index_page_check(block->frame)); - ut_ad(level != ULINT_UNDEFINED); - btr_page_free_low(index, block, level, false, mtr); + /* MDEV-15528 FIXME: Zero out the page after the redo log for + this mini-transaction has been durably written. + This must be done unconditionally if + srv_immediate_scrub_data_uncompressed is set. */ } /**************************************************************//** @@ -1092,17 +982,17 @@ void btr_page_get_father(dict_index_t* index, buf_block_t* block, mtr_t* mtr, mem_heap_free(heap); } +/** PAGE_INDEX_ID value for freed index B-trees */ +static const index_id_t BTR_FREED_INDEX_ID = 0; + /** Free a B-tree root page. btr_free_but_not_root() must already have been called. In a persistent tablespace, the caller must invoke fsp_init_file_page() before mtr.commit(). -@param[in,out] block index root page -@param[in,out] mtr mini-transaction */ -static -void -btr_free_root( - buf_block_t* block, - mtr_t* mtr) +@param[in,out] block index root page +@param[in,out] mtr mini-transaction +@param[in] invalidate whether to invalidate PAGE_INDEX_ID */ +static void btr_free_root(buf_block_t* block, mtr_t* mtr, bool invalidate) { fseg_header_t* header; @@ -1116,31 +1006,18 @@ btr_free_root( #ifdef UNIV_BTR_DEBUG ut_a(btr_root_fseg_validate(header, block->page.id.space())); #endif /* UNIV_BTR_DEBUG */ + if (invalidate) { + btr_page_set_index_id( + buf_block_get_frame(block), + buf_block_get_page_zip(block), + BTR_FREED_INDEX_ID, mtr); + } while (!fseg_free_step(header, true, mtr)) { /* Free the entire segment in small steps. */ } } -/** PAGE_INDEX_ID value for freed index B-trees */ -static const index_id_t BTR_FREED_INDEX_ID = 0; - -/** Invalidate an index root page so that btr_free_root_check() -will not find it. -@param[in,out] block index root page -@param[in,out] mtr mini-transaction */ -static -void -btr_free_root_invalidate( - buf_block_t* block, - mtr_t* mtr) -{ - btr_page_set_index_id( - buf_block_get_frame(block), - buf_block_get_page_zip(block), - BTR_FREED_INDEX_ID, mtr); -} - /** Prepare to free a B-tree. @param[in] page_id page id @param[in] zip_size ROW_FORMAT=COMPRESSED page size, or 0 @@ -1254,11 +1131,8 @@ btr_create( PAGE_HEADER + PAGE_BTR_SEG_LEAF, mtr)) { /* Not enough space for new segment, free root segment before return. */ - btr_free_root(block, mtr); - if (!index->table->is_temporary()) { - btr_free_root_invalidate(block, mtr); - } - + btr_free_root(block, mtr, + !index->table->is_temporary()); return(FIL_NULL); } @@ -1394,8 +1268,7 @@ btr_free_if_exists( btr_free_but_not_root(root, mtr->get_log_mode()); mtr->set_named_space_id(page_id.space()); - btr_free_root(root, mtr); - btr_free_root_invalidate(root, mtr); + btr_free_root(root, mtr, true); } /** Free an index tree in a temporary tablespace. @@ -1410,7 +1283,7 @@ void btr_free(const page_id_t page_id) if (block) { btr_free_but_not_root(block, MTR_LOG_NO_REDO); - btr_free_root(block, &mtr); + btr_free_root(block, &mtr, false); } mtr.commit(); } diff --git a/storage/innobase/btr/btr0bulk.cc b/storage/innobase/btr/btr0bulk.cc index 9cad745aa0d..6a9b71987c5 100644 --- a/storage/innobase/btr/btr0bulk.cc +++ b/storage/innobase/btr/btr0bulk.cc @@ -663,11 +663,11 @@ PageBulk::latch() /* In case the block is S-latched by page_cleaner. */ if (!buf_page_optimistic_get(RW_X_LATCH, m_block, m_modify_clock, __FILE__, __LINE__, &m_mtr)) { - m_block = buf_page_get_gen( - page_id_t(m_index->table->space_id, m_page_no), - m_index->table->space->zip_size(), - RW_X_LATCH, m_block, BUF_GET_IF_IN_POOL, - __FILE__, __LINE__, &m_mtr, &m_err); + m_block = buf_page_get_gen(page_id_t(m_index->table->space_id, + m_page_no), + 0, RW_X_LATCH, + m_block, BUF_GET_IF_IN_POOL, + __FILE__, __LINE__, &m_mtr, &m_err); if (m_err != DB_SUCCESS) { return (m_err); @@ -1033,7 +1033,7 @@ BtrBulk::finish(dberr_t err) root_page_bulk.copyIn(first_rec); /* Remove last page. */ - btr_page_free_low(m_index, last_block, m_root_level, false, &mtr); + btr_page_free(m_index, last_block, &mtr); /* Do not flush the last page. */ last_block->page.flush_observer = NULL; diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 0b61e340f81..1ba0febd415 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -8145,8 +8145,7 @@ btr_free_externally_stored_field( } next_page_no = mach_read_from_4(page + FIL_PAGE_NEXT); - btr_page_free_low(index, ext_block, 0, - true, &mtr); + btr_page_free(index, ext_block, &mtr, true); if (page_zip != NULL) { mach_write_to_4(field_ref + BTR_EXTERN_PAGE_NO, @@ -8172,12 +8171,7 @@ btr_free_externally_stored_field( next_page_no = mach_read_from_4( page + FIL_PAGE_DATA + BTR_BLOB_HDR_NEXT_PAGE_NO); - - /* We must supply the page level (= 0) as an argument - because we did not store it on the page (we save the - space overhead from an index page header. */ - btr_page_free_low(index, ext_block, 0, - true, &mtr); + btr_page_free(index, ext_block, &mtr, true); mlog_write_ulint(field_ref + BTR_EXTERN_PAGE_NO, next_page_no, diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 2828e63889a..24df4d76911 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1592,7 +1592,7 @@ buf_chunk_init( /* Round down to a multiple of page size, although it already should be. */ - mem_size = ut_2pow_round(mem_size, ulint(srv_page_size)); + mem_size = ut_2pow_round<ulint>(mem_size, srv_page_size); DBUG_EXECUTE_IF("ib_buf_chunk_init_fails", return(NULL);); @@ -1625,7 +1625,7 @@ buf_chunk_init( chunk->blocks = (buf_block_t*) chunk->mem; /* Align a pointer to the first frame. Note that when - os_large_page_size is smaller than srv_page_size, + opt_large_page_size is smaller than srv_page_size, we may allocate one fewer block than requested. When it is bigger, we may allocate more blocks than requested. */ @@ -4252,7 +4252,8 @@ buf_page_get_gen( Skip the assertion on space_page_size. */ break; case BUF_PEEK_IF_IN_POOL: - /* In this mode, the caller may pass a dummy page size, + case BUF_GET_IF_IN_POOL: + /* The caller may pass a dummy page size, because it does not really matter. */ break; default: @@ -4261,7 +4262,6 @@ buf_page_get_gen( ut_ad(rw_latch == RW_NO_LATCH); /* fall through */ case BUF_GET: - case BUF_GET_IF_IN_POOL: case BUF_GET_IF_IN_POOL_OR_WATCH: case BUF_GET_POSSIBLY_FREED: fil_space_t* s = fil_space_acquire_for_io(page_id.space()); @@ -6116,9 +6116,7 @@ database_corrupted: page_not_corrupt: bpage = bpage; ); if (recv_recovery_is_on()) { - /* Pages must be uncompressed for crash recovery. */ - ut_a(uncompressed); - recv_recover_page(TRUE, (buf_block_t*) bpage); + recv_recover_page(bpage); } /* If space is being truncated then avoid ibuf operation. @@ -6137,7 +6135,7 @@ database_corrupted: << " encrypted. However key " "management plugin or used " << "key_version " << key_version - << "is not found or" + << " is not found or" " used encryption algorithm or method does not match." " Can't continue opening the table."; } else { diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index f4271c8d738..d8ec476cb4b 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -186,7 +186,7 @@ buf_read_page_low( thd_wait_end(NULL); } - if (*err != DB_SUCCESS) { + if (UNIV_UNLIKELY(*err != DB_SUCCESS)) { if (IORequest::ignore_missing(type) || *err == DB_TABLESPACE_DELETED) { buf_read_page_handle_error(bpage); diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index c59f6107c5c..02da027f7a4 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -1424,7 +1424,7 @@ next: look to see if it is already in the tablespace cache. */ if (const fil_space_t* space = fil_space_for_table_exists_in_mem( - space_id, table_name.m_name, false, flags)) { + space_id, table_name.m_name, flags)) { /* Recovery can open a datafile that does not match SYS_DATAFILES. If they don't match, update SYS_DATAFILES. */ @@ -2799,11 +2799,16 @@ dict_load_tablespace( /* The tablespace may already be open. */ table->space = fil_space_for_table_exists_in_mem( - table->space_id, table->name.m_name, false, table->flags); + table->space_id, table->name.m_name, table->flags); if (table->space) { return; } + if (ignore_err == DICT_ERR_IGNORE_DROP) { + table->file_unreadable = true; + return; + } + if (!(ignore_err & DICT_ERR_IGNORE_RECOVER_LOCK)) { ib::error() << "Failed to find tablespace for table " << table->name << " in the cache. Attempting" diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 1ac4b96d4f4..1f13e2abfcc 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1169,9 +1169,7 @@ fil_space_free( rw_lock_x_unlock(&space->latch); } - bool need_mutex = !recv_recovery_on; - - if (need_mutex) { + if (!recv_recovery_is_on()) { log_mutex_enter(); } @@ -1182,7 +1180,7 @@ fil_space_free( UT_LIST_REMOVE(fil_system.named_spaces, space); } - if (need_mutex) { + if (!recv_recovery_is_on()) { log_mutex_exit(); } @@ -1241,9 +1239,8 @@ fil_space_create( UT_LIST_INIT(space->chain, &fil_node_t::chain); if ((purpose == FIL_TYPE_TABLESPACE || purpose == FIL_TYPE_IMPORT) - && !recv_recovery_on + && !recv_recovery_is_on() && id > fil_system.max_assigned_id) { - if (!fil_system.space_id_reuse_warned) { fil_system.space_id_reuse_warned = true; @@ -2847,7 +2844,7 @@ fil_rename_tablespace( ut_ad(strchr(old_file_name, OS_PATH_SEPARATOR) != NULL); ut_ad(strchr(new_file_name, OS_PATH_SEPARATOR) != NULL); - if (!recv_recovery_on) { + if (!recv_recovery_is_on()) { fil_name_write_rename(id, old_file_name, new_file_name); log_mutex_enter(); } @@ -2869,7 +2866,7 @@ fil_rename_tablespace( node->name = new_file_name; } - if (!recv_recovery_on) { + if (!recv_recovery_is_on()) { log_mutex_exit(); } @@ -3904,9 +3901,6 @@ memory cache. Note that if we have not done a crash recovery at the database startup, there may be many tablespaces which are not yet in the memory cache. @param[in] id Tablespace ID @param[in] name Tablespace name used in fil_space_create(). -@param[in] print_error_if_does_not_exist - Print detailed error information to the -error log if a matching tablespace is not found from memory. @param[in] table_flags table flags @return the tablespace @retval NULL if no matching tablespace exists in the memory cache */ @@ -3914,7 +3908,6 @@ fil_space_t* fil_space_for_table_exists_in_mem( ulint id, const char* name, - bool print_error_if_does_not_exist, ulint table_flags) { const ulint expected_flags = dict_tf_to_fsp_flags(table_flags); @@ -3936,7 +3929,8 @@ fil_space_for_table_exists_in_mem( << ", but the tablespace" " with that id has name " << space->name << "." " Have you deleted or moved .ibd files?"; - goto error_exit; + ib::info() << TROUBLESHOOT_DATADICT_MSG; + goto func_exit; } /* Adjust the flags that are in FSP_FLAGS_MEM_MASK. @@ -3951,17 +3945,6 @@ fil_space_for_table_exists_in_mem( return space; } - if (print_error_if_does_not_exist) { - ib::error() << "Table " << name - << " in the InnoDB data dictionary" - " has tablespace id " << id - << ", but tablespace with that id" - " or name does not exist. Have" - " you deleted or moved .ibd files?"; -error_exit: - ib::info() << TROUBLESHOOT_DATADICT_MSG; - } - func_exit: mutex_exit(&fil_system.mutex); return NULL; diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index d43b2352a58..da15fb38530 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -553,13 +553,9 @@ xdes_get_offset( * FSP_EXTENT_SIZE); } -/***********************************************************//** -Inits a file page whose prior contents should be ignored. */ -static -void -fsp_init_file_page_low( -/*===================*/ - buf_block_t* block) /*!< in: pointer to a page */ +/** Initialize a file page whose prior contents should be ignored. +@param[in,out] block buffer pool block */ +void fsp_apply_init_file_page(buf_block_t* block) { page_t* page = buf_block_get_frame(block); @@ -581,84 +577,34 @@ fsp_init_file_page_low( #ifdef UNIV_DEBUG /** Assert that the mini-transaction is compatible with updating an allocation bitmap page. -@param[in] id tablespace identifier @param[in] mtr mini-transaction */ -static -void -fsp_space_modify_check( - const fil_space_t* space, - const mtr_t* mtr) +void fil_space_t::modify_check(const mtr_t& mtr) const { - switch (mtr->get_log_mode()) { + switch (mtr.get_log_mode()) { case MTR_LOG_SHORT_INSERTS: case MTR_LOG_NONE: /* These modes are only allowed within a non-bitmap page when there is a higher-level redo log record written. */ - ut_ad(space->purpose == FIL_TYPE_TABLESPACE - || space->purpose == FIL_TYPE_TEMPORARY); + ut_ad(purpose == FIL_TYPE_TABLESPACE + || purpose == FIL_TYPE_TEMPORARY); break; case MTR_LOG_NO_REDO: - ut_ad(space->purpose == FIL_TYPE_TEMPORARY - || space->purpose == FIL_TYPE_IMPORT - || space->redo_skipped_count); + ut_ad(purpose == FIL_TYPE_TEMPORARY + || purpose == FIL_TYPE_IMPORT + || redo_skipped_count); return; case MTR_LOG_ALL: - /* We may only write redo log for a persistent tablespace. */ - ut_ad(space->purpose == FIL_TYPE_TABLESPACE); - ut_ad(mtr->is_named_space(space->id)); + /* We may only write redo log for a persistent + tablespace. */ + ut_ad(purpose == FIL_TYPE_TABLESPACE); + ut_ad(mtr.is_named_space(id)); return; } - ut_ad(0); -} -#endif /* UNIV_DEBUG */ - -/** Initialize a file page. -@param[in,out] block file page -@param[in,out] mtr mini-transaction */ -static -void -fsp_init_file_page(buf_block_t* block, mtr_t* mtr) -{ - fsp_init_file_page_low(block); - - mlog_write_initial_log_record(buf_block_get_frame(block), - MLOG_INIT_FILE_PAGE2, mtr); + ut_ad(!"invalid log mode"); } - -#ifdef UNIV_DEBUG -static -void -fsp_init_file_page(const fil_space_t* space, buf_block_t* block, mtr_t* mtr) -{ - ut_d(fsp_space_modify_check(space, mtr)); - ut_ad(space->id == block->page.id.space()); - fsp_init_file_page(block, mtr); -} -#else /* UNIV_DEBUG */ -# define fsp_init_file_page(space, block, mtr) fsp_init_file_page(block, mtr) #endif -/***********************************************************//** -Parses a redo log record of a file page init. -@return end of log record or NULL */ -byte* -fsp_parse_init_file_page( -/*=====================*/ - byte* ptr, /*!< in: buffer */ - byte* end_ptr MY_ATTRIBUTE((unused)), /*!< in: buffer end */ - buf_block_t* block) /*!< in: block or NULL */ -{ - ut_ad(ptr != NULL); - ut_ad(end_ptr != NULL); - - if (block) { - fsp_init_file_page_low(block); - } - - return(ptr); -} - /**********************************************************************//** Writes the space id and flags to a tablespace header. The flags contain row type, physical/compressed page size, and logical/uncompressed page @@ -784,7 +730,7 @@ fsp_try_extend_data_file_with_pages( ulint size; ut_a(!is_system_tablespace(space->id)); - ut_d(fsp_space_modify_check(space, mtr)); + ut_d(space->modify_check(*mtr)); size = mach_read_from_4(header + FSP_SIZE); ut_ad(size == space->size_in_header); @@ -854,7 +800,7 @@ fsp_try_extend_data_file(fil_space_t* space, fsp_header_t* header, mtr_t* mtr) "ran out of space. Please add another file or use" " 'autoextend' for the last file in setting"; - ut_d(fsp_space_modify_check(space, mtr)); + ut_d(space->modify_check(*mtr)); if (space->id == TRX_SYS_SPACE && !srv_sys_space.can_auto_extend_last_file()) { @@ -924,8 +870,7 @@ fsp_try_extend_data_file(fil_space_t* space, fsp_header_t* header, mtr_t* mtr) /* We ignore any fragments of a full megabyte when storing the size to the space header */ - space->size_in_header = ut_calc_align_down( - space->size, (1024 * 1024) / ps); + space->size_in_header = ut_2pow_round(space->size, (1024 * 1024) / ps); mlog_write_ulint( header + FSP_SIZE, space->size_in_header, MLOG_4BYTES, mtr); @@ -974,7 +919,7 @@ fsp_fill_free_list( ulint i; ut_ad(page_offset(header) == FSP_HEADER_OFFSET); - ut_d(fsp_space_modify_check(space, mtr)); + ut_d(space->modify_check(*mtr)); /* Check if we can fill free list from above the free list limit */ size = mach_read_from_4(header + FSP_SIZE); @@ -1007,8 +952,8 @@ fsp_fill_free_list( while ((init_space && i < 1) || ((i + FSP_EXTENT_SIZE <= size) && (count < FSP_FREE_ADD))) { - bool init_xdes - = (ut_2pow_remainder(i, space->physical_size()) == 0); + const bool init_xdes = 0 + == ut_2pow_remainder(i, ulint(space->physical_size())); space->free_limit = i + FSP_EXTENT_SIZE; mlog_write_ulint(header + FSP_FREE_LIMIT, i + FSP_EXTENT_SIZE, @@ -1281,7 +1226,7 @@ fsp_alloc_free_page( ulint free; const ulint space_id = space->id; - ut_d(fsp_space_modify_check(space, mtr)); + ut_d(space->modify_check(*mtr)); header = fsp_get_space_header(space, mtr); /* Get the hinted descriptor */ @@ -1368,8 +1313,10 @@ fsp_alloc_free_page( The page is marked as free and clean. @param[in,out] space tablespace @param[in] offset page number +@param[in] log whether to write MLOG_INIT_FREE_PAGE record @param[in,out] mtr mini-transaction */ -static void fsp_free_page(fil_space_t* space, page_no_t offset, mtr_t* mtr) +static void fsp_free_page(fil_space_t* space, page_no_t offset, + bool log, mtr_t* mtr) { fsp_header_t* header; xdes_t* descr; @@ -1377,7 +1324,7 @@ static void fsp_free_page(fil_space_t* space, page_no_t offset, mtr_t* mtr) ulint frag_n_used; ut_ad(mtr); - ut_d(fsp_space_modify_check(space, mtr)); + ut_d(space->modify_check(*mtr)); /* fprintf(stderr, "Freeing page %lu in space %lu\n", page, space); */ @@ -1423,6 +1370,17 @@ static void fsp_free_page(fil_space_t* space, page_no_t offset, mtr_t* mtr) return; } + if (UNIV_UNLIKELY(!log)) { + /* The last page freed in BtrBulk::finish() must be + written with redo logging disabled for the page + itself. The modifications of the allocation data + structures are covered by redo log. */ + } else if (byte* log_ptr = mlog_open(mtr, 11)) { + log_ptr = mlog_write_initial_log_record_low( + MLOG_INIT_FREE_PAGE, space->id, offset, log_ptr, mtr); + mlog_close(mtr, log_ptr); + } + const ulint bit = offset % FSP_EXTENT_SIZE; xdes_set_bit(descr, XDES_FREE_BIT, bit, TRUE, mtr); @@ -1686,16 +1644,18 @@ fsp_alloc_seg_inode( /** Frees a file segment inode. @param[in,out] space tablespace @param[in,out] inode segment inode +@param[in] log whether to write MLOG_INIT_FREE_PAGE record @param[in,out] mtr mini-transaction */ static void fsp_free_seg_inode( fil_space_t* space, fseg_inode_t* inode, + bool log, mtr_t* mtr) { page_t* page; fsp_header_t* space_header; - ut_d(fsp_space_modify_check(space, mtr)); + ut_d(space->modify_check(*mtr)); page = page_align(inode); @@ -1728,7 +1688,7 @@ static void fsp_free_seg_inode( flst_remove(space_header + FSP_SEG_INODES_FREE, page + FSEG_INODE_PAGE_NODE, mtr); - fsp_free_page(space, page_get_page_no(page), mtr); + fsp_free_page(space, page_get_page_no(page), log, mtr); } } @@ -1946,7 +1906,7 @@ fseg_create( <= srv_page_size - FIL_PAGE_DATA_END); mtr_x_lock(&space->latch, mtr); - ut_d(fsp_space_modify_check(space, mtr)); + ut_d(space->modify_check(*mtr)); if (page != 0) { block = buf_page_get(page_id_t(space->id, page), @@ -2011,7 +1971,7 @@ fseg_create( ut_ad(!has_done_reservation || block != NULL); if (block == NULL) { - fsp_free_seg_inode(space, inode, mtr); + fsp_free_seg_inode(space, inode, true, mtr); goto funct_exit; } @@ -2120,7 +2080,7 @@ fseg_fill_free_list( ut_ad(inode && mtr); ut_ad(!((page_offset(inode) - FSEG_ARR_OFFSET) % FSEG_INODE_SIZE)); - ut_d(fsp_space_modify_check(space, mtr)); + ut_d(space->modify_check(*mtr)); reserved = fseg_n_reserved_pages_low(inode, &used, mtr); @@ -2186,7 +2146,7 @@ fseg_alloc_free_extent( ut_ad(!((page_offset(inode) - FSEG_ARR_OFFSET) % FSEG_INODE_SIZE)); ut_ad(mach_read_from_4(inode + FSEG_MAGIC_N) == FSEG_MAGIC_N_VALUE); - ut_d(fsp_space_modify_check(space, mtr)); + ut_d(space->modify_check(*mtr)); if (flst_get_len(inode + FSEG_FREE) > 0) { /* Segment free list is not empty, allocate from it */ @@ -2271,7 +2231,7 @@ fseg_alloc_free_page_low( seg_id = mach_read_from_8(seg_inode + FSEG_ID); ut_ad(seg_id); - ut_d(fsp_space_modify_check(space, mtr)); + ut_d(space->modify_check(*mtr)); ut_ad(fil_page_get_type(page_align(seg_inode)) == FIL_PAGE_INODE); reserved = fseg_n_reserved_pages_low(seg_inode, &used, mtr); @@ -2779,6 +2739,7 @@ fseg_mark_page_used( @param[in] offset page number @param[in] ahi whether we may need to drop the adaptive hash index +@param[in] log whether to write MLOG_INIT_FREE_PAGE record @param[in,out] mtr mini-transaction */ static void @@ -2789,6 +2750,7 @@ fseg_free_page_low( #ifdef BTR_CUR_HASH_ADAPT bool ahi, #endif /* BTR_CUR_HASH_ADAPT */ + bool log, mtr_t* mtr) { xdes_t* descr; @@ -2803,7 +2765,7 @@ fseg_free_page_low( ut_ad(mach_read_from_4(seg_inode + FSEG_MAGIC_N) == FSEG_MAGIC_N_VALUE); ut_ad(!((page_offset(seg_inode) - FSEG_ARR_OFFSET) % FSEG_INODE_SIZE)); - ut_d(fsp_space_modify_check(space, mtr)); + ut_d(space->modify_check(*mtr)); #ifdef BTR_CUR_HASH_ADAPT /* Drop search system page hash index if the page is found in the pool and is hashed */ @@ -2842,7 +2804,7 @@ fseg_free_page_low( } } - fsp_free_page(space, offset, mtr); + fsp_free_page(space, offset, log, mtr); return; } @@ -2897,39 +2859,45 @@ fseg_free_page_low( } #ifndef BTR_CUR_HASH_ADAPT -# define fseg_free_page_low(inode, space, offset, ahi, mtr) \ - fseg_free_page_low(inode, space, offset, mtr) +# define fseg_free_page_low(inode, space, offset, ahi, log, mtr) \ + fseg_free_page_low(inode, space, offset, log, mtr) #endif /* !BTR_CUR_HASH_ADAPT */ -/**********************************************************************//** -Frees a single page of a segment. */ +/** Free a page in a file segment. +@param[in,out] seg_header file segment header +@param[in,out] space tablespace +@param[in] offset page number +@param[in] ahi whether we may need to drop the adaptive +hash index +@param[in] log whether to write MLOG_INIT_FREE_PAGE record +@param[in,out] mtr mini-transaction */ void fseg_free_page_func( - fseg_header_t* seg_header, /*!< in: segment header */ - ulint space_id,/*!< in: space id */ - ulint page, /*!< in: page offset */ + fseg_header_t* seg_header, + fil_space_t* space, + ulint offset, #ifdef BTR_CUR_HASH_ADAPT - bool ahi, /*!< in: whether we may need to drop - the adaptive hash index */ + bool ahi, #endif /* BTR_CUR_HASH_ADAPT */ - mtr_t* mtr) /*!< in/out: mini-transaction */ + bool log, + mtr_t* mtr) { DBUG_ENTER("fseg_free_page"); fseg_inode_t* seg_inode; buf_block_t* iblock; - fil_space_t* space = mtr_x_lock_space(space_id, mtr); + mtr_x_lock(&space->latch, mtr); - DBUG_LOG("fseg_free_page", "space_id: " << space_id - << ", page_no: " << page); + DBUG_LOG("fseg_free_page", "space_id: " << space->id + << ", page_no: " << offset); - seg_inode = fseg_inode_get(seg_header, space_id, space->zip_size(), + seg_inode = fseg_inode_get(seg_header, space->id, space->zip_size(), mtr, &iblock); fil_block_check_type(*iblock, FIL_PAGE_INODE, mtr); - fseg_free_page_low(seg_inode, space, page, ahi, mtr); + fseg_free_page_low(seg_inode, space, offset, ahi, log, mtr); - ut_d(buf_page_set_file_page_was_freed(page_id_t(space_id, page))); + ut_d(buf_page_set_file_page_was_freed(page_id_t(space->id, offset))); DBUG_VOID_RETURN; } @@ -2996,7 +2964,7 @@ fseg_free_extent( ut_a(!memcmp(descr + XDES_ID, seg_inode + FSEG_ID, 8)); ut_ad(mach_read_from_4(seg_inode + FSEG_MAGIC_N) == FSEG_MAGIC_N_VALUE); - ut_d(fsp_space_modify_check(space, mtr)); + ut_d(space->modify_check(*mtr)); first_page_in_extent = page - (page % FSP_EXTENT_SIZE); @@ -3117,7 +3085,7 @@ fseg_free_step_func( if (n == ULINT_UNDEFINED) { /* Freeing completed: free the segment inode */ - fsp_free_seg_inode(space, inode, mtr); + fsp_free_seg_inode(space, inode, true, mtr); DBUG_RETURN(TRUE); } @@ -3125,13 +3093,13 @@ fseg_free_step_func( fseg_free_page_low( inode, space, fseg_get_nth_frag_page_no(inode, n, mtr), - ahi, mtr); + ahi, true, mtr); n = fseg_find_last_used_frag_page_slot(inode, mtr); if (n == ULINT_UNDEFINED) { /* Freeing completed: free the segment inode */ - fsp_free_seg_inode(space, inode, mtr); + fsp_free_seg_inode(space, inode, true, mtr); DBUG_RETURN(TRUE); } @@ -3196,7 +3164,7 @@ fseg_free_step_not_header_func( return(TRUE); } - fseg_free_page_low(inode, space, page_no, ahi, mtr); + fseg_free_page_low(inode, space, page_no, ahi, true, mtr); return(FALSE); } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 828773fe761..c4fe6cdd449 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -4017,12 +4017,6 @@ static int innodb_init_params() innodb_log_checksums = innodb_log_checksums_func_update( NULL, innodb_log_checksums); -#ifdef HAVE_LINUX_LARGE_PAGES - if ((os_use_large_pages = my_use_large_pages)) { - os_large_page_size = opt_large_page_size; - } -#endif - row_rollback_on_timeout = (ibool) innobase_rollback_on_timeout; srv_locks_unsafe_for_binlog = (ibool) innobase_locks_unsafe_for_binlog; diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 346caae40c2..639836c9da9 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -2090,8 +2090,9 @@ ibuf_remove_free_page(void) the free list was so long that they cannot have taken the last page from it. */ + compile_time_assert(IBUF_SPACE_ID == 0); fseg_free_page(header_page + IBUF_HEADER + IBUF_TREE_SEG_HEADER, - IBUF_SPACE_ID, page_no, false, &mtr); + fil_system.sys_space, page_no, false, true, &mtr); const page_id_t page_id(IBUF_SPACE_ID, page_no); diff --git a/storage/innobase/include/btr0btr.h b/storage/innobase/include/btr0btr.h index 2585780cfd6..adfd31aa870 100644 --- a/storage/innobase/include/btr0btr.h +++ b/storage/innobase/include/btr0btr.h @@ -676,16 +676,6 @@ btr_page_alloc( for x-latching and initializing the page */ MY_ATTRIBUTE((warn_unused_result)); -/**************************************************************//** -Frees a file page used in an index tree. NOTE: cannot free field external -storage pages because the page must contain info on its level. */ -void -btr_page_free( -/*==========*/ - dict_index_t* index, /*!< in: index tree */ - buf_block_t* block, /*!< in: block to be freed, x-latched */ - mtr_t* mtr) /*!< in: mtr */ - MY_ATTRIBUTE((nonnull)); /** Empty an index page (possibly the root page). @see btr_page_create(). @param[in,out] block page to be emptied @param[in,out] page_zip compressed page frame, or NULL @@ -711,18 +701,16 @@ btr_page_create( dict_index_t* index, /*!< in: index */ ulint level, /*!< in: the B-tree level of the page */ mtr_t* mtr); /*!< in: mtr */ -/**************************************************************//** -Frees a file page used in an index tree. Can be used also to BLOB -external storage pages. */ -void -btr_page_free_low( -/*==============*/ - dict_index_t* index, /*!< in: index tree */ - buf_block_t* block, /*!< in: block to be freed, x-latched */ - ulint level, /*!< in: page level (ULINT_UNDEFINED=BLOB) */ - bool blob, /*!< in: blob page */ - mtr_t* mtr) /*!< in: mtr */ - MY_ATTRIBUTE((nonnull(1,2))); + +/** Free an index page. +@param[in,out] index index tree +@param[in,out] block block to be freed +@param[in,out] mtr mini-transaction +@param[in] blob whether this is freeing a BLOB page */ +MY_ATTRIBUTE((nonnull)) +void btr_page_free(dict_index_t* index, buf_block_t* block, mtr_t* mtr, + bool blob = false); + /**************************************************************//** Gets the root node of a tree and x- or s-latches it. @return root page, x- or s-latched */ diff --git a/storage/innobase/include/dict0types.h b/storage/innobase/include/dict0types.h index e3b536b83ee..80b918b2df0 100644 --- a/storage/innobase/include/dict0types.h +++ b/storage/innobase/include/dict0types.h @@ -78,7 +78,10 @@ enum dict_err_ignore_t { Silently load a missing tablespace, and do not load incomplete index definitions. */ - DICT_ERR_IGNORE_ALL = 0xFFFF /*!< ignore all errors */ + /** ignore all errors above */ + DICT_ERR_IGNORE_ALL = 15, + /** prepare to drop the table; do not attempt to load tablespace */ + DICT_ERR_IGNORE_DROP = 31 }; /** Quiescing states for flushing tables to disk. */ diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 18449785e61..b58d4e85ac9 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -105,7 +105,7 @@ struct fil_space_t { bool is_being_truncated; #ifdef UNIV_DEBUG /** reference count for operations who want to skip redo log in the - file space in order to make fsp_space_modify_check pass. */ + file space in order to make modify_check() pass. */ Atomic_counter<ulint> redo_skipped_count; #endif fil_type_t purpose;/*!< purpose */ @@ -198,6 +198,12 @@ struct fil_space_t { fil_node_t* add(const char* name, pfs_os_file_t handle, ulint size, bool is_raw, bool atomic_write, ulint max_pages = ULINT_MAX); +#ifdef UNIV_DEBUG + /** Assert that the mini-transaction is compatible with + updating an allocation bitmap page. + @param[in] mtr mini-transaction */ + void modify_check(const mtr_t& mtr) const; +#endif /* UNIV_DEBUG */ /** Try to reserve free extents. @param[in] n_free_now current number of free extents @@ -1338,9 +1344,6 @@ memory cache. Note that if we have not done a crash recovery at the database startup, there may be many tablespaces which are not yet in the memory cache. @param[in] id Tablespace ID @param[in] name Tablespace name used in fil_space_create(). -@param[in] print_error_if_does_not_exist - Print detailed error information to the -error log if a matching tablespace is not found from memory. @param[in] table_flags table flags @return the tablespace @retval NULL if no matching tablespace exists in the memory cache */ @@ -1348,7 +1351,6 @@ fil_space_t* fil_space_for_table_exists_in_mem( ulint id, const char* name, - bool print_error_if_does_not_exist, ulint table_flags); /** Try to extend a tablespace if it is smaller than the specified size. @@ -1554,16 +1556,12 @@ fil_names_write_if_was_clean( return(was_clean); } -extern volatile bool recv_recovery_on; - /** During crash recovery, open a tablespace if it had not been opened yet, to get valid size and flags. @param[in,out] space tablespace */ -inline -void -fil_space_open_if_needed( - fil_space_t* space) +inline void fil_space_open_if_needed(fil_space_t* space) { + ut_d(extern volatile bool recv_recovery_on); ut_ad(recv_recovery_on); if (space->size == 0) { @@ -1571,10 +1569,7 @@ fil_space_open_if_needed( until the files are opened for the first time. fil_space_get_size() will open the file and adjust the size and flags. */ -#ifdef UNIV_DEBUG - ulint size = -#endif /* UNIV_DEBUG */ - fil_space_get_size(space->id); + ut_d(ulint size =) fil_space_get_size(space->id); ut_ad(size == space->size); } } diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index 9a89971bb0d..897d9d29d63 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -484,24 +484,30 @@ fsp_reserve_free_extents( mtr_t* mtr, ulint n_pages = 2); -/**********************************************************************//** -Frees a single page of a segment. */ +/** Free a page in a file segment. +@param[in,out] seg_header file segment header +@param[in,out] space tablespace +@param[in] offset page number +@param[in] ahi whether we may need to drop the adaptive +hash index +@param[in] log whether to write MLOG_INIT_FREE_PAGE record +@param[in,out] mtr mini-transaction */ void fseg_free_page_func( - fseg_header_t* seg_header, /*!< in: segment header */ - ulint space_id, /*!< in: space id */ - ulint page, /*!< in: page offset */ + fseg_header_t* seg_header, + fil_space_t* space, + ulint offset, #ifdef BTR_CUR_HASH_ADAPT - bool ahi, /*!< in: whether we may need to drop - the adaptive hash index */ + bool ahi, #endif /* BTR_CUR_HASH_ADAPT */ - mtr_t* mtr); /*!< in/out: mini-transaction */ + bool log, + mtr_t* mtr); #ifdef BTR_CUR_HASH_ADAPT -# define fseg_free_page(header, space_id, page, ahi, mtr) \ - fseg_free_page_func(header, space_id, page, ahi, mtr) +# define fseg_free_page(header, space, offset, ahi, log, mtr) \ + fseg_free_page_func(header, space, offset, ahi, log, mtr) #else /* BTR_CUR_HASH_ADAPT */ -# define fseg_free_page(header, space_id, page, ahi, mtr) \ - fseg_free_page_func(header, space_id, page, mtr) +# define fseg_free_page(header, space, offset, ahi, log, mtr) \ + fseg_free_page_func(header, space, offset, log, mtr) #endif /* BTR_CUR_HASH_ADAPT */ /** Determine whether a page is free. @param[in,out] space tablespace @@ -602,15 +608,30 @@ inline bool fsp_descr_page(const page_id_t page_id, ulint physical_size) return (page_id.page_no() & (physical_size - 1)) == FSP_XDES_OFFSET; } -/***********************************************************//** -Parses a redo log record of a file page init. -@return end of log record or NULL */ -byte* -fsp_parse_init_file_page( -/*=====================*/ - byte* ptr, /*!< in: buffer */ - byte* end_ptr, /*!< in: buffer end */ - buf_block_t* block); /*!< in: block or NULL */ +/** Initialize a file page whose prior contents should be ignored. +@param[in,out] block buffer pool block */ +void fsp_apply_init_file_page(buf_block_t* block); + +/** Initialize a file page. +@param[in] space tablespace +@param[in,out] block file page +@param[in,out] mtr mini-transaction */ +inline void fsp_init_file_page( +#ifdef UNIV_DEBUG + const fil_space_t* space, +#endif + buf_block_t* block, mtr_t* mtr) +{ + ut_d(space->modify_check(*mtr)); + ut_ad(space->id == block->page.id.space()); + fsp_apply_init_file_page(block); + mlog_write_initial_log_record(block->frame, MLOG_INIT_FILE_PAGE2, mtr); +} + +#ifndef UNIV_DEBUG +# define fsp_init_file_page(space, block, mtr) fsp_init_file_page(block, mtr) +#endif + #ifdef UNIV_BTR_PRINT /*******************************************************************//** Writes info of a segment. */ @@ -769,8 +790,9 @@ xdes_get_bit( @return descriptor index */ inline ulint xdes_calc_descriptor_index(ulint zip_size, ulint offset) { - return(ut_2pow_remainder(offset, zip_size ? zip_size : srv_page_size) - / FSP_EXTENT_SIZE); + return ut_2pow_remainder<ulint>(offset, + zip_size ? zip_size : srv_page_size) + / FSP_EXTENT_SIZE; } /** Determine the descriptor page number for a page. @@ -795,7 +817,8 @@ inline ulint xdes_calc_descriptor_page(ulint zip_size, ulint offset) ut_ad(!zip_size || zip_size > XDES_ARR_OFFSET + (zip_size / FSP_EXTENT_SIZE) * XDES_SIZE); - return ut_2pow_round(offset, zip_size ? zip_size : srv_page_size); + return ut_2pow_round<ulint>(offset, + zip_size ? zip_size : srv_page_size); } #endif /* UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 3f0d149d5f1..e088f248681 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2009, Google Inc. -Copyright (c) 2017, 2018, MariaDB Corporation. +Copyright (c) 2017, 2019, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -559,11 +559,12 @@ struct log_t{ uint32_t subformat; /** individual log file size in bytes, including the header */ lsn_t file_size; + private: /** lsn used to fix coordinates within the log group */ lsn_t lsn; /** the byte offset of the above lsn */ lsn_t lsn_offset; - + public: /** used only in recovery: recovery scan succeeded up to this lsn in this log group */ lsn_t scanned_lsn; @@ -580,8 +581,9 @@ struct log_t{ /** Set the field values to correspond to a given lsn. */ void set_fields(lsn_t lsn) { - lsn_offset = calc_lsn_offset(lsn); - this->lsn = lsn; + lsn_t c_lsn_offset = calc_lsn_offset(lsn); + set_lsn(lsn); + set_lsn_offset(c_lsn_offset); } /** Read a log segment to log_sys.buf. @@ -600,6 +602,10 @@ struct log_t{ { n_files = 0; } + void set_lsn(lsn_t a_lsn); + lsn_t get_lsn() const { return lsn; } + void set_lsn_offset(lsn_t a_lsn); + lsn_t get_lsn_offset() const { return lsn_offset; } } log; /** The fields involved in the log buffer flush @{ */ @@ -760,6 +766,17 @@ inline lsn_t log_t::files::calc_lsn_offset(lsn_t lsn) const return l + LOG_FILE_HDR_SIZE * (1 + l / (file_size - LOG_FILE_HDR_SIZE)); } +inline void log_t::files::set_lsn(lsn_t a_lsn) { + ut_ad(log_sys.mutex.is_owned() || log_sys.write_mutex.is_owned()); + lsn = a_lsn; +} + +inline void log_t::files::set_lsn_offset(lsn_t a_lsn) { + ut_ad(log_sys.mutex.is_owned() || log_sys.write_mutex.is_owned()); + ut_ad((lsn % OS_FILE_LOG_BLOCK_SIZE) == (a_lsn % OS_FILE_LOG_BLOCK_SIZE)); + lsn_offset = a_lsn; +} + /** Test if flush order mutex is owned. */ #define log_flush_order_mutex_own() \ mutex_own(&log_sys.log_flush_order_mutex) diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h index 5fe00d65f2c..5d5ccac885a 100644 --- a/storage/innobase/include/log0recv.h +++ b/storage/innobase/include/log0recv.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2018, MariaDB Corporation. +Copyright (c) 2017, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -40,7 +40,7 @@ Created 9/20/1997 Heikki Tuuri extern bool recv_writer_thread_active; /** @return whether recovery is currently running. */ -#define recv_recovery_is_on() recv_recovery_on +#define recv_recovery_is_on() UNIV_UNLIKELY(recv_recovery_on) /** Find the latest checkpoint in the log header. @param[out] max_field LOG_CHECKPOINT_1 or LOG_CHECKPOINT_2 @@ -49,12 +49,9 @@ dberr_t recv_find_max_checkpoint(ulint* max_field) MY_ATTRIBUTE((nonnull, warn_unused_result)); -/** Apply the hashed log records to the page, if the page lsn is less than the -lsn of a log record. -@param just_read_in whether the page recently arrived to the I/O handler -@param block the page in the buffer pool */ -void -recv_recover_page(bool just_read_in, buf_block_t* block); +/** Apply any buffered redo log to a page that was just read from a data file. +@param[in,out] bpage buffer pool page */ +ATTRIBUTE_COLD void recv_recover_page(buf_page_t* bpage); /** Start recovering from a redo log checkpoint. @see recv_recovery_from_checkpoint_finish diff --git a/storage/innobase/include/mem0mem.h b/storage/innobase/include/mem0mem.h index b013d597393..a1f7282b3e2 100644 --- a/storage/innobase/include/mem0mem.h +++ b/storage/innobase/include/mem0mem.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2018, MariaDB Corporation. +Copyright (c) 2017, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -77,7 +77,7 @@ is the maximum size for a single allocated buffer: */ /** Space needed when allocating for a user a field of length N. The space is allocated only in multiples of UNIV_MEM_ALIGNMENT. */ -#define MEM_SPACE_NEEDED(N) ut_calc_align((N), UNIV_MEM_ALIGNMENT) +#define MEM_SPACE_NEEDED(N) UT_CALC_ALIGN((N), UNIV_MEM_ALIGNMENT) #ifdef UNIV_DEBUG /** Macro for memory heap creation. @@ -348,8 +348,8 @@ struct mem_block_info_t { #define MEM_FREED_BLOCK_MAGIC_N 547711122 /* Header size for a memory heap block */ -#define MEM_BLOCK_HEADER_SIZE ut_calc_align(sizeof(mem_block_info_t),\ - UNIV_MEM_ALIGNMENT) +#define MEM_BLOCK_HEADER_SIZE UT_CALC_ALIGN(sizeof(mem_block_info_t),\ + UNIV_MEM_ALIGNMENT) #include "mem0mem.ic" #endif diff --git a/storage/innobase/include/os0proc.h b/storage/innobase/include/os0proc.h index 823da83c63e..30ef295a7ab 100644 --- a/storage/innobase/include/os0proc.h +++ b/storage/innobase/include/os0proc.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -42,12 +42,6 @@ typedef unsigned long int os_process_id_t; system with os_mem_alloc_large(). */ extern Atomic_counter<ulint> os_total_large_mem_allocated; -/** Whether to use large pages in the buffer pool */ -extern my_bool os_use_large_pages; - -/** Large page size. This may be a boot-time option on some platforms */ -extern uint os_large_page_size; - /** Converts the current process id to a number. @return process id as a number */ ulint diff --git a/storage/innobase/include/page0zip.h b/storage/innobase/include/page0zip.h index c5376967bd8..b462833e4c7 100644 --- a/storage/innobase/include/page0zip.h +++ b/storage/innobase/include/page0zip.h @@ -471,16 +471,14 @@ page_zip_copy_recs( dict_index_t* index, /*!< in: index of the B-tree */ mtr_t* mtr); /*!< in: mini-transaction */ -/**********************************************************************//** -Parses a log record of compressing an index page. -@return end of log record or NULL */ -byte* -page_zip_parse_compress( -/*====================*/ - byte* ptr, /*!< in: buffer */ - byte* end_ptr, /*!< in: buffer end */ - page_t* page, /*!< out: uncompressed page */ - page_zip_des_t* page_zip); /*!< out: compressed page */ +/** Parse and optionally apply MLOG_ZIP_PAGE_COMPRESS. +@param[in] ptr log record +@param[in] end_ptr end of log +@param[in,out] block ROW_FORMAT=COMPRESSED block, or NULL for parsing only +@return end of log record +@retval NULL if the log record is incomplete */ +byte* page_zip_parse_compress(const byte* ptr, const byte* end_ptr, + buf_block_t* block); #endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/include/row0merge.h b/storage/innobase/include/row0merge.h index 7d49f0ee346..1bc3bc060dc 100644 --- a/storage/innobase/include/row0merge.h +++ b/storage/innobase/include/row0merge.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -292,6 +292,12 @@ row_merge_drop_table( dict_table_t* table) /*!< in: table instance to drop */ MY_ATTRIBUTE((nonnull, warn_unused_result)); +/** Write an MLOG_INDEX_LOAD record to indicate in the redo-log +that redo-logging of individual index pages was disabled, and +the flushing of such pages to the data files was completed. +@param[in] index an index tree on which redo logging was disabled */ +void row_merge_write_redo(const dict_index_t* index); + /** Build indexes on a table by reading a clustered index, creating a temporary file containing index entries, merge sorting these index entries and inserting sorted index entries to indexes. diff --git a/storage/innobase/include/ut0ut.h b/storage/innobase/include/ut0ut.h index 6ca661378b1..65c5d63953c 100644 --- a/storage/innobase/include/ut0ut.h +++ b/storage/innobase/include/ut0ut.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -127,26 +128,23 @@ Calculates fast the remainder of n/m when m is a power of two. @param n in: numerator @param m in: denominator, must be a power of two @return the remainder of n/m */ -#define ut_2pow_remainder(n, m) ((n) & ((m) - 1)) +template <typename T> inline T ut_2pow_remainder(T n, T m){return n & (m - 1);} /*************************************************************//** Calculates the biggest multiple of m that is not bigger than n when m is a power of two. In other words, rounds n down to m * k. @param n in: number to round down @param m in: alignment, must be a power of two @return n rounded down to the biggest possible integer multiple of m */ -#define ut_2pow_round(n, m) ((n) & ~((m) - 1)) -/** Align a number down to a multiple of a power of two. -@param n in: number to round down -@param m in: alignment, must be a power of two -@return n rounded down to the biggest possible integer multiple of m */ -#define ut_calc_align_down(n, m) ut_2pow_round(n, m) +template <typename T> inline T ut_2pow_round(T n, T m) { return n & ~(m - 1); } /********************************************************//** Calculates the smallest multiple of m that is not smaller than n when m is a power of two. In other words, rounds n up to m * k. @param n in: number to round up @param m in: alignment, must be a power of two @return n rounded up to the smallest possible integer multiple of m */ -#define ut_calc_align(n, m) (((n) + ((m) - 1)) & ~((m) - 1)) +#define UT_CALC_ALIGN(n, m) ((n + m - 1) & ~(m - 1)) +template <typename T> inline T ut_calc_align(T n, T m) +{ return UT_CALC_ALIGN(n, m); } /*************************************************************//** Calculates fast the 2-logarithm of a number, rounded upward to an diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index a39b46b53be..a6f8ea99dcf 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Google Inc. -Copyright (c) 2014, 2018, MariaDB Corporation. +Copyright (c) 2014, 2019, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -193,10 +193,8 @@ void log_buffer_extend(ulong len) log_sys.is_extending = true; - while (ut_calc_align_down(log_sys.buf_free, - OS_FILE_LOG_BLOCK_SIZE) - != ut_calc_align_down(log_sys.buf_next_to_write, - OS_FILE_LOG_BLOCK_SIZE)) { + while ((log_sys.buf_free ^ log_sys.buf_next_to_write) + & (OS_FILE_LOG_BLOCK_SIZE - 1)) { /* Buffer might have >1 blocks to write still. */ log_mutex_exit_all(); @@ -205,9 +203,8 @@ void log_buffer_extend(ulong len) log_mutex_enter_all(); } - ulong move_start = ut_calc_align_down( - log_sys.buf_free, - OS_FILE_LOG_BLOCK_SIZE); + ulong move_start = ut_2pow_round(log_sys.buf_free, + ulong(OS_FILE_LOG_BLOCK_SIZE)); ulong move_end = log_sys.buf_free; /* store the last log block in buffer */ @@ -896,8 +893,8 @@ log_buffer_switch() ut_ad(log_write_mutex_own()); const byte* old_buf = log_sys.buf; - ulint area_end = ut_calc_align(log_sys.buf_free, - OS_FILE_LOG_BLOCK_SIZE); + ulong area_end = ut_calc_align( + log_sys.buf_free, ulong(OS_FILE_LOG_BLOCK_SIZE)); if (log_sys.first_in_use) { log_sys.first_in_use = false; @@ -1031,8 +1028,9 @@ loop: start_offset = log_sys.buf_next_to_write; end_offset = log_sys.buf_free; - area_start = ut_calc_align_down(start_offset, OS_FILE_LOG_BLOCK_SIZE); - area_end = ut_calc_align(end_offset, OS_FILE_LOG_BLOCK_SIZE); + area_start = ut_2pow_round(start_offset, + ulint(OS_FILE_LOG_BLOCK_SIZE)); + area_end = ut_calc_align(end_offset, ulint(OS_FILE_LOG_BLOCK_SIZE)); ut_ad(area_end - area_start > 0); @@ -1189,14 +1187,11 @@ synchronization objects! this lsn @return false if there was a flush batch of the same type running, which means that we could not start this flush batch */ -static -bool -log_preflush_pool_modified_pages( - lsn_t new_oldest) +static bool log_preflush_pool_modified_pages(lsn_t new_oldest) { bool success; - if (recv_recovery_on) { + if (recv_recovery_is_on()) { /* If the recovery is running, we must first apply all log records to their respective file pages to get the right modify lsn values to these pages: otherwise, there diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 4840bf3088a..6305f5ec1d2 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -670,7 +670,7 @@ DECLARE_THREAD(recv_writer_thread)( mutex_enter(&recv_sys->writer_mutex); - if (!recv_recovery_on) { + if (!recv_recovery_is_on()) { mutex_exit(&recv_sys->writer_mutex); break; } @@ -765,7 +765,7 @@ recv_sys_debug_free(void) /* wake page cleaner up to progress */ if (!srv_read_only_mode) { - ut_ad(!recv_recovery_on); + ut_ad(!recv_recovery_is_on()); ut_ad(!recv_writer_thread_active); os_event_reset(buf_flush_event); os_event_set(recv_sys->flush_start); @@ -987,11 +987,12 @@ recv_find_max_checkpoint_0(ulint* max_field) *max_field = field; max_no = checkpoint_no; - log_sys.log.lsn = mach_read_from_8( - buf + LOG_CHECKPOINT_LSN); - log_sys.log.lsn_offset = static_cast<ib_uint64_t>( - mach_read_from_4(buf + OFFSET_HIGH32)) << 32 - | mach_read_from_4(buf + OFFSET_LOW32); + log_sys.log.set_lsn(mach_read_from_8( + buf + LOG_CHECKPOINT_LSN)); + log_sys.log.set_lsn_offset( + lsn_t(mach_read_from_4(buf + OFFSET_HIGH32)) + << 32 + | mach_read_from_4(buf + OFFSET_LOW32)); } } @@ -1152,10 +1153,10 @@ recv_find_max_checkpoint(ulint* max_field) if (checkpoint_no >= max_no) { *max_field = field; max_no = checkpoint_no; - log_sys.log.lsn = mach_read_from_8( - buf + LOG_CHECKPOINT_LSN); - log_sys.log.lsn_offset = mach_read_from_8( - buf + LOG_CHECKPOINT_OFFSET); + log_sys.log.set_lsn(mach_read_from_8( + buf + LOG_CHECKPOINT_LSN)); + log_sys.log.set_lsn_offset(mach_read_from_8( + buf + LOG_CHECKPOINT_OFFSET)); log_sys.next_checkpoint_no = checkpoint_no; } } @@ -1550,7 +1551,7 @@ parse_log: break; case MLOG_INIT_FILE_PAGE2: /* Allow anything in page_type when creating a page. */ - ptr = fsp_parse_init_file_page(ptr, end_ptr, block); + if (block) fsp_apply_init_file_page(block); break; case MLOG_INIT_FREE_PAGE: /* The page can be zero-filled and its previous @@ -1577,7 +1578,7 @@ parse_log: break; case MLOG_ZIP_PAGE_COMPRESS: /* Allow anything in page_type when creating a page. */ - ptr = page_zip_parse_compress(ptr, end_ptr, page, page_zip); + ptr = page_zip_parse_compress(ptr, end_ptr, block); break; case MLOG_ZIP_PAGE_COMPRESS_NO_DATA: if (NULL != (ptr = mlog_parse_index( @@ -1659,6 +1660,8 @@ recv_get_fil_addr_struct( ulint space, /*!< in: space id */ ulint page_no)/*!< in: page number */ { + ut_ad(mutex_own(&recv_sys->mutex)); + recv_addr_t* recv_addr; for (recv_addr = static_cast<recv_addr_t*>( @@ -1732,10 +1735,6 @@ recv_add_to_hash_table( HASH_INSERT(recv_addr_t, addr_hash, recv_sys->addr_hash, recv_fold(space, page_no), recv_addr); recv_sys->n_addrs++; -#if 0 - fprintf(stderr, "Inserting log rec for space %lu, page %lu\n", - space, page_no); -#endif } UT_LIST_ADD_LAST(recv_addr->rec_list, recv); @@ -1804,45 +1803,20 @@ recv_data_copy_to_buf( /** Apply the hashed log records to the page, if the page lsn is less than the lsn of a log record. -@param just_read_in whether the page recently arrived to the I/O handler -@param block the page in the buffer pool */ -void -recv_recover_page(bool just_read_in, buf_block_t* block) +@param[in,out] block buffer pool page +@param[in,out] mtr mini-transaction +@param[in,out] recv_addr recovery address */ +static void recv_recover_page(buf_block_t* block, mtr_t& mtr, + recv_addr_t* recv_addr) { page_t* page; page_zip_des_t* page_zip; - recv_addr_t* recv_addr; - lsn_t start_lsn; - lsn_t end_lsn; - lsn_t page_lsn; - lsn_t page_newest_lsn; - mtr_t mtr; - - mutex_enter(&(recv_sys->mutex)); - - if (recv_sys->apply_log_recs == FALSE) { - - /* Log records should not be applied now */ - - mutex_exit(&(recv_sys->mutex)); - - return; - } - - recv_addr = recv_get_fil_addr_struct(block->page.id.space(), - block->page.id.page_no()); - - if ((recv_addr == NULL) - || (recv_addr->state == RECV_BEING_PROCESSED) - || (recv_addr->state == RECV_PROCESSED)) { - ut_ad(recv_addr == NULL || recv_needed_recovery); - - mutex_exit(&(recv_sys->mutex)); - - return; - } + ut_ad(mutex_own(&recv_sys->mutex)); + ut_ad(recv_sys->apply_log_recs); ut_ad(recv_needed_recovery); + ut_ad(recv_addr->state != RECV_BEING_PROCESSED); + ut_ad(recv_addr->state != RECV_PROCESSED); if (UNIV_UNLIKELY(srv_print_verbose_log == 2)) { fprintf(stderr, "Applying log to page %u:%u\n", @@ -1852,56 +1826,30 @@ recv_recover_page(bool just_read_in, buf_block_t* block) DBUG_LOG("ib_log", "Applying log to page " << block->page.id); recv_addr->state = RECV_BEING_PROCESSED; - - mutex_exit(&(recv_sys->mutex)); - - mtr_start(&mtr); - mtr_set_log_mode(&mtr, MTR_LOG_NONE); + mutex_exit(&recv_sys->mutex); page = block->frame; page_zip = buf_block_get_page_zip(block); - if (just_read_in) { - /* Move the ownership of the x-latch on the page to - this OS thread, so that we can acquire a second - x-latch on it. This is needed for the operations to - the page to pass the debug checks. */ - - rw_lock_x_lock_move_ownership(&block->lock); + /* The page may have been modified in the buffer pool. + FIL_PAGE_LSN would only be updated right before flushing. */ + lsn_t page_lsn = buf_page_get_newest_modification(&block->page); + if (!page_lsn) { + page_lsn = mach_read_from_8(page + FIL_PAGE_LSN); } - ibool success = buf_page_get_known_nowait( - RW_X_LATCH, block, BUF_KEEP_OLD, - __FILE__, __LINE__, &mtr); - ut_a(success); - - buf_block_dbg_add_level(block, SYNC_NO_ORDER_CHECK); - - /* Read the newest modification lsn from the page */ - page_lsn = mach_read_from_8(page + FIL_PAGE_LSN); - - /* It may be that the page has been modified in the buffer - pool: read the newest modification lsn there */ - - page_newest_lsn = buf_page_get_newest_modification(&block->page); - - if (page_newest_lsn) { - - page_lsn = page_newest_lsn; - } - - start_lsn = end_lsn = 0; - - fil_space_t* space = fil_space_acquire(block->page.id.space()); + lsn_t start_lsn = 0, end_lsn = 0; for (recv_t* recv = UT_LIST_GET_FIRST(recv_addr->rec_list); recv; recv = UT_LIST_GET_NEXT(rec_list, recv)) { + ut_ad(recv->start_lsn); end_lsn = recv->end_lsn; - ut_ad(end_lsn <= log_sys.log.scanned_lsn); - ut_ad(recv->start_lsn); - if (recv->start_lsn >= page_lsn) { + if (recv->start_lsn < page_lsn) { + /* Ignore this record, because there are later changes + for this page. */ + } else { if (!start_lsn) { start_lsn = recv->start_lsn; } @@ -1923,10 +1871,8 @@ recv_recover_page(bool just_read_in, buf_block_t* block) if (recv->len > RECV_DATA_BLOCK_SIZE) { /* We have to copy the record body to a separate buffer */ - - buf = static_cast<byte*>(ut_malloc_nokey( - recv->len)); - + buf = static_cast<byte*> + (ut_malloc_nokey(recv->len)); recv_data_copy_to_buf(buf, recv); } else { buf = reinterpret_cast<byte*>(recv->data) @@ -1955,15 +1901,10 @@ recv_recover_page(bool just_read_in, buf_block_t* block) } } - space->release(); - #ifdef UNIV_ZIP_DEBUG - if (fil_page_index_page_check(page)) { - page_zip_des_t* page_zip = buf_block_get_page_zip(block); - - ut_a(!page_zip - || page_zip_validate_low(page_zip, page, NULL, FALSE)); - } + ut_ad(!fil_page_index_page_check(page) + || !page_zip + || page_zip_validate_low(page_zip, page, NULL, FALSE)); #endif /* UNIV_ZIP_DEBUG */ if (start_lsn) { @@ -1976,8 +1917,7 @@ recv_recover_page(bool just_read_in, buf_block_t* block) lsn values of page */ mtr.discard_modifications(); - - mtr_commit(&mtr); + mtr.commit(); ib_time_t time = ut_time(); @@ -1987,6 +1927,7 @@ recv_recover_page(bool just_read_in, buf_block_t* block) recv_max_page_lsn = page_lsn; } + ut_ad(recv_addr->state == RECV_BEING_PROCESSED); recv_addr->state = RECV_PROCESSED; ut_a(recv_sys->n_addrs > 0); @@ -1997,52 +1938,76 @@ recv_recover_page(bool just_read_in, buf_block_t* block) INNODB_EXTEND_TIMEOUT_INTERVAL, "To recover: " ULINTPF " pages from log", n); } } +} + +/** Apply any buffered redo log to a page that was just read from a data file. +@param[in,out] bpage buffer pool page */ +void recv_recover_page(buf_page_t* bpage) +{ + mtr_t mtr; + mtr.start(); + mtr.set_log_mode(MTR_LOG_NONE); + + ut_ad(buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); + buf_block_t* block = reinterpret_cast<buf_block_t*>(bpage); + + /* Move the ownership of the x-latch on the page to + this OS thread, so that we can acquire a second + x-latch on it. This is needed for the operations to + the page to pass the debug checks. */ + rw_lock_x_lock_move_ownership(&block->lock); + buf_block_dbg_add_level(block, SYNC_NO_ORDER_CHECK); + ibool success = buf_page_get_known_nowait( + RW_X_LATCH, block, BUF_KEEP_OLD, + __FILE__, __LINE__, &mtr); + ut_a(success); + + mutex_enter(&recv_sys->mutex); + if (!recv_sys->apply_log_recs) { + } else if (recv_addr_t* recv_addr = recv_get_fil_addr_struct( + bpage->id.space(), bpage->id.page_no())) { + switch (recv_addr->state) { + case RECV_BEING_PROCESSED: + case RECV_PROCESSED: + break; + default: + recv_recover_page(block, mtr, recv_addr); + goto func_exit; + } + } + mtr.commit(); +func_exit: mutex_exit(&recv_sys->mutex); + ut_ad(mtr.has_committed()); } /** Reads in pages which have hashed log records, from an area around a given page number. -@param[in] page_id page id -@return number of pages found */ -static ulint recv_read_in_area(const page_id_t page_id) +@param[in] page_id page id */ +static void recv_read_in_area(const page_id_t page_id) { - recv_addr_t* recv_addr; ulint page_nos[RECV_READ_AHEAD_AREA]; - ulint low_limit; - ulint n; - - low_limit = page_id.page_no() + ulint page_no = page_id.page_no() - (page_id.page_no() % RECV_READ_AHEAD_AREA); + ulint* p = page_nos; - n = 0; - - for (ulint page_no = low_limit; - page_no < low_limit + RECV_READ_AHEAD_AREA; - page_no++) { - - recv_addr = recv_get_fil_addr_struct(page_id.space(), page_no); - - const page_id_t cur_page_id(page_id.space(), page_no); - - if (recv_addr && !buf_page_peek(cur_page_id)) { - - mutex_enter(&(recv_sys->mutex)); - - if (recv_addr->state == RECV_NOT_PROCESSED) { - recv_addr->state = RECV_BEING_READ; - - page_nos[n] = page_no; - - n++; - } - - mutex_exit(&(recv_sys->mutex)); + for (const ulint up_limit = page_no + RECV_READ_AHEAD_AREA; + page_no < up_limit; page_no++) { + recv_addr_t* recv_addr = recv_get_fil_addr_struct( + page_id.space(), page_no); + if (recv_addr + && recv_addr->state == RECV_NOT_PROCESSED + && !buf_page_peek(page_id_t(page_id.space(), page_no))) { + recv_addr->state = RECV_BEING_READ; + *p++ = page_no; } } - buf_read_recv_pages(FALSE, page_id.space(), page_nos, n); - return(n); + mutex_exit(&recv_sys->mutex); + buf_read_recv_pages(FALSE, page_id.space(), page_nos, + ulint(p - page_nos)); + mutex_enter(&recv_sys->mutex); } /** Apply the hash table of stored log records to persistent data pages. @@ -2106,55 +2071,49 @@ void recv_apply_hashed_log_recs(bool last_batch) } } + mtr_t mtr; + for (ulint i = 0; i < hash_get_n_cells(recv_sys->addr_hash); i++) { for (recv_addr_t* recv_addr = static_cast<recv_addr_t*>( HASH_GET_FIRST(recv_sys->addr_hash, i)); recv_addr; recv_addr = static_cast<recv_addr_t*>( HASH_GET_NEXT(addr_hash, recv_addr))) { - - if (recv_addr->state == RECV_DISCARDED - || !UT_LIST_GET_LEN(recv_addr->rec_list)) { -not_found: + if (!UT_LIST_GET_LEN(recv_addr->rec_list)) { +ignore: ut_a(recv_sys->n_addrs); recv_sys->n_addrs--; continue; } - fil_space_t* space = fil_space_acquire_for_io( - recv_addr->space); - if (!space) { - goto not_found; + switch (recv_addr->state) { + case RECV_BEING_READ: + case RECV_BEING_PROCESSED: + case RECV_PROCESSED: + continue; + case RECV_DISCARDED: + goto ignore; + case RECV_NOT_PROCESSED: + break; } - const page_id_t page_id(recv_addr->space, - recv_addr->page_no); - const ulint zip_size = space->zip_size(); - - if (recv_addr->state == RECV_NOT_PROCESSED) { - mutex_exit(&recv_sys->mutex); - - if (buf_page_peek(page_id)) { - mtr_t mtr; - mtr.start(); - - buf_block_t* block = buf_page_get( - page_id, zip_size, - RW_X_LATCH, &mtr); - - buf_block_dbg_add_level( - block, SYNC_NO_ORDER_CHECK); - - recv_recover_page(FALSE, block); - mtr.commit(); - } else { - recv_read_in_area(page_id); - } - - mutex_enter(&recv_sys->mutex); + const page_id_t page_id(recv_addr->space, + recv_addr->page_no); + + mtr.start(); + mtr.set_log_mode(MTR_LOG_NONE); + if (buf_block_t* block = buf_page_get_gen( + page_id, 0, RW_X_LATCH, + NULL, BUF_GET_IF_IN_POOL, + __FILE__, __LINE__, &mtr, NULL)) { + buf_block_dbg_add_level( + block, SYNC_NO_ORDER_CHECK); + recv_recover_page(block, mtr, recv_addr); + ut_ad(mtr.has_committed()); + } else { + mtr.commit(); + recv_read_in_area(page_id); } - - space->release_for_io(); } } @@ -2395,6 +2354,15 @@ recv_report_corrupt_log( return(true); } +/** Report a MLOG_INDEX_LOAD operation. +@param[in] space_id tablespace identifier */ +ATTRIBUTE_COLD static void recv_mlog_index_load(ulint space_id) +{ + if (log_optimized_ddl_op) { + log_optimized_ddl_op(space_id); + } +} + /** Parse log records from a buffer and optionally store them to a hash table to wait merging to file pages. @param[in] checkpoint_lsn the LSN of the latest checkpoint @@ -2416,6 +2384,7 @@ bool recv_parse_log_recs(lsn_t checkpoint_lsn, store_t store, bool apply) byte* body; ut_ad(log_mutex_own()); + ut_ad(mutex_own(&recv_sys->mutex)); ut_ad(recv_sys->parse_start_lsn != 0); loop: ptr = recv_sys->buf + recv_sys->recovered_offset; @@ -2550,9 +2519,7 @@ loop: /* fall through */ case MLOG_INDEX_LOAD: if (type == MLOG_INDEX_LOAD) { - if (log_optimized_ddl_op) { - log_optimized_ddl_op(space); - } + recv_mlog_index_load(space); } /* fall through */ case MLOG_FILE_NAME: @@ -2706,10 +2673,7 @@ corrupted_log: break; #endif /* UNIV_LOG_LSN_DEBUG */ case MLOG_INDEX_LOAD: - /* Mariabackup FIXME: Report an error - when encountering MLOG_INDEX_LOAD on - --prepare or already on --backup. */ - ut_a(srv_operation == SRV_OPERATION_NORMAL); + recv_mlog_index_load(space); break; case MLOG_FILE_NAME: case MLOG_FILE_DELETE: @@ -2983,6 +2947,8 @@ recv_scan_log_recs( *group_scanned_lsn = scanned_lsn; + mutex_enter(&recv_sys->mutex); + if (more_data && !recv_sys->found_corrupt_log) { /* Try to parse more log records */ @@ -2992,7 +2958,8 @@ recv_scan_log_recs( || recv_sys->found_corrupt_fs || recv_sys->mlog_checkpoint_lsn == recv_sys->recovered_lsn); - return(true); + finished = true; + goto func_exit; } if (*store_to_hash != STORE_NO @@ -3013,6 +2980,8 @@ recv_scan_log_recs( } } +func_exit: + mutex_exit(&recv_sys->mutex); return(finished); } @@ -3657,8 +3626,8 @@ recv_reset_logs( log_sys.lsn = ut_uint64_align_up(lsn, OS_FILE_LOG_BLOCK_SIZE); - log_sys.log.lsn = log_sys.lsn; - log_sys.log.lsn_offset = LOG_FILE_HDR_SIZE; + log_sys.log.set_lsn(log_sys.lsn); + log_sys.log.set_lsn_offset(LOG_FILE_HDR_SIZE); log_sys.buf_next_to_write = 0; log_sys.write_lsn = log_sys.lsn; diff --git a/storage/innobase/os/os0proc.cc b/storage/innobase/os/os0proc.cc index 7b8e4bde442..33e65484003 100644 --- a/storage/innobase/os/os0proc.cc +++ b/storage/innobase/os/os0proc.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2019, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -25,6 +26,9 @@ Created 9/30/1995 Heikki Tuuri *******************************************************/ #include "univ.i" +#ifdef HAVE_LINUX_LARGE_PAGES +# include "mysqld.h" +#endif /* FreeBSD for example has only MAP_ANON, Linux has MAP_ANONYMOUS and MAP_ANON but MAP_ANON is marked as deprecated */ @@ -38,12 +42,6 @@ MAP_ANON but MAP_ANON is marked as deprecated */ system with os_mem_alloc_large(). */ Atomic_counter<ulint> os_total_large_mem_allocated; -/** Whether to use large pages in the buffer pool */ -my_bool os_use_large_pages; - -/** Large page size. This may be a boot-time option on some platforms */ -uint os_large_page_size; - /** Converts the current process id to a number. @return process id as a number */ ulint @@ -66,18 +64,18 @@ os_mem_alloc_large( { void* ptr; ulint size; -#if defined HAVE_LINUX_LARGE_PAGES && defined UNIV_LINUX +#ifdef HAVE_LINUX_LARGE_PAGES int shmid; struct shmid_ds buf; - if (!os_use_large_pages || !os_large_page_size) { + if (!my_use_large_pages || !opt_large_page_size) { goto skip; } - /* Align block size to os_large_page_size */ - ut_ad(ut_is_2pow(os_large_page_size)); - size = ut_2pow_round(*n + (os_large_page_size - 1), - os_large_page_size); + /* Align block size to opt_large_page_size */ + ut_ad(ut_is_2pow(opt_large_page_size)); + size = ut_2pow_round(*n + opt_large_page_size - 1, + ulint(opt_large_page_size)); shmid = shmget(IPC_PRIVATE, (size_t) size, SHM_HUGETLB | SHM_R | SHM_W); if (shmid < 0) { @@ -107,7 +105,7 @@ os_mem_alloc_large( ib::warn() << "Using conventional memory pool"; skip: -#endif /* HAVE_LINUX_LARGE_PAGES && UNIV_LINUX */ +#endif /* HAVE_LINUX_LARGE_PAGES */ #ifdef _WIN32 SYSTEM_INFO system_info; @@ -115,10 +113,8 @@ skip: /* Align block size to system page size */ ut_ad(ut_is_2pow(system_info.dwPageSize)); - /* system_info.dwPageSize is only 32-bit. Casting to ulint is required - on 64-bit Windows. */ - size = *n = ut_2pow_round(*n + (system_info.dwPageSize - 1), - (ulint) system_info.dwPageSize); + size = *n = ut_2pow_round<ulint>(*n + (system_info.dwPageSize - 1), + system_info.dwPageSize); ptr = VirtualAlloc(NULL, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); if (!ptr) { @@ -157,12 +153,12 @@ os_mem_free_large( { ut_a(os_total_large_mem_allocated >= size); -#if defined HAVE_LINUX_LARGE_PAGES && defined UNIV_LINUX - if (os_use_large_pages && os_large_page_size && !shmdt(ptr)) { +#ifdef HAVE_LINUX_LARGE_PAGES + if (my_use_large_pages && opt_large_page_size && !shmdt(ptr)) { os_total_large_mem_allocated -= size; return; } -#endif /* HAVE_LINUX_LARGE_PAGES && UNIV_LINUX */ +#endif /* HAVE_LINUX_LARGE_PAGES */ #ifdef _WIN32 /* When RELEASE memory, the size parameter must be 0. Do not use MEM_RELEASE with MEM_DECOMMIT. */ diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index ae70c29913f..542db15f3bc 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -4861,23 +4861,20 @@ page_zip_copy_recs( page_zip_compress_write_log(page_zip, page, index, mtr); } -/**********************************************************************//** -Parses a log record of compressing an index page. -@return end of log record or NULL */ -byte* -page_zip_parse_compress( -/*====================*/ - byte* ptr, /*!< in: buffer */ - byte* end_ptr,/*!< in: buffer end */ - page_t* page, /*!< out: uncompressed page */ - page_zip_des_t* page_zip)/*!< out: compressed page */ +/** Parse and optionally apply MLOG_ZIP_PAGE_COMPRESS. +@param[in] ptr log record +@param[in] end_ptr end of log +@param[in,out] block ROW_FORMAT=COMPRESSED block, or NULL for parsing only +@return end of log record +@retval NULL if the log record is incomplete */ +byte* page_zip_parse_compress(const byte* ptr, const byte* end_ptr, + buf_block_t* block) { ulint size; ulint trailer_size; ut_ad(ptr != NULL); ut_ad(end_ptr!= NULL); - ut_ad(!page == !page_zip); if (UNIV_UNLIKELY(ptr + (2 + 2) > end_ptr)) { @@ -4894,14 +4891,22 @@ page_zip_parse_compress( return(NULL); } - if (page) { - if (!page_zip || page_zip_get_size(page_zip) < size) { + if (block) { + ut_ad(buf_block_get_state(block) == BUF_BLOCK_FILE_PAGE); + page_zip_des_t* page_zip = buf_block_get_page_zip(block); + if (!page_zip || page_zip_get_size(page_zip) < size + || block->page.id.page_no() < 3) { corrupt: recv_sys->found_corrupt_log = TRUE; return(NULL); } + memset(page_zip->data, 0, page_zip_get_size(page_zip)); + mach_write_to_4(FIL_PAGE_OFFSET + + page_zip->data, block->page.id.page_no()); + mach_write_to_4(FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID + + page_zip->data, block->page.id.space()); memcpy(page_zip->data + FIL_PAGE_PREV, ptr, 4); memcpy(page_zip->data + FIL_PAGE_NEXT, ptr + 4, 4); memcpy(page_zip->data + FIL_PAGE_TYPE, ptr + 8, size); @@ -4911,14 +4916,14 @@ corrupt: memcpy(page_zip->data + page_zip_get_size(page_zip) - trailer_size, ptr + 8 + size, trailer_size); - if (UNIV_UNLIKELY(!page_zip_decompress(page_zip, page, + if (UNIV_UNLIKELY(!page_zip_decompress(page_zip, block->frame, TRUE))) { goto corrupt; } } - return(ptr + 8 + size + trailer_size); + return(const_cast<byte*>(ptr) + 8 + size + trailer_size); } #endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index 16b8fcc8536..1ac726fd8cd 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -1806,5 +1806,9 @@ exit: ib::info() << "InnoDB_FTS: inserted " << count << " records"; } + if (psort_info[0].psort_common->trx->get_flush_observer()) { + row_merge_write_redo(aux_index); + } + return(error); } diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 5c8a7c47435..dd3d1434418 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -4518,17 +4518,14 @@ row_merge_drop_table( that redo-logging of individual index pages was disabled, and the flushing of such pages to the data files was completed. @param[in] index an index tree on which redo logging was disabled */ -static -void -row_merge_write_redo( - const dict_index_t* index) +void row_merge_write_redo(const dict_index_t* index) { - mtr_t mtr; - byte* log_ptr; - ut_ad(!index->table->is_temporary()); + ut_ad(!(index->type & (DICT_SPATIAL | DICT_FTS))); + + mtr_t mtr; mtr.start(); - log_ptr = mlog_open(&mtr, 11 + 8); + byte* log_ptr = mlog_open(&mtr, 11 + 8); log_ptr = mlog_write_initial_log_record_low( MLOG_INDEX_LOAD, index->table->space_id, index->page, log_ptr, &mtr); @@ -5069,7 +5066,10 @@ func_exit: = dict_table_get_first_index(new_table); index != NULL; index = dict_table_get_next_index(index)) { - row_merge_write_redo(index); + if (!(index->type + & (DICT_FTS | DICT_SPATIAL))) { + row_merge_write_redo(index); + } } } } diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 8f0b7727b49..1225c7f35f4 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -2772,7 +2772,7 @@ row_mysql_drop_garbage_tables() btr_pcur_commit_specify_mtr(&pcur, &mtr); if (dict_load_table(table_name, true, - DICT_ERR_IGNORE_ALL)) { + DICT_ERR_IGNORE_DROP)) { row_drop_table_for_mysql(table_name, trx, SQLCOM_DROP_TABLE); trx_commit_for_mysql(trx); diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc index df5adbd5186..dbd45979c93 100644 --- a/storage/innobase/trx/trx0undo.cc +++ b/storage/innobase/trx/trx0undo.cc @@ -831,35 +831,30 @@ trx_undo_free_page( undo log page; the caller must have reserved the rollback segment mutex */ { - page_t* header_page; - page_t* undo_page; - fil_addr_t last_addr; - trx_rsegf_t* rseg_header; - ulint hist_size; const ulint space = rseg->space->id; ut_a(hdr_page_no != page_no); ut_ad(mutex_own(&(rseg->mutex))); - undo_page = trx_undo_page_get(page_id_t(space, page_no), mtr); + page_t* undo_page = trx_undo_page_get(page_id_t(space, page_no), mtr); + page_t* header_page = trx_undo_page_get(page_id_t(space, hdr_page_no), + mtr); - header_page = trx_undo_page_get(page_id_t(space, hdr_page_no), mtr); + flst_remove(TRX_UNDO_SEG_HDR + TRX_UNDO_PAGE_LIST + header_page, + TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_NODE + undo_page, mtr); - flst_remove(header_page + TRX_UNDO_SEG_HDR + TRX_UNDO_PAGE_LIST, - undo_page + TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_NODE, mtr); + fseg_free_page(TRX_UNDO_SEG_HDR + TRX_UNDO_FSEG_HEADER + header_page, + rseg->space, page_no, false, true, mtr); - fseg_free_page(header_page + TRX_UNDO_SEG_HDR + TRX_UNDO_FSEG_HEADER, - space, page_no, false, mtr); - - last_addr = flst_get_last(header_page + TRX_UNDO_SEG_HDR - + TRX_UNDO_PAGE_LIST, mtr); + const fil_addr_t last_addr = flst_get_last( + TRX_UNDO_SEG_HDR + TRX_UNDO_PAGE_LIST + header_page, mtr); rseg->curr_size--; if (in_history) { - rseg_header = trx_rsegf_get(rseg->space, rseg->page_no, mtr); - - hist_size = mtr_read_ulint(rseg_header + TRX_RSEG_HISTORY_SIZE, - MLOG_4BYTES, mtr); + trx_rsegf_t* rseg_header = trx_rsegf_get( + rseg->space, rseg->page_no, mtr); + uint32_t hist_size = mach_read_from_4( + rseg_header + TRX_RSEG_HISTORY_SIZE); ut_ad(hist_size > 0); mlog_write_ulint(rseg_header + TRX_RSEG_HISTORY_SIZE, hist_size - 1, MLOG_4BYTES, mtr); diff --git a/storage/myisam/mysql-test/storage_engine/alter_table_online.rdiff b/storage/myisam/mysql-test/storage_engine/alter_table_online.rdiff index 5ae99e2035c..58c7620f3b9 100644 --- a/storage/myisam/mysql-test/storage_engine/alter_table_online.rdiff +++ b/storage/myisam/mysql-test/storage_engine/alter_table_online.rdiff @@ -14,7 +14,7 @@ +# Also, this problem may cause a chain effect (more errors of different kinds in the test). +# ------------------------------------------- ALTER ONLINE TABLE t1 MODIFY b BIGINT <CUSTOM_COL_OPTIONS>; --ERROR 0A000: LOCK=NONE is not supported. Reason: Cannot change column type INPLACE. Try LOCK=SHARED. +-ERROR 0A000: LOCK=NONE is not supported. Reason: Cannot change column type. Try LOCK=SHARED. +ERROR 0A000: LOCK=NONE is not supported for this operation. Try LOCK=SHARED. +# ERROR: Statement ended with errno 1845, errname ER_ALTER_OPERATION_NOT_SUPPORTED (expected results: ER_ALTER_OPERATION_NOT_SUPPORTED_REASON) ALTER ONLINE TABLE t1 ENGINE=MEMORY; diff --git a/storage/myisammrg/mysql-test/storage_engine/alter_table_online.rdiff b/storage/myisammrg/mysql-test/storage_engine/alter_table_online.rdiff index 854a00cfd81..857854a6115 100644 --- a/storage/myisammrg/mysql-test/storage_engine/alter_table_online.rdiff +++ b/storage/myisammrg/mysql-test/storage_engine/alter_table_online.rdiff @@ -61,7 +61,7 @@ +# Also, this problem may cause a chain effect (more errors of different kinds in the test). +# ------------------------------------------- ALTER ONLINE TABLE t1 MODIFY b BIGINT <CUSTOM_COL_OPTIONS>; --ERROR 0A000: LOCK=NONE is not supported. Reason: Cannot change column type INPLACE. Try LOCK=SHARED. +-ERROR 0A000: LOCK=NONE is not supported. Reason: Cannot change column type. Try LOCK=SHARED. +ERROR 0A000: LOCK=NONE/SHARED is not supported for this operation. Try LOCK=EXCLUSIVE. +# ERROR: Statement ended with errno 1845, errname ER_ALTER_OPERATION_NOT_SUPPORTED (expected results: ER_ALTER_OPERATION_NOT_SUPPORTED_REASON) ALTER ONLINE TABLE t1 ENGINE=MEMORY; diff --git a/storage/rocksdb/CMakeLists.txt b/storage/rocksdb/CMakeLists.txt index 6bb8c9f6d67..e4906638f6c 100644 --- a/storage/rocksdb/CMakeLists.txt +++ b/storage/rocksdb/CMakeLists.txt @@ -5,8 +5,8 @@ MACRO(SKIP_ROCKSDB_PLUGIN msg) RETURN() ENDMACRO() -IF (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/Makefile") - SKIP_ROCKSDB_PLUGIN("Missing Makefile in rocksdb directory. Try \"git submodule update\".") +IF (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/rocksdb/CMakeLists.txt") + SKIP_ROCKSDB_PLUGIN("Missing CMakeLists.txt in rocksdb directory. Try \"git submodule update\".") ENDIF() CHECK_LIBRARY_EXISTS(rt timer_delete "" HAVE_TIMER_DELETE) diff --git a/storage/spider/spd_db_mysql.cc b/storage/spider/spd_db_mysql.cc index 6aca388ca32..b5ce958b927 100644 --- a/storage/spider/spd_db_mysql.cc +++ b/storage/spider/spd_db_mysql.cc @@ -1951,6 +1951,9 @@ int spider_db_mbase::connect( connect_retry_count--; my_sleep((ulong) connect_retry_interval); } else { +#ifdef SPIDER_NET_HAS_THD + db_conn->net.thd = NULL; +#endif if (connect_mutex) pthread_mutex_unlock(&spider_open_conn_mutex); break; diff --git a/storage/spider/spd_environ.h b/storage/spider/spd_environ.h index 5e66a912582..ded2927482b 100644 --- a/storage/spider/spd_environ.h +++ b/storage/spider/spd_environ.h @@ -25,6 +25,7 @@ #if defined(MARIADB_BASE_VERSION) && MYSQL_VERSION_ID >= 100100 #define SPIDER_SUPPORT_CREATE_OR_REPLACE_TABLE +#define SPIDER_NET_HAS_THD #endif #if defined(MARIADB_BASE_VERSION) && MYSQL_VERSION_ID >= 100211 |