diff options
Diffstat (limited to 'storage')
44 files changed, 1114 insertions, 700 deletions
diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index 489b612a17a..1be76a6b5e9 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -2,7 +2,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2014, 2017, MariaDB Corporation. +Copyright (c) 2014, 2017, MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -169,8 +169,7 @@ btr_root_block_get( if (!block) { if (index && index->table) { - index->table->is_encrypted = TRUE; - index->table->corrupted = FALSE; + index->table->file_unreadable = true; ib_push_warning(index->table->thd, DB_DECRYPTION_FAILED, "Table %s in tablespace %lu is encrypted but encryption service or" @@ -183,6 +182,7 @@ btr_root_block_get( } btr_assert_not_corrupted(block, index); + #ifdef UNIV_BTR_DEBUG if (!dict_index_is_ibuf(index)) { const page_t* root = buf_block_get_frame(block); @@ -5418,7 +5418,7 @@ btr_validate_index( page_t* root = btr_root_get(index, &mtr); - if (root == NULL && index->table->is_encrypted) { + if (root == NULL && !index->table->is_readable()) { err = DB_DECRYPTION_FAILED; mtr_commit(&mtr); return err; diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 773b03775be..157f8a37646 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -1109,7 +1109,11 @@ retry_page_get: buf_mode, file, line, mtr, &err); tree_blocks[n_blocks] = block; + /* Note that block==NULL signifies either an error or change + buffering. */ + if (err != DB_SUCCESS) { + ut_ad(block == NULL); if (err == DB_DECRYPTION_FAILED) { ib_push_warning((void *)NULL, DB_DECRYPTION_FAILED, @@ -1117,7 +1121,7 @@ retry_page_get: " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } goto func_exit; @@ -1230,7 +1234,7 @@ retry_page_get: " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } goto func_exit; @@ -1259,7 +1263,7 @@ retry_page_get: " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } goto func_exit; @@ -2057,7 +2061,7 @@ btr_cur_open_at_index_side_func( ulint offsets_[REC_OFFS_NORMAL_SIZE]; ulint* offsets = offsets_; dberr_t err = DB_SUCCESS; - + rec_offs_init(offsets_); estimate = latch_mode & BTR_ESTIMATE; @@ -2158,6 +2162,8 @@ btr_cur_open_at_index_side_func( BUF_GET, file, line, mtr, &err); tree_blocks[n_blocks] = block; + ut_ad((block != NULL) == (err == DB_SUCCESS)); + if (err != DB_SUCCESS) { if (err == DB_DECRYPTION_FAILED) { ib_push_warning((void *)NULL, @@ -2166,7 +2172,7 @@ btr_cur_open_at_index_side_func( " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } goto exit_loop; @@ -2516,6 +2522,8 @@ btr_cur_open_at_rnd_pos_func( BUF_GET, file, line, mtr, &err); tree_blocks[n_blocks] = block; + ut_ad((block != NULL) == (err == DB_SUCCESS)); + if (err != DB_SUCCESS) { if (err == DB_DECRYPTION_FAILED) { ib_push_warning((void *)NULL, @@ -2524,8 +2532,9 @@ btr_cur_open_at_rnd_pos_func( " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } + goto exit_loop; } @@ -5323,6 +5332,8 @@ btr_estimate_n_rows_in_range_on_level( NULL, BUF_GET_POSSIBLY_FREED, __FILE__, __LINE__, &mtr, &err); + ut_ad((block != NULL) == (err == DB_SUCCESS)); + if (err != DB_SUCCESS) { if (err == DB_DECRYPTION_FAILED) { ib_push_warning((void *)NULL, @@ -5331,7 +5342,7 @@ btr_estimate_n_rows_in_range_on_level( " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } mtr_commit(&mtr); @@ -5481,6 +5492,11 @@ btr_estimate_n_rows_in_range_low( &cursor, 0, __FILE__, __LINE__, &mtr); + if (!index->table->is_readable()) { + mtr_commit(&mtr); + return (0); + } + ut_ad(!page_rec_is_infimum(btr_cur_get_rec(&cursor))); /* We should count the border if there are any records to @@ -5518,6 +5534,10 @@ btr_estimate_n_rows_in_range_low( mtr_commit(&mtr); + if (!index->table->is_readable()) { + return (0); + } + mtr_start(&mtr); cursor.path_arr = path2; @@ -5997,6 +6017,12 @@ btr_estimate_number_of_different_key_vals( because otherwise our algorithm would give a wrong estimate for an index where there is just one key value. */ + if (!index->table->is_readable()) { + mtr_commit(&mtr); + goto exit_loop; + } + + page = btr_cur_get_page(&cursor); rec = page_rec_get_next(page_get_infimum_rec(page)); @@ -6080,6 +6106,7 @@ btr_estimate_number_of_different_key_vals( mtr_commit(&mtr); } +exit_loop: /* If we saw k borders between different key values on n_sample_pages leaf pages, we can estimate how many there will be in index->stat_n_leaf_pages */ diff --git a/storage/innobase/btr/btr0defragment.cc b/storage/innobase/btr/btr0defragment.cc index 06cd1315590..c33b78a7ed0 100644 --- a/storage/innobase/btr/btr0defragment.cc +++ b/storage/innobase/btr/btr0defragment.cc @@ -227,7 +227,7 @@ btr_defragment_add_index( page = buf_block_get_frame(block); } - if (page == NULL && index->table->is_encrypted) { + if (page == NULL && !index->table->is_readable()) { mtr_commit(&mtr); *err = DB_DECRYPTION_FAILED; return NULL; diff --git a/storage/innobase/btr/btr0pcur.cc b/storage/innobase/btr/btr0pcur.cc index 445aa3504b7..fdde6f5d3e7 100644 --- a/storage/innobase/btr/btr0pcur.cc +++ b/storage/innobase/btr/btr0pcur.cc @@ -418,6 +418,11 @@ btr_pcur_move_to_next_page( cursor->old_stored = false; page = btr_pcur_get_page(cursor); + + if (UNIV_UNLIKELY(!page)) { + return; + } + next_page_no = btr_page_get_next(page, mtr); ut_ad(next_page_no != FIL_NULL); @@ -438,6 +443,10 @@ btr_pcur_move_to_next_page( block->page.size, mode, btr_pcur_get_btr_cur(cursor)->index, mtr); + if (UNIV_UNLIKELY(!next_block)) { + return; + } + next_page = buf_block_get_frame(next_block); #ifdef UNIV_BTR_DEBUG ut_a(page_is_comp(next_page) == page_is_comp(page)); diff --git a/storage/innobase/btr/btr0scrub.cc b/storage/innobase/btr/btr0scrub.cc index 307d6d0ec6c..f427a137c4d 100644 --- a/storage/innobase/btr/btr0scrub.cc +++ b/storage/innobase/btr/btr0scrub.cc @@ -140,6 +140,7 @@ btr_scrub_lock_dict_func(ulint space_id, bool lock_to_close_table, } else { return false; } + os_thread_sleep(250000); time_t now = time(0); @@ -571,7 +572,7 @@ btr_scrub_table_needs_scrubbing( return false; } - if (table->corrupted) { + if (!table->is_readable()) { return false; } diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index a1efa446f76..1346a8ece4a 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1507,8 +1507,6 @@ buf_block_init( block->page.state = BUF_BLOCK_NOT_USED; block->page.buf_fix_count = 0; block->page.io_fix = BUF_IO_NONE; - block->page.flush_observer = NULL; - block->page.key_version = 0; block->page.encrypted = false; block->page.real_size = 0; block->page.write_size = 0; @@ -3794,7 +3792,6 @@ buf_page_get_zip( ibool discard_attempted = FALSE; ibool must_read; buf_pool_t* buf_pool = buf_pool_get(page_id); - buf_page_t* rpage = NULL; buf_pool->stat.n_page_gets++; @@ -3813,7 +3810,18 @@ lookup: /* Page not in buf_pool: needs to be read from file */ ut_ad(!hash_lock); - buf_read_page(page_id, page_size, &rpage); + dberr_t err = buf_read_page(page_id, page_size); + + if (err != DB_SUCCESS) { + ib::error() + << "Reading compressed page " + << page_id + << " failed with error: " + << ut_strerr(err); + + goto err_exit; + } + #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG ut_a(++buf_dbg_counter % 5771 || buf_validate()); @@ -4290,7 +4298,6 @@ loop: } if (block == NULL) { - buf_page_t* bpage=NULL; /* Page not in buf_pool: needs to be read from file */ @@ -4348,7 +4355,18 @@ loop: return(NULL); } - if (buf_read_page(page_id, page_size, &bpage)) { + /* Call path is buf_read_page() -> buf_read_page_low() + (_fil_io()) -> buf_page_io_complete() -> + buf_decrypt_after_read() here fil_space_t* is used + and we decrypt -> buf_page_check_corrupt() where + page checksums are compared. Decryption/decompression + is handled lower level, error handling is handled on lower + level, here we need only to know is page really corrupted + or encrypted page with correct checksum. */ + + dberr_t local_err = buf_read_page(page_id, page_size); + + if (local_err == DB_SUCCESS) { buf_read_ahead_random(page_id, page_size, ibuf_inside(mtr)); @@ -4356,49 +4374,32 @@ loop: } else if (retries < BUF_PAGE_READ_MAX_RETRIES) { ++retries; - bool corrupted = false; - - if (bpage) { - corrupted = buf_page_check_corrupt(bpage); - } - - /* Do not try again for encrypted pages */ - if (corrupted && bpage->encrypted) { - BPageMutex* pmutex = buf_page_get_mutex(bpage); - - buf_pool = buf_pool_from_bpage(bpage); - buf_pool_mutex_enter(buf_pool); - mutex_enter(pmutex); - - ut_ad(buf_pool->n_pend_reads > 0); - my_atomic_addlint(&buf_pool->n_pend_reads, -1); - buf_page_set_io_fix(bpage, BUF_IO_NONE); - mutex_exit(pmutex); - buf_LRU_free_page(bpage, true); - buf_pool_mutex_exit(buf_pool); - rw_lock_x_unlock_gen(&((buf_block_t*) bpage)->lock, - BUF_IO_READ); - - if (err) { - *err = DB_DECRYPTION_FAILED; - } - - return (NULL); - } + ib::info() << "Retry: " << retries << "/" << BUF_PAGE_READ_MAX_RETRIES; DBUG_EXECUTE_IF( "innodb_page_corruption_retries", retries = BUF_PAGE_READ_MAX_RETRIES; ); } else { - bool corrupted = false; + if (err) { + *err = local_err; + } + + /* Encrypted pages that are not corrupted are marked + as encrypted and that fact is later pushed to + user thread. */ + if (local_err == DB_DECRYPTION_FAILED) { + return (NULL); + } - if (bpage) { - corrupted = buf_page_check_corrupt(bpage); + /* Try to set table as corrupted instead of + asserting. */ + if (page_id.space() > TRX_SYS_SPACE && + dict_set_corrupted_by_space(page_id.space())) { + return (NULL); } - if (corrupted && !bpage->encrypted) { - ib::fatal() << "Unable to read page " << page_id + ib::fatal() << "Unable to read page " << page_id << " into the buffer pool after " << BUF_PAGE_READ_MAX_RETRIES << " attempts." " The most probable cause of this error may" @@ -4410,28 +4411,6 @@ loop: " innodb_force_recovery." " Please see " REFMAN " for more" " details. Aborting..."; - } else { - BPageMutex* pmutex = buf_page_get_mutex(bpage); - - buf_pool = buf_pool_from_bpage(bpage); - buf_pool_mutex_enter(buf_pool); - mutex_enter(pmutex); - - ut_ad(buf_pool->n_pend_reads > 0); - my_atomic_addlint(&buf_pool->n_pend_reads, -1); - buf_page_set_io_fix(bpage, BUF_IO_NONE); - mutex_exit(pmutex); - buf_LRU_free_page(bpage, true); - buf_pool_mutex_exit(buf_pool); - rw_lock_x_unlock_gen(&((buf_block_t*) bpage)->lock, - BUF_IO_READ); - - if (err) { - *err = DB_DECRYPTION_FAILED; - } - - return (NULL); - } } #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG @@ -5198,7 +5177,6 @@ buf_page_init_low( bpage->newest_modification = 0; bpage->oldest_modification = 0; bpage->write_size = 0; - bpage->key_version = 0; bpage->encrypted = false; bpage->real_size = 0; bpage->slot = NULL; @@ -5793,55 +5771,51 @@ buf_page_monitor( /********************************************************************//** Mark a table with the specified space pointed by bpage->id.space() corrupted. Also remove the bpage from LRU list. -@return TRUE if successful */ +@param[in,out] bpage Block */ static -ibool +void buf_mark_space_corrupt( -/*===================*/ - buf_page_t* bpage) /*!< in: pointer to the block in question */ + buf_page_t* bpage) { buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); const ibool uncompressed = (buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); ib_uint32_t space = bpage->id.space(); - ibool ret = TRUE; - - if (!bpage->encrypted) { - /* First unfix and release lock on the bpage */ - buf_pool_mutex_enter(buf_pool); - mutex_enter(buf_page_get_mutex(bpage)); - ut_ad(buf_page_get_io_fix(bpage) == BUF_IO_READ); - ut_ad(bpage->buf_fix_count == 0); + BPageMutex* block_mutex = buf_page_get_mutex(bpage); - /* Set BUF_IO_NONE before we remove the block from LRU list */ - buf_page_set_io_fix(bpage, BUF_IO_NONE); + /* First unfix and release lock on the bpage */ + buf_pool_mutex_enter(buf_pool); + mutex_enter(block_mutex); + ut_ad(buf_page_get_io_fix(bpage) == BUF_IO_READ); + ut_ad(bpage->buf_fix_count == 0); - if (uncompressed) { - rw_lock_x_unlock_gen( - &((buf_block_t*) bpage)->lock, - BUF_IO_READ); - } + /* Set BUF_IO_NONE before we remove the block from LRU list */ + buf_page_set_io_fix(bpage, BUF_IO_NONE); - mutex_exit(buf_page_get_mutex(bpage)); + if (uncompressed) { + rw_lock_x_unlock_gen( + &((buf_block_t*) bpage)->lock, + BUF_IO_READ); } - /* Find the table with specified space id, and mark it corrupted */ - if (dict_set_corrupted_by_space(space)) { - if (!bpage->encrypted) { - buf_LRU_free_one_page(bpage); - } + mutex_exit(block_mutex); + + /* If block is not encrypted find the table with specified + space id, and mark it corrupted. Encrypted tables + are marked unusable later e.g. in ::open(). */ + if (!bpage->encrypted) { + dict_set_corrupted_by_space(space); } else { - ret = FALSE; + dict_set_encrypted_by_space(space); } - if (!bpage->encrypted) { - ut_ad(buf_pool->n_pend_reads > 0); - buf_pool->n_pend_reads--; + /* After this point bpage can't be referenced. */ + buf_LRU_free_one_page(bpage); - buf_pool_mutex_exit(buf_pool); - } + ut_ad(buf_pool->n_pend_reads > 0); + buf_pool->n_pend_reads--; - return(ret); + buf_pool_mutex_exit(buf_pool); } /********************************************************************//** @@ -5849,18 +5823,28 @@ Check if page is maybe compressed, encrypted or both when we encounter corrupted page. Note that we can't be 100% sure if page is corrupted or decrypt/decompress just failed. @param[in,out] bpage Page -@return true if page corrupted, false if not */ -UNIV_INTERN -bool -buf_page_check_corrupt( - buf_page_t* bpage) +@return DB_SUCCESS if page has been read and is not corrupted, +@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted, +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but +after decryption normal page checksum does not match. +@retval DB_TABLESPACE_DELETED if accessed tablespace is not found */ +static +dberr_t +buf_page_check_corrupt(buf_page_t* bpage) { byte* dst_frame = (bpage->zip.data) ? bpage->zip.data : ((buf_block_t*) bpage)->frame; - fil_space_t* space = fil_space_acquire_silent(bpage->id.space()); + FilSpace space(bpage->id.space(), true); bool still_encrypted = false; + dberr_t err = DB_SUCCESS; bool corrupted = false; - fil_space_crypt_t* crypt_data = space ? space->crypt_data : NULL; + fil_space_crypt_t* crypt_data = NULL; + + if (!space()) { + return(DB_TABLESPACE_DELETED); + } + + crypt_data = space()->crypt_data; /* In buf_decrypt_after_read we have either decrypted the page if page post encryption checksum matches and used key_id is found @@ -5879,10 +5863,12 @@ buf_page_check_corrupt( /* If traditional checksums match, we assume that page is not anymore encrypted. */ corrupted = buf_page_is_corrupted( - true, dst_frame, bpage->size, space); + true, dst_frame, bpage->size, space()); if (!corrupted) { bpage->encrypted = false; + } else { + err = DB_PAGE_CORRUPTED; } } @@ -5893,16 +5879,17 @@ buf_page_check_corrupt( buf_page_io_complete(). */ } else if (still_encrypted || (bpage->encrypted && corrupted)) { bpage->encrypted = true; - corrupted = true; + err = DB_DECRYPTION_FAILED; ib::error() << "The page " << bpage->id << " in file " - << (space && space->name ? space->name : "NULL") + << (space()->name ? space()->name : "NULL") << " cannot be decrypted."; ib::info() << "However key management plugin or used key_version " - << bpage->key_version << " is not found or" + << mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) + << " is not found or" " used encryption algorithm or method does not match."; if (bpage->id.space() != TRX_SYS_SPACE) { @@ -5914,31 +5901,31 @@ buf_page_check_corrupt( } } - if (space) { - fil_space_release(space); - } - - return corrupted; + return (err); } /********************************************************************//** Completes an asynchronous read or write request of a file page to or from the buffer pool. -@return true if successful */ -bool +@param[in,out] bpage Page to complete +@param[in] evict whether or not to evict the page + from LRU list. +@return DB_SUCCESS if page has been read and is not corrupted, +DB_PAGE_CORRUPTED if page based on checksum check is corrupted, +DB_DECRYPTION_FAILED if page post encryption checksum matches but +after decryption normal page checksum does not match. +in write only DB_SUCCESS is possible. */ +dberr_t buf_page_io_complete( -/*=================*/ - buf_page_t* bpage, /*!< in: pointer to the block in question */ - bool evict) /*!< in: whether or not to evict the page - from LRU list. */ - + buf_page_t* bpage, + bool evict) { enum buf_io_fix io_type; buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); const ibool uncompressed = (buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); - byte* frame = NULL; - bool corrupted = false; + byte* frame = NULL; + dberr_t err = DB_SUCCESS; ut_a(buf_page_in_file(bpage)); @@ -5952,8 +5939,9 @@ buf_page_io_complete( ut_ad(io_type == BUF_IO_READ || io_type == BUF_IO_WRITE); if (io_type == BUF_IO_READ) { - ulint read_page_no; - ulint read_space_id; + ulint read_page_no = 0; + ulint read_space_id = 0; + uint key_version = 0; ut_ad(bpage->zip.data != NULL || ((buf_block_t*)bpage)->frame != NULL); @@ -5967,21 +5955,24 @@ buf_page_io_complete( if (bpage->size.is_compressed()) { frame = bpage->zip.data; - buf_pool->n_pend_unzip++; + (void) my_atomic_addlint(&buf_pool->n_pend_unzip, 1); if (uncompressed && !buf_zip_decompress((buf_block_t*) bpage, FALSE)) { - buf_pool->n_pend_unzip--; + (void) my_atomic_addlint(&buf_pool->n_pend_unzip, -1); ib::info() << "Page " << bpage->id << " zip_decompress failure."; + err = DB_PAGE_CORRUPTED; + goto database_corrupted; } - buf_pool->n_pend_unzip--; + + (void) my_atomic_addlint(&buf_pool->n_pend_unzip, -1); } else { ut_a(uncompressed); frame = ((buf_block_t*) bpage)->frame; @@ -5993,6 +5984,8 @@ buf_page_io_complete( read_page_no = mach_read_from_4(frame + FIL_PAGE_OFFSET); read_space_id = mach_read_from_4( frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + key_version = mach_read_from_4( + frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); if (bpage->id.space() == TRX_SYS_SPACE && buf_dblwr_page_inside(bpage->id.page_no())) { @@ -6016,30 +6009,30 @@ buf_page_io_complete( << ", should be " << bpage->id; } - corrupted = buf_page_check_corrupt(bpage); + err = buf_page_check_corrupt(bpage); database_corrupted: - if (corrupted) { + if (err != DB_SUCCESS) { /* Not a real corruption if it was triggered by error injection */ DBUG_EXECUTE_IF( "buf_page_import_corrupt_failure", if (bpage->id.space() > srv_undo_tablespaces_open - && bpage->id.space() != SRV_TMP_SPACE_ID - && buf_mark_space_corrupt(bpage)) { + && bpage->id.space() != SRV_TMP_SPACE_ID) { + buf_mark_space_corrupt(bpage); ib::info() << "Simulated IMPORT " "corruption"; - return(true); + return(err); } + err = DB_SUCCESS; goto page_not_corrupt; ); - if (!bpage->encrypted) { + if (err == DB_PAGE_CORRUPTED) { fil_system_enter(); fil_space_t* space = fil_space_get_by_id(bpage->id.space()); - fil_system_exit(); ib::error() << "Database page corruption on disk" @@ -6049,6 +6042,8 @@ database_corrupted: << ". You may have to recover from " << "a backup."; + fil_system_exit(); + buf_page_print(frame, bpage->size, BUF_PAGE_PRINT_NO_CRASH); @@ -6071,35 +6066,13 @@ database_corrupted: /* If page space id is larger than TRX_SYS_SPACE (0), we will attempt to mark the corresponding table as corrupted instead of crashing server */ - - if (bpage->id.space() > TRX_SYS_SPACE - && buf_mark_space_corrupt(bpage)) { - return(false); + if (bpage->id.space() > srv_undo_tablespaces_open + && bpage->id.space() != SRV_TMP_SPACE_ID) { + buf_mark_space_corrupt(bpage); + return(err); } else { - if (!bpage->encrypted) { - ib::fatal() - << "Aborting because of a" - " corrupt database page in" - " the system tablespace. Or, " - " there was a failure in" - " tagging the tablespace " - " as corrupt."; - } - - ib_push_warning(innobase_get_trx(), DB_DECRYPTION_FAILED, - "Table in tablespace %u encrypted." - "However key management plugin or used key_id %lu is not found or" - " used encryption algorithm or method does not match." - " Can't continue opening the table.", - bpage->id.space(), bpage->key_version); - - if (bpage->encrypted && bpage->id.space() > TRX_SYS_SPACE) { - buf_mark_space_corrupt(bpage); - } else { - ut_error; - } - - return(false); + ib::fatal() + << "Ending processing because of a corrupt database page."; } } } @@ -6130,7 +6103,7 @@ database_corrupted: << bpage->id.space() << " encrypted. However key " "management plugin or used " - << "key_version " << bpage->key_version + << "key_version " << key_version << "is not found or" " used encryption algorithm or method does not match." " Can't continue opening the table."; @@ -6232,7 +6205,7 @@ database_corrupted: buf_pool_mutex_exit(buf_pool); - return(true); + return(err); } /*********************************************************************//** @@ -6257,7 +6230,7 @@ buf_all_freed_instance( const buf_block_t* block = buf_chunk_not_freed(chunk); - if (UNIV_LIKELY_NULL(block) && block->page.key_version == 0) { + if (UNIV_LIKELY_NULL(block)) { ib::fatal() << "Page " << block->page.id << " still fixed or dirty"; } @@ -7406,13 +7379,11 @@ buf_page_encrypt_before_write( switch (bpage->id.page_no()) { case 0: /* Page 0 of a tablespace is not encrypted/compressed */ - ut_ad(bpage->key_version == 0); return src_frame; case TRX_SYS_PAGE_NO: if (bpage->id.space() == 0) { /* don't encrypt/compress page as it contains address to dblwr buffer */ - bpage->key_version = 0; return src_frame; } } @@ -7425,14 +7396,13 @@ buf_page_encrypt_before_write( && (!crypt_data->is_default_encryption() || srv_encrypt_tables); - if (!encrypted) { - bpage->key_version = 0; - } bool page_compressed = FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags); if (!encrypted && !page_compressed) { - /* No need to encrypt or page compress the page */ + /* No need to encrypt or page compress the page. + Clear key-version & crypt-checksum. */ + memset(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); return src_frame; } @@ -7452,10 +7422,6 @@ buf_page_encrypt_before_write( src_frame, dst_frame); - uint32_t key_version = mach_read_from_4( - dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - ut_ad(key_version == 0 || key_version >= bpage->key_version); - bpage->key_version = key_version; bpage->real_size = page_size.physical(); slot->out_buf = dst_frame = tmp; @@ -7517,15 +7483,12 @@ buf_page_decrypt_after_read(buf_page_t* bpage) bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame); buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); bool success = true; - - bpage->key_version = key_version; - if (bpage->id.page_no() == 0) { /* File header pages are not encrypted/compressed */ return (true); } - FilSpace space(bpage->id.space(), true); + FilSpace space(bpage->id.space(), false, true); /* Page is encrypted if encryption information is found from tablespace and page contains used key_version. This is true @@ -7561,6 +7524,14 @@ buf_page_decrypt_after_read(buf_page_t* bpage) if (!fil_space_verify_crypt_checksum( dst_frame, size, bpage->id.space(), bpage->id.page_no())) { + + /* Mark page encrypted in case it should + be. */ + if (key_version && space()->crypt_data && + space()->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED) { + bpage->encrypted=true; + } + return (false); } @@ -7570,7 +7541,7 @@ buf_page_decrypt_after_read(buf_page_t* bpage) ut_d(fil_page_type_validate(dst_frame)); /* decrypt using crypt_buf to dst_frame */ - byte* res = fil_space_decrypt(space, + byte* res = fil_space_decrypt(space(), slot->crypt_buf, dst_frame, &bpage->encrypted); diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index f99fc6434de..5bdf9c3c1e7 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -527,20 +527,20 @@ buf_dblwr_process() ++i, ++page_no_dblwr) { byte* page = *i; ulint space_id = page_get_space_id(page); - fil_space_t* space = fil_space_get(space_id); + FilSpace space(space_id, true); - if (space == NULL) { + if (!space()) { /* Maybe we have dropped the tablespace and this page once belonged to it: do nothing */ continue; } - fil_space_open_if_needed(space); - const ulint page_no = page_get_page_no(page); const page_id_t page_id(space_id, page_no); - if (page_no >= space->size) { + fil_space_open_if_needed(const_cast<fil_space_t*>(space())); + + if (page_no >= space()->size) { /* Do not report the warning if the tablespace is scheduled for truncation or was truncated @@ -555,7 +555,7 @@ buf_dblwr_process() continue; } - const page_size_t page_size(space->flags); + const page_size_t page_size(space()->flags); ut_ad(!buf_page_is_zeroes(page, page_size)); /* We want to ensure that for partial reads the @@ -563,13 +563,14 @@ buf_dblwr_process() memset(read_buf, 0x0, page_size.physical()); IORequest request; + IORequest write_request(IORequest::WRITE); request.dblwr_recover(); /* Read in the actual page from the file */ dberr_t err = fil_io( - request, true, - page_id, page_size, + request, true, + page_id, page_size, 0, page_size.physical(), read_buf, NULL); if (err != DB_SUCCESS) { @@ -655,8 +656,6 @@ buf_dblwr_process() /* Write the good page from the doublewrite buffer to the intended position. */ - IORequest write_request(IORequest::WRITE); - fil_io(write_request, true, page_id, page_size, 0, page_size.physical(), const_cast<byte*>(page), NULL); diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 366e92a7112..f917eff9069 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1121,7 +1121,12 @@ buf_flush_write_block_low( /* true means we want to evict this page from the LRU list as well. */ +#ifdef UNIV_DEBUG + dberr_t err = +#endif buf_page_io_complete(bpage, true); + + ut_ad(err == DB_SUCCESS); } fil_space_release(space); diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 368c1fcb9ae..afc96c0e154 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2016 MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -95,15 +95,18 @@ buffer buf_pool if it is not already there, in which case does nothing. Sets the io_fix flag and sets an exclusive lock on the buffer frame. The flag is cleared and the x-lock released by an i/o-handler thread. -@param[out] err DB_SUCCESS, DB_TABLESPACE_DELETED or - DB_TABLESPACE_TRUNCATED if we are trying - to read from a non-existent tablespace, a +@param[out] err DB_SUCCESS, DB_TABLESPACE_DELETED if we are + trying to read from a non-existent tablespace, or a tablespace which is just now being dropped, - or a tablespace which is truncated + DB_PAGE_CORRUPTED if page based on checksum + check is corrupted, or DB_DECRYPTION_FAILED + if page post encryption checksum matches but + after decryption normal page checksum does not match. @param[in] sync true if synchronous aio is desired @param[in] type IO type, SIMULATED, IGNORE_MISSING @param[in] mode BUF_READ_IBUF_PAGES_ONLY, ..., -@param[in] page_id page id +@param[in] page_id Page id +@param[in] page_size Page size @param[in] unzip true=request uncompressed page @return 1 if a read request was queued, 0 if the page already resided in buf_pool, or if the page is in the doublewrite buffer blocks in @@ -118,11 +121,9 @@ buf_read_page_low( ulint mode, const page_id_t& page_id, const page_size_t& page_size, - bool unzip, - buf_page_t** rbpage) /*!< out: page */ + bool unzip) { buf_page_t* bpage; - *err = DB_SUCCESS; if (page_id.space() == TRX_SYS_SPACE @@ -213,19 +214,13 @@ buf_read_page_low( if (sync) { /* The i/o is already completed when we arrive from fil_read */ + *err = buf_page_io_complete(bpage); - if (!buf_page_io_complete(bpage)) { - if (rbpage) { - *rbpage = bpage; - } + if (*err != DB_SUCCESS) { return(0); } } - if (rbpage) { - *rbpage = bpage; - } - return(1); } @@ -256,7 +251,7 @@ buf_read_ahead_random( ulint ibuf_mode; ulint count; ulint low, high; - dberr_t err; + dberr_t err = DB_SUCCESS; ulint i; const ulint buf_read_ahead_random_area = BUF_READ_AHEAD_AREA(buf_pool); @@ -369,19 +364,28 @@ read_ahead: const page_id_t cur_page_id(page_id.space(), i); if (!ibuf_bitmap_page(cur_page_id, page_size)) { - buf_page_t* rpage = NULL; count += buf_read_page_low( &err, false, IORequest::DO_NOT_WAKE, ibuf_mode, - cur_page_id, page_size, false, &rpage); + cur_page_id, page_size, false); - if (err == DB_TABLESPACE_DELETED) { + switch(err) { + case DB_SUCCESS: + break; + case DB_TABLESPACE_DELETED: ib::warn() << "Random readahead trying to" " access page " << cur_page_id << " in nonexisting or" " being-dropped tablespace"; break; + case DB_DECRYPTION_FAILED: + ib::error() + << "Random readahead failed to decrypt page " + << cur_page_id; + break; + default: + ut_error; } } } @@ -412,17 +416,23 @@ read_ahead: buffer buf_pool if it is not already there. Sets the io_fix flag and sets an exclusive lock on the buffer frame. The flag is cleared and the x-lock released by the i/o-handler thread. + @param[in] page_id page id @param[in] page_size page size -@return TRUE if page has been read in, FALSE in case of failure */ -ibool + +@return DB_SUCCESS if page has been read and is not corrupted, +@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted, +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but +after decryption normal page checksum does not match. +@retval DB_TABLESPACE_DELETED if tablespace .ibd file is missing */ +UNIV_INTERN +dberr_t buf_read_page( const page_id_t& page_id, - const page_size_t& page_size, - buf_page_t** bpage) /*!< out: page */ + const page_size_t& page_size) { ulint count; - dberr_t err; + dberr_t err = DB_SUCCESS; /* We do synchronous IO because our AIO completion code is sub-optimal. See buf_page_io_complete(), we have to @@ -432,19 +442,17 @@ buf_read_page( count = buf_read_page_low( &err, true, - 0, BUF_READ_ANY_PAGE, page_id, page_size, false, bpage); + 0, BUF_READ_ANY_PAGE, page_id, page_size, false); + /* Page corruption and decryption failures are already reported + in above function. */ srv_stats.buf_pool_reads.add(count); - if (err == DB_TABLESPACE_DELETED) { - ib::error() << "trying to read page " << page_id - << " in nonexisting or being-dropped tablespace"; - } /* Increment number of I/O operations used for LRU policy. */ buf_LRU_stat_inc_io(); - return(count > 0); + return(err); } /** High-level function which reads a page asynchronously from a file to the @@ -453,23 +461,38 @@ an exclusive lock on the buffer frame. The flag is cleared and the x-lock released by the i/o-handler thread. @param[in] page_id page id @param[in] page_size page size -@param[in] sync true if synchronous aio is desired -@return TRUE if page has been read in, FALSE in case of failure */ -ibool +@param[in] sync true if synchronous aio is desired */ +void buf_read_page_background( const page_id_t& page_id, const page_size_t& page_size, bool sync) { ulint count; - dberr_t err; - buf_page_t* rbpage = NULL; + dberr_t err = DB_SUCCESS; count = buf_read_page_low( &err, sync, IORequest::DO_NOT_WAKE | IORequest::IGNORE_MISSING, BUF_READ_ANY_PAGE, - page_id, page_size, false, &rbpage); + page_id, page_size, false); + + switch(err) { + case DB_SUCCESS: + break; + case DB_TABLESPACE_DELETED: + ib::error() << "Background page read trying to read page " << page_id + << " in nonexisting or being-dropped tablespace"; + break; + + case DB_DECRYPTION_FAILED: + ib::error() + << "Background Page read failed to decrypt page " + << page_id; + break; + default: + ut_error; + } srv_stats.buf_pool_reads.add(count); @@ -479,8 +502,6 @@ buf_read_page_background( buffer pool. Since this function is called from buffer pool load these IOs are deliberate and are not part of normal workload we can ignore these in our heuristics. */ - - return(count > 0); } /** Applies linear read-ahead if in the buf_pool the page is a border page of @@ -525,7 +546,7 @@ buf_read_ahead_linear( ulint new_offset; ulint fail_count; ulint low, high; - dberr_t err; + dberr_t err = DB_SUCCESS; ulint i; const ulint buf_read_ahead_linear_area = BUF_READ_AHEAD_AREA(buf_pool); @@ -730,19 +751,30 @@ buf_read_ahead_linear( const page_id_t cur_page_id(page_id.space(), i); if (!ibuf_bitmap_page(cur_page_id, page_size)) { - buf_page_t* rpage = NULL; count += buf_read_page_low( &err, false, IORequest::DO_NOT_WAKE, - ibuf_mode, cur_page_id, page_size, false, &rpage); + ibuf_mode, cur_page_id, page_size, false); - if (err == DB_TABLESPACE_DELETED) { + switch(err) { + case DB_SUCCESS: + break; + case DB_TABLESPACE_DELETED: ib::warn() << "linear readahead trying to" " access page " << page_id_t(page_id.space(), i) << " in nonexisting or being-dropped" " tablespace"; + break; + + case DB_DECRYPTION_FAILED: + ib::error() + << "Linear readahead failed to decrypt page " + << page_id_t(page_id.space(), i); + break; + default: + ut_error; } } } @@ -795,10 +827,9 @@ buf_read_ibuf_merge_pages( for (ulint i = 0; i < n_stored; i++) { const page_id_t page_id(space_ids[i], page_nos[i]); + dberr_t err = DB_SUCCESS; buf_pool_t* buf_pool = buf_pool_get(page_id); - buf_page_t* rpage = NULL; - bool found; const page_size_t page_size(fil_space_get_page_size( space_ids[i], &found)); @@ -816,19 +847,28 @@ buf_read_ibuf_merge_pages( os_thread_sleep(500000); } - dberr_t err; - buf_read_page_low(&err, sync && (i + 1 == n_stored), 0, BUF_READ_ANY_PAGE, page_id, page_size, - true, &rpage); + true); - if (err == DB_TABLESPACE_DELETED) { + switch(err) { + case DB_SUCCESS: + break; + case DB_TABLESPACE_DELETED: /* We have deleted or are deleting the single-table tablespace: remove the entries for that page */ ibuf_merge_or_delete_for_page(NULL, page_id, &page_size, FALSE); + break; + case DB_DECRYPTION_FAILED: + ib::error() + << "Failed to decrypt insert buffer page " + << page_id; + break; + default: + ut_error; } } @@ -856,7 +896,7 @@ buf_read_recv_pages( ulint n_stored) { ulint count; - dberr_t err; + dberr_t err = DB_SUCCESS; ulint i; fil_space_t* space = fil_space_get(space_id); @@ -872,7 +912,6 @@ buf_read_recv_pages( for (i = 0; i < n_stored; i++) { buf_pool_t* buf_pool; const page_id_t cur_page_id(space_id, page_nos[i]); - buf_page_t* rpage = NULL; count = 0; @@ -899,13 +938,19 @@ buf_read_recv_pages( &err, true, 0, BUF_READ_ANY_PAGE, - cur_page_id, page_size, true, &rpage); + cur_page_id, page_size, true); } else { buf_read_page_low( &err, false, IORequest::DO_NOT_WAKE, BUF_READ_ANY_PAGE, - cur_page_id, page_size, true, &rpage); + cur_page_id, page_size, true); + } + + if (err == DB_DECRYPTION_FAILED) { + ib::error() + << "Recovery failed to decrypt read page " + << cur_page_id; } } diff --git a/storage/innobase/dict/dict0crea.cc b/storage/innobase/dict/dict0crea.cc index c1bd5c2d368..1c965d6254d 100644 --- a/storage/innobase/dict/dict0crea.cc +++ b/storage/innobase/dict/dict0crea.cc @@ -890,7 +890,7 @@ dict_create_index_tree_step( mtr_start(&mtr); - const bool missing = index->table->ibd_file_missing + const bool missing = !index->table->is_readable() || dict_table_is_discarded(index->table); if (!missing) { @@ -963,7 +963,7 @@ dict_create_index_tree_in_mem( /* Currently this function is being used by temp-tables only. Import/Discard of temp-table is blocked and so this assert. */ - ut_ad(index->table->ibd_file_missing == 0 + ut_ad(index->table->file_unreadable == 0 && !dict_table_is_discarded(index->table)); page_no = btr_create( diff --git a/storage/innobase/dict/dict0defrag_bg.cc b/storage/innobase/dict/dict0defrag_bg.cc index 016b774217f..b7830e22153 100644 --- a/storage/innobase/dict/dict0defrag_bg.cc +++ b/storage/innobase/dict/dict0defrag_bg.cc @@ -320,19 +320,8 @@ dict_stats_save_defrag_stats( { dberr_t ret; - if (index->table->ibd_file_missing) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Cannot save defragment stats because " - ".ibd file is missing.\n"); - return (DB_TABLESPACE_DELETED); - } - if (dict_index_is_corrupted(index)) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Cannot save defragment stats because " - "index is corrupted.\n"); - return(DB_CORRUPTION); + if (!index->table->is_readable()) { + return (dict_stats_report_error(index->table, true)); } if (dict_index_is_univ(index)) { diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index c6420129164..99e75524e49 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -1178,12 +1178,25 @@ dict_table_open_on_name( if (table != NULL) { - /* If table is encrypted return table */ + /* If table is encrypted or corrupted */ if (ignore_err == DICT_ERR_IGNORE_NONE - && table->is_encrypted) { + && !table->is_readable()) { /* Make life easy for drop table. */ dict_table_prevent_eviction(table); + if (table->corrupted) { + + if (!dict_locked) { + mutex_exit(&dict_sys->mutex); + } + + ib::error() << + "Table " << table->name << " is corrupted. Please " + "drop the table and recreate."; + + DBUG_RETURN(NULL); + } + if (table->can_be_evicted) { dict_move_to_mru(table); } @@ -1194,23 +1207,7 @@ dict_table_open_on_name( mutex_exit(&dict_sys->mutex); } - DBUG_RETURN(table); - } - /* If table is corrupted, return NULL */ - else if (ignore_err == DICT_ERR_IGNORE_NONE - && table->corrupted) { - /* Make life easy for drop table. */ - dict_table_prevent_eviction(table); - if (!dict_locked) { - mutex_exit(&dict_sys->mutex); - } - - ib::info() << "Table " - << table->name - << " is corrupted. Please drop the table" - " and recreate it"; - - DBUG_RETURN(NULL); + DBUG_RETURN (table); } if (table->can_be_evicted) { @@ -6067,11 +6064,29 @@ dict_set_corrupted_by_space( /* mark the table->corrupted bit only, since the caller could be too deep in the stack for SYS_INDEXES update */ - table->corrupted = TRUE; + table->corrupted = true; + table->file_unreadable = true; return(TRUE); } + +/** Flags a table with specified space_id encrypted in the data dictionary +cache +@param[in] space_id Tablespace id */ +UNIV_INTERN +void +dict_set_encrypted_by_space(ulint space_id) +{ + dict_table_t* table; + + table = dict_find_single_table_by_space(space_id); + + if (table) { + table->file_unreadable = true; + } +} + /**********************************************************************//** Flags an index corrupted both in the data dictionary cache and in the SYS_INDEXES */ @@ -6581,7 +6596,8 @@ dict_table_schema_check( } } - if (table->ibd_file_missing) { + if (!table->is_readable() && + fil_space_get(table->space) == NULL) { /* missing tablespace */ ut_snprintf(errstr, errstr_sz, diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index da72126793f..7b2239d80ff 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -2430,7 +2430,7 @@ dict_load_indexes( dict_mem_index_free(index); goto func_exit; } else if (index->page == FIL_NULL - && !table->ibd_file_missing + && table->is_readable() && (!(index->type & DICT_FTS))) { ib::error() << "Trying to load index " << index->name @@ -2551,7 +2551,7 @@ dict_load_table_low(table_name_t& name, const rec_t* rec, dict_table_t** table) *table = dict_mem_table_create( name.m_name, space_id, n_cols + n_v_col, n_v_col, flags, flags2); (*table)->id = table_id; - (*table)->ibd_file_missing = FALSE; + (*table)->file_unreadable = false; return(NULL); } @@ -2709,7 +2709,7 @@ dict_load_tablespace( if (table->flags2 & DICT_TF2_DISCARDED) { ib::warn() << "Tablespace for table " << table->name << " is set as discarded."; - table->ibd_file_missing = TRUE; + table->file_unreadable = true; return; } @@ -2754,7 +2754,7 @@ dict_load_tablespace( if (err != DB_SUCCESS) { /* We failed to find a sensible tablespace file */ - table->ibd_file_missing = TRUE; + table->file_unreadable = true; } ut_free(filepath); @@ -2886,9 +2886,10 @@ err_exit: were not allowed while the table is being locked by a transaction. */ dict_err_ignore_t index_load_err = !(ignore_err & DICT_ERR_IGNORE_RECOVER_LOCK) - && table->ibd_file_missing + && !table->is_readable() ? DICT_ERR_IGNORE_ALL : ignore_err; + err = dict_load_indexes(table, heap, index_load_err); if (err == DB_INDEX_CORRUPT) { @@ -2947,7 +2948,7 @@ err_exit: of the error condition, since the user may want to dump data from the clustered index. However we load the foreign key information only if all indexes were loaded. */ - if (!cached || table->ibd_file_missing) { + if (!cached || !table->is_readable()) { /* Don't attempt to load the indexes from disk. */ } else if (err == DB_SUCCESS) { err = dict_load_foreigns(table->name.m_name, NULL, @@ -2981,12 +2982,12 @@ err_exit: table = NULL; } else if (dict_index_is_corrupted(index) - && !table->ibd_file_missing) { + && table->is_readable()) { /* It is possible we force to load a corrupted clustered index if srv_load_corrupted is set. Mark the table as corrupted in this case */ - table->corrupted = TRUE; + table->corrupted = true; } } @@ -2995,7 +2996,7 @@ func_exit: ut_ad(!table || ignore_err != DICT_ERR_IGNORE_NONE - || table->ibd_file_missing + || !table->is_readable() || !table->corrupted); if (table && table->fts) { diff --git a/storage/innobase/dict/dict0mem.cc b/storage/innobase/dict/dict0mem.cc index 7a6f09569a6..230706976dc 100644 --- a/storage/innobase/dict/dict0mem.cc +++ b/storage/innobase/dict/dict0mem.cc @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index 537a70c2069..d0337116393 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2009, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -913,10 +913,11 @@ dict_stats_update_transient_for_index( index->stat_n_leaf_pages = size; - /* We don't handle the return value since it will be false - only when some thread is dropping the table and we don't - have to empty the statistics of the to be dropped index */ - btr_estimate_number_of_different_key_vals(index); + /* Do not continue if table decryption has failed or + table is already marked as corrupted. */ + if (index->is_readable()) { + btr_estimate_number_of_different_key_vals(index); + } } } @@ -967,8 +968,9 @@ dict_stats_update_transient( continue; } - /* Do not continue if table decryption has failed. */ - if (index->table->is_encrypted) { + /* Do not continue if table decryption has failed or + table is already marked as corrupted. */ + if (!index->is_readable()) { break; } @@ -2419,6 +2421,61 @@ dict_stats_save_index_stat( return(ret); } +/** Report error if statistic update for a table failed because +.ibd file is missing, table decryption failed or table is corrupted. +@param[in,out] table Table +@param[in] defragment true if statistics is for defragment +@return DB_DECRYPTION_FAILED, DB_TABLESPACE_DELETED or DB_CORRUPTION +@retval DB_DECRYPTION_FAILED if decryption of the table failed +@retval DB_TABLESPACE_DELETED if .ibd file is missing +@retval DB_CORRUPTION if table is marked as corrupted */ +dberr_t +dict_stats_report_error( + dict_table_t* table, + bool defragment) +{ + dberr_t err; + + FilSpace space(table->space); + + if (space()) { + if (table->corrupted) { + ib::info() + << "Cannot save" + << (defragment ? " defragment" : " ") + << "statistics because " + " table " << table->name + << " in file " << space()->chain.start->name + << " is corrupted."; + err = DB_CORRUPTION; + } else { + ib::info() + << "Cannot save" + << (defragment ? " defragment" : " ") + << "statistics because " + " table " << table->name + << " in file " << space()->chain.start->name + << " can't be decrypted."; + err = DB_DECRYPTION_FAILED; + } + } else { + ib::info() + << "Cannot save" + << (defragment ? " defragment" : " ") + << "statistics for " + " table " << table->name + << " because .ibd file is missing." + " For help, please " + "refer to " REFMAN "innodb-troubleshooting.html."; + err = DB_TABLESPACE_DELETED; + } + + dict_stats_empty_table(table, defragment); + + return (err); +} + + /** Save the table's statistics into the persistent statistics storage. @param[in] table_orig table whose stats to save @param[in] only_for_index if this is non-NULL, then stats for indexes @@ -2438,6 +2495,11 @@ dict_stats_save( char db_utf8[MAX_DB_UTF8_LEN]; char table_utf8[MAX_TABLE_UTF8_LEN]; + if (table_orig->is_readable()) { + } else { + return (dict_stats_report_error(table_orig)); + } + table = dict_stats_snapshot_create(table_orig); dict_fs2utf8(table->name.m_name, db_utf8, sizeof(db_utf8), @@ -3160,15 +3222,8 @@ dict_stats_update( { ut_ad(!mutex_own(&dict_sys->mutex)); - if (table->ibd_file_missing) { - - ib::warn() << "Cannot calculate statistics for table " - << table->name - << " because the .ibd file is missing. " - << TROUBLESHOOTING_MSG; - - dict_stats_empty_table(table, true); - return(DB_TABLESPACE_DELETED); + if (!table->is_readable()) { + return (dict_stats_report_error(table)); } else if (srv_force_recovery >= SRV_FORCE_NO_IBUF_MERGE) { /* If we have set a high innodb_force_recovery level, do not calculate statistics, as a badly corrupted index can @@ -3860,7 +3915,7 @@ if the persistent stats do not exist. */ dberr_t dict_stats_rename_index( /*====================*/ - const dict_table_t* table, /*!< in: table whose index + dict_table_t* table, /*!< in: table whose index is renamed */ const char* old_index_name, /*!< in: old index name */ const char* new_index_name) /*!< in: new index name */ @@ -3877,6 +3932,12 @@ dict_stats_rename_index( char dbname_utf8[MAX_DB_UTF8_LEN]; char tablename_utf8[MAX_TABLE_UTF8_LEN]; + if (table->is_readable()) { + } else { + return (dict_stats_report_error(table, true)); + } + + dict_fs2utf8(table->name.m_name, dbname_utf8, sizeof(dbname_utf8), tablename_utf8, sizeof(tablename_utf8)); diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 016b7c3d24e..6ad6acf0759 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -439,16 +439,18 @@ fil_space_set_crypt_data( /****************************************************************** Parse a MLOG_FILE_WRITE_CRYPT_DATA log entry -@param[in] ptr Log entry start +@param[in,out] ptr Log entry start @param[in] end_ptr Log entry end @param[in] block buffer block +@param[out] err DB_SUCCESS or DB_DECRYPTION_FAILED @return position on log buffer */ UNIV_INTERN -const byte* +byte* fil_parse_write_crypt_data( - const byte* ptr, + byte* ptr, const byte* end_ptr, - const buf_block_t* block) + const buf_block_t* block, + dberr_t* err) { /* check that redo log entry is complete */ uint entry_size = @@ -460,6 +462,8 @@ fil_parse_write_crypt_data( 4 + // size of key_id 1; // fil_encryption_t + *err = DB_SUCCESS; + if (ptr + entry_size > end_ptr) { return NULL; } @@ -506,6 +510,11 @@ fil_parse_write_crypt_data( fil_space_release(space); } + /* Check is used key found from encryption plugin */ + if (crypt_data->should_encrypt() && !crypt_data->is_key_found()) { + *err = DB_DECRYPTION_FAILED; + } + return ptr; } @@ -663,10 +672,12 @@ fil_space_encrypt( } bool corrupted = buf_page_is_corrupted(true, tmp_mem, page_size, space); + memcpy(tmp_mem+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, src+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 8); bool different = memcmp(src, tmp_mem, page_size.physical()); if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) { - fprintf(stderr, "ok %d corrupted %d corrupted1 %d err %d different %d\n", ok , corrupted, corrupted1, err, different); + fprintf(stderr, "ok %d corrupted %d corrupted1 %d err %d different %d\n", + ok , corrupted, corrupted1, err, different); fprintf(stderr, "src_frame\n"); buf_page_print(src_frame, page_size, BUF_PAGE_PRINT_NO_CRASH); fprintf(stderr, "encrypted_frame\n"); @@ -719,26 +730,6 @@ fil_space_decrypt( return false; } - if (crypt_data == NULL) { - if (!(space == 0 && offset == 0) && key_version != 0) { - /* FIL_PAGE_FILE_FLUSH_LSN field i.e. - FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION - should be only defined for the - first page in a system tablespace - data file (ibdata*, not *.ibd), if not - clear it. */ - - DBUG_LOG("crypt", - "Page " << page_id_t(space, offset) - << " carries key_version " << key_version - << " (should be undefined)"); - - memset(src_frame - + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 4); - } - - return false; - } ut_a(crypt_data != NULL && crypt_data->is_encrypted()); @@ -789,9 +780,6 @@ fil_space_decrypt( memcpy(tmp_frame + page_size.physical() - FIL_PAGE_DATA_END, src_frame + page_size.physical() - FIL_PAGE_DATA_END, FIL_PAGE_DATA_END); - - // clear key-version & crypt-checksum from dst - memset(tmp_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); } srv_stats.pages_decrypted.inc(); @@ -1782,11 +1770,11 @@ fil_crypt_rotate_page( bool modified = false; int needs_scrubbing = BTR_SCRUB_SKIP_PAGE; lsn_t block_lsn = block->page.newest_modification; - uint kv = block->page.key_version; + byte* frame = buf_block_get_frame(block); + uint kv = mach_read_from_4(frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); /* check if tablespace is closing after reading page */ - if (space->is_stopping()) { - byte* frame = buf_block_get_frame(block); + if (!space->is_stopping()) { if (kv == 0 && fil_crypt_is_page_uninitialized(frame, page_size)) { @@ -1808,9 +1796,6 @@ fil_crypt_rotate_page( FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id, MLOG_4BYTES, &mtr); - /* update block */ - block->page.key_version = key_state->key_version; - /* statistics */ state->crypt_stat.pages_modified++; } else { @@ -1902,6 +1887,8 @@ fil_crypt_rotate_page( state->end_lsn = block_lsn; } } + } else { + mtr_commit(&mtr); } if (sleeptime_ms) { @@ -2474,7 +2461,7 @@ fil_space_get_scrub_status( } #endif /* UNIV_INNOCHECKSUM */ -/** +/********************************************************************* Verify that post encryption checksum match calculated checksum. This function should be called only if tablespace contains crypt_data metadata (this is strong indication that tablespace is encrypted). @@ -2482,10 +2469,10 @@ Function also verifies that traditional checksum does not match calculated checksum as if it does page could be valid unencrypted, encrypted, or corrupted. -@param[in,out] page page frame (checksum is temporarily modified) +@param[in] page Page to verify @param[in] page_size page size -@param[in] space tablespace identifier -@param[in] offset page number +@param[in] space_id Tablespace id +@param[in] pageno Page no @return true if page is encrypted AND OK, false otherwise */ UNIV_INTERN bool @@ -2496,18 +2483,19 @@ fil_space_verify_crypt_checksum( bool strict_check, /*!< --strict-check */ FILE* log_file, /*!< --log */ #endif /* UNIV_INNOCHECKSUM */ - ulint space, - ulint offset) + ulint space_id, + ulint pageno) { uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); /* If page is not encrypted, return false */ if (key_version == 0) { - return false; + return(false); } srv_checksum_algorithm_t algorithm = static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm); + /* If no checksum is used, can't continue checking. */ if (algorithm == SRV_CHECKSUM_ALGORITHM_NONE) { return(true); @@ -2531,36 +2519,29 @@ fil_space_verify_crypt_checksum( return (true); } - /* Compressed pages use different checksum method. We first store - the post encryption checksum on checksum location and after function - restore the original. */ - if (page_size.is_compressed()) { - ib_uint32_t old = static_cast<ib_uint32_t>(mach_read_from_4( - page + FIL_PAGE_SPACE_OR_CHKSUM)); - - mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); + ib_uint32_t cchecksum1 = 0; + ib_uint32_t cchecksum2 = 0; - bool valid = page_zip_verify_checksum(page, - page_size.physical() -#ifdef UNIV_INNOCHECKSUM - , offset, - strict_check, - log_file != NULL, - log_file -#endif - ); - - mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, old); - - return (valid); + /* Calculate checksums */ + if (page_size.is_compressed()) { + cchecksum1 = page_zip_calc_checksum( + page, page_size.physical(), SRV_CHECKSUM_ALGORITHM_CRC32); + if(cchecksum1 != checksum) { + cchecksum2 = page_zip_calc_checksum( + page, page_size.physical(), + SRV_CHECKSUM_ALGORITHM_INNODB); + } + } else { + cchecksum1 = buf_calc_page_crc32(page); + if (cchecksum1 != checksum) { + cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum( + page); + } } /* If stored checksum matches one of the calculated checksums page is not corrupted. */ - ib_uint32_t cchecksum1 = buf_calc_page_crc32(page); - ib_uint32_t cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum( - page); bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2 || checksum == BUF_NO_CHECKSUM_MAGIC); @@ -2591,22 +2572,29 @@ fil_space_verify_crypt_checksum( ulint checksum1 = mach_read_from_4( page + FIL_PAGE_SPACE_OR_CHKSUM); - ulint checksum2 = mach_read_from_4( - page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); + ulint checksum2 = checksum1; + bool valid; #ifdef UNIV_INNOCHECKSUM # define CKARGS page, checksum1, checksum2, \ - offset, log_file != NULL, log_file, algorithm + pageno, log_file != NULL, log_file, algorithm #else # define CKARGS page, checksum1, checksum2 #endif - bool valid = buf_page_is_checksum_valid_crc32( - CKARGS, false - /* FIXME: also try the original crc32 that was - buggy on big-endian architectures? */) - || buf_page_is_checksum_valid_innodb(CKARGS); + if (page_size.is_compressed()) { + valid = (checksum1 == cchecksum1); + } else { + checksum2 = mach_read_from_4( + page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); + + valid = buf_page_is_checksum_valid_crc32( + CKARGS, false + /* FIXME: also try the original crc32 that was + buggy on big-endian architectures? */) + || buf_page_is_checksum_valid_innodb(CKARGS); #undef CKARGS + } if (encrypted && valid) { /* If page is encrypted and traditional checksums match, @@ -2617,11 +2605,11 @@ fil_space_verify_crypt_checksum( "Page " ULINTPF ":" ULINTPF " may be corrupted." " Post encryption checksum %u" " stored [" ULINTPF ":" ULINTPF "] key_version %u\n", - space, offset, checksum, checksum1, checksum2, + space_id, pageno, checksum, checksum1, checksum2, key_version); #else /* UNIV_INNOCHECKSUM */ ib::error() - << " Page " << space << ":" << offset + << " Page " << space_id << ":" << pageno << " may be corrupted." " Post encryption checksum " << checksum << " stored [" << checksum1 << ":" << checksum2 @@ -2632,3 +2620,4 @@ fil_space_verify_crypt_checksum( return(encrypted); } + diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 84c2f07b361..ee24c667de8 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -306,7 +306,9 @@ fil_write( } /*******************************************************************//** -Returns the table space by a given id, NULL if not found. */ +Returns the table space by a given id, NULL if not found. +It is unsafe to dereference the returned pointer. It is fine to check +for NULL. */ fil_space_t* fil_space_get_by_id( /*================*/ @@ -324,8 +326,7 @@ fil_space_get_by_id( return(space); } -/*******************************************************************//** -Returns the table space by a given name, NULL if not found. */ +/** Returns the table space by a given name, NULL if not found. */ UNIV_INLINE fil_space_t* fil_space_get_by_name( @@ -878,7 +879,7 @@ fil_flush_low(fil_space_t* space) { ut_ad(mutex_own(&fil_system->mutex)); ut_ad(space); - ut_ad(!space->stop_new_ops); + ut_ad(!space->is_stopping()); if (fil_buffering_disabled(space)) { @@ -2271,9 +2272,9 @@ for concurrency control. (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted or truncated */ -inline +UNIV_INTERN fil_space_t* -fil_space_acquire_low(ulint id, bool silent, bool for_io = false) +fil_space_acquire_low(ulint id, bool silent, bool for_io) { fil_space_t* space; @@ -2297,32 +2298,6 @@ fil_space_acquire_low(ulint id, bool silent, bool for_io = false) return(space); } -/** Acquire a tablespace when it could be dropped concurrently. -Used by background threads that do not necessarily hold proper locks -for concurrency control. -@param[in] id tablespace ID -@param[in] for_io whether to look up the tablespace while performing I/O - (possibly executing TRUNCATE) -@return the tablespace -@retval NULL if missing or being deleted or truncated */ -fil_space_t* -fil_space_acquire(ulint id, bool for_io) -{ - return(fil_space_acquire_low(id, false, for_io)); -} - -/** Acquire a tablespace that may not exist. -Used by background threads that do not necessarily hold proper locks -for concurrency control. -@param[in] id tablespace ID -@return the tablespace -@retval NULL if missing or being deleted */ -fil_space_t* -fil_space_acquire_silent(ulint id) -{ - return(fil_space_acquire_low(id, true)); -} - /** Release a tablespace acquired with fil_space_acquire(). @param[in,out] space tablespace to release */ void @@ -3066,7 +3041,7 @@ checked @return whether the table is accessible */ bool fil_table_accessible(const dict_table_t* table) { - if (UNIV_UNLIKELY(table->ibd_file_missing || table->corrupted)) { + if (UNIV_UNLIKELY(!table->is_readable())) { return(false); } @@ -5473,6 +5448,7 @@ fil_aio_wait( mutex_enter(&fil_system->mutex); fil_node_complete_io(node, type); + ulint purpose = node->space->purpose; mutex_exit(&fil_system->mutex); @@ -5484,7 +5460,7 @@ fil_aio_wait( deadlocks in the i/o system. We keep tablespace 0 data files always open, and use a special i/o thread to serve insert buffer requests. */ - switch (node->space->purpose) { + switch (purpose) { case FIL_TYPE_TABLESPACE: case FIL_TYPE_TEMPORARY: case FIL_TYPE_IMPORT: @@ -5493,7 +5469,25 @@ fil_aio_wait( /* async single page writes from the dblwr buffer don't have access to the page */ if (message != NULL) { - buf_page_io_complete(static_cast<buf_page_t*>(message)); + buf_page_t *bpage = static_cast<buf_page_t*>(message); + const page_id_t page_id = bpage->id; + dberr_t err = buf_page_io_complete(bpage); + + if (err != DB_SUCCESS) { + + /* In crash recovery set log corruption on + and produce only an error to fail InnoDB startup. */ + if (recv_recovery_is_on()) { + recv_sys->found_corrupt_log = true; + } + + ib::error() + << (type == IORequestRead ? "Read" : "Write") + << "operation failed for " << node->name + << " page " << page_id + << " error= " << ut_strerr(err); + } + } return; case FIL_TYPE_LOG: @@ -5518,7 +5512,7 @@ fil_flush( if (fil_space_t* space = fil_space_get_by_id(space_id)) { if (space->purpose != FIL_TYPE_TEMPORARY - && !space->is_stopping()) { + && !space->stop_new_ops) { fil_flush_low(space); } } @@ -6453,7 +6447,6 @@ fil_space_validate_for_mtr_commit( mini-transaction, we should have !space->stop_new_ops. This is guaranteed by meta-data locks or transactional locks, or dict_operation_lock (X-lock in DROP, S-lock in purge). - However, a file I/O thread can invoke change buffer merge while fil_check_pending_operations() is waiting for operations to quiesce. This is not a problem, because @@ -6510,7 +6503,6 @@ fil_names_dirty_and_write( ut_ad(log_mutex_own()); ut_d(fil_space_validate_for_mtr_commit(space)); ut_ad(space->max_lsn == log_sys->lsn); - UT_LIST_ADD_LAST(fil_system->named_spaces, space); fil_names_write(space, mtr); @@ -6543,9 +6535,7 @@ fil_names_clear( "increase_mtr_checkpoint_size", mtr_checkpoint_size = 75 * 1024; ); - ut_ad(log_mutex_own()); - if (log_sys->append_on_checkpoint) { mtr_write_log(log_sys->append_on_checkpoint); do_write = true; @@ -6556,7 +6546,6 @@ fil_names_clear( for (fil_space_t* space = UT_LIST_GET_FIRST(fil_system->named_spaces); space != NULL; ) { fil_space_t* next = UT_LIST_GET_NEXT(named_spaces, space); - ut_ad(space->max_lsn > 0); if (space->max_lsn < lsn) { /* The tablespace was last dirtied before the @@ -6597,7 +6586,6 @@ fil_names_clear( } else { ut_ad(!mtr.has_modifications()); } - return(do_write); } diff --git a/storage/innobase/fsp/fsp0file.cc b/storage/innobase/fsp/fsp0file.cc index b8ad49a254f..3887851c0d9 100644 --- a/storage/innobase/fsp/fsp0file.cc +++ b/storage/innobase/fsp/fsp0file.cc @@ -58,11 +58,6 @@ Datafile::shutdown() ut_free(m_name); m_name = NULL; - - /* The fil_space_t::crypt_data was freed in - fil_space_free_low(). Invalidate our redundant pointer. */ - m_crypt_info = NULL; - free_filepath(); free_first_page(); } diff --git a/storage/innobase/fsp/fsp0sysspace.cc b/storage/innobase/fsp/fsp0sysspace.cc index 974140fe565..cf918cda861 100644 --- a/storage/innobase/fsp/fsp0sysspace.cc +++ b/storage/innobase/fsp/fsp0sysspace.cc @@ -336,6 +336,10 @@ SysTablespace::shutdown() { Tablespace::shutdown(); + if (m_crypt_info) { + fil_space_destroy_crypt_data(&m_crypt_info); + } + m_auto_extend_last_file = 0; m_last_file_size_max = 0; m_created_new_raw = 0; @@ -943,6 +947,9 @@ SysTablespace::open_or_create( name(), space_id(), flags(), is_temp ? FIL_TYPE_TEMPORARY : FIL_TYPE_TABLESPACE, m_crypt_info, false); + + /* Crypt info is stored to fil_space_t */ + m_crypt_info = NULL; } ut_a(fil_validate()); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index c486b0d4c4d..dbd5b0b745e 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -6601,7 +6601,7 @@ ha_innobase::open( /* Mark this table as corrupted, so the drop table or force recovery can still use it, but not others. */ - ib_table->corrupted = true; + ib_table->file_unreadable = true; dict_table_close(ib_table, FALSE, FALSE); ib_table = NULL; is_part = NULL; @@ -6629,7 +6629,7 @@ ha_innobase::open( ib_table->thd = (void*)thd; /* No point to init any statistics if tablespace is still encrypted. */ - if (!ib_table->is_encrypted) { + if (ib_table->is_readable()) { dict_stats_init(ib_table); } else { ib_table->stat_initialized = 1; @@ -6637,7 +6637,9 @@ ha_innobase::open( MONITOR_INC(MONITOR_TABLE_OPEN); - bool no_tablespace; + bool no_tablespace = false; + bool encrypted = false; + FilSpace space; if (dict_table_is_discarded(ib_table)) { @@ -6652,21 +6654,28 @@ ha_innobase::open( no_tablespace = false; - } else if (ib_table->ibd_file_missing) { + } else if (!ib_table->is_readable()) { + space = fil_space_acquire_silent(ib_table->space); - ib_senderrf( - thd, IB_LOG_LEVEL_WARN, - ER_TABLESPACE_MISSING, norm_name); - - /* This means we have no idea what happened to the tablespace - file, best to play it safe. */ - - no_tablespace = true; - } else if (ib_table->is_encrypted) { - /* This means that tablespace was found but we could not - decrypt encrypted page. */ - no_tablespace = true; - ib_table->ibd_file_missing = true; + if (space()) { + if (space()->crypt_data && space()->crypt_data->is_encrypted()) { + /* This means that tablespace was found but we could not + decrypt encrypted page. */ + no_tablespace = true; + encrypted = true; + } else { + no_tablespace = true; + } + } else { + ib_senderrf( + thd, IB_LOG_LEVEL_WARN, + ER_TABLESPACE_MISSING, norm_name); + + /* This means we have no idea what happened to the tablespace + file, best to play it safe. */ + + no_tablespace = true; + } } else { no_tablespace = false; } @@ -6679,34 +6688,31 @@ ha_innobase::open( /* If table has no talespace but it has crypt data, check is tablespace made unaccessible because encryption service or used key_id is not available. */ - if (ib_table) { + if (encrypted) { bool warning_pushed = false; - fil_space_crypt_t* crypt_data = ib_table->crypt_data; - - if (crypt_data && crypt_data->should_encrypt()) { - if (!encryption_key_id_exists(crypt_data->key_id)) { - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - HA_ERR_DECRYPTION_FAILED, - "Table %s is encrypted but encryption service or" - " used key_id %u is not available. " - " Can't continue reading table.", - ib_table->name, crypt_data->key_id); - ret_err = HA_ERR_DECRYPTION_FAILED; - warning_pushed = true; - } + if (!encryption_key_id_exists(space()->crypt_data->key_id)) { + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + HA_ERR_DECRYPTION_FAILED, + "Table %s in file %s is encrypted but encryption service or" + " used key_id %u is not available. " + " Can't continue reading table.", + ib_table->name, space()->chain.start->name, + space()->crypt_data->key_id); + ret_err = HA_ERR_DECRYPTION_FAILED; + warning_pushed = true; } /* If table is marked as encrypted then we push warning if it has not been already done as used key_id might be found but it is incorrect. */ - if (ib_table->is_encrypted && !warning_pushed) { + if (!warning_pushed) { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, HA_ERR_DECRYPTION_FAILED, - "Table %s is encrypted but encryption service or" + "Table %s in file %s is encrypted but encryption service or" " used key_id is not available. " " Can't continue reading table.", - ib_table->name); + ib_table->name, space()->chain.start->name); ret_err = HA_ERR_DECRYPTION_FAILED; } } @@ -6864,8 +6870,7 @@ ha_innobase::open( if (m_prebuilt->table == NULL || dict_table_is_temporary(m_prebuilt->table) - || m_prebuilt->table->persistent_autoinc - || m_prebuilt->table->ibd_file_missing) { + || !m_prebuilt->table->is_readable()) { } else if (const Field* ai = table->found_next_number_field) { initialize_auto_increment(m_prebuilt->table, ai); } @@ -10315,6 +10320,21 @@ ha_innobase::general_fetch( DB_FORCED_ABORT, 0, m_user_thd)); } + if (m_prebuilt->table->is_readable()) { + } else { + if (m_prebuilt->table->corrupted) { + DBUG_RETURN(HA_ERR_CRASHED); + } else { + FilSpace space(m_prebuilt->table->space, true); + + if (space()) { + DBUG_RETURN(HA_ERR_DECRYPTION_FAILED); + } else { + DBUG_RETURN(HA_ERR_NO_SUCH_TABLE); + } + } + } + innobase_srv_conc_enter_innodb(m_prebuilt); ret = row_search_mvcc( @@ -13646,7 +13666,7 @@ ha_innobase::discard_or_import_tablespace( user may want to set the DISCARD flag in order to IMPORT a new tablespace. */ - if (dict_table->ibd_file_missing) { + if (!dict_table->is_readable()) { ib_senderrf( m_prebuilt->trx->mysql_thd, IB_LOG_LEVEL_WARN, ER_TABLESPACE_MISSING, @@ -13656,7 +13676,7 @@ ha_innobase::discard_or_import_tablespace( err = row_discard_tablespace_for_mysql( dict_table->name.m_name, m_prebuilt->trx); - } else if (!dict_table->ibd_file_missing) { + } else if (dict_table->is_readable()) { /* Commit the transaction in order to release the table lock. */ trx_commit_for_mysql(m_prebuilt->trx); @@ -15325,7 +15345,8 @@ ha_innobase::check( DBUG_RETURN(HA_ADMIN_CORRUPT); - } else if (m_prebuilt->table->ibd_file_missing) { + } else if (!m_prebuilt->table->is_readable() && + fil_space_get(m_prebuilt->table->space) == NULL) { ib_senderrf( thd, IB_LOG_LEVEL_ERROR, diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 6f931d8d5a6..788e8281692 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -4863,7 +4863,7 @@ new_clustered_failed: clustered index of the old table, later. */ if (new_clustered || !ctx->online - || user_table->ibd_file_missing + || !user_table->is_readable() || dict_table_is_discarded(user_table)) { /* No need to allocate a modification log. */ ut_ad(!ctx->add_index[a]->online_log); @@ -5543,32 +5543,6 @@ ha_innobase::prepare_inplace_alter_table( DBUG_RETURN(false); } - indexed_table = m_prebuilt->table; - - if (indexed_table->is_encrypted) { - String str; - const char* engine= table_type(); - push_warning_printf(m_user_thd, Sql_condition::WARN_LEVEL_WARN, - HA_ERR_DECRYPTION_FAILED, - "Table %s is encrypted but encryption service or" - " used key_id is not available. " - " Can't continue reading table.", - indexed_table->name); - get_error_message(HA_ERR_DECRYPTION_FAILED, &str); - my_error(ER_GET_ERRMSG, MYF(0), HA_ERR_DECRYPTION_FAILED, str.c_ptr(), engine); - - DBUG_RETURN(true); - } - - if (indexed_table->corrupted - || dict_table_get_first_index(indexed_table) == NULL - || dict_index_is_corrupted( - dict_table_get_first_index(indexed_table))) { - /* The clustered index is corrupted. */ - my_error(ER_CHECK_NO_SUCH_TABLE, MYF(0)); - DBUG_RETURN(true); - } - /* ALTER TABLE will not implicitly move a table from a single-table tablespace to the system tablespace when innodb_file_per_table=OFF. But it will implicitly move a table from the system tablespace to a @@ -5580,6 +5554,8 @@ ha_innobase::prepare_inplace_alter_table( NULL, NULL); + indexed_table = m_prebuilt->table; + info.set_tablespace_type(indexed_table->space != TRX_SYS_SPACE); if (ha_alter_info->handler_flags & Alter_inplace_info::ADD_INDEX) { @@ -5588,6 +5564,39 @@ ha_innobase::prepare_inplace_alter_table( } } + if (indexed_table->is_readable()) { + } else { + if (indexed_table->corrupted) { + /* Handled below */ + } else { + FilSpace space(indexed_table->space, true); + + if (space()) { + String str; + const char* engine= table_type(); + + push_warning_printf(m_user_thd, Sql_condition::WARN_LEVEL_WARN, + HA_ERR_DECRYPTION_FAILED, + "Table %s in file %s is encrypted but encryption service or" + " used key_id is not available. " + " Can't continue reading table.", + indexed_table->name, space()->chain.start->name); + + my_error(ER_GET_ERRMSG, MYF(0), HA_ERR_DECRYPTION_FAILED, str.c_ptr(), engine); + DBUG_RETURN(true); + } + } + } + + if (indexed_table->corrupted + || dict_table_get_first_index(indexed_table) == NULL + || dict_index_is_corrupted( + dict_table_get_first_index(indexed_table))) { + /* The clustered index is corrupted. */ + my_error(ER_CHECK_NO_SUCH_TABLE, MYF(0)); + DBUG_RETURN(true); + } + if (ha_alter_info->handler_flags & Alter_inplace_info::CHANGE_CREATE_OPTION) { const char* invalid_opt = info.create_options_are_invalid(); @@ -6375,17 +6384,17 @@ ok_exit: dict_index_t* pk = dict_table_get_first_index(m_prebuilt->table); ut_ad(pk != NULL); + if (!m_prebuilt->table->is_readable() + || dict_table_is_discarded(m_prebuilt->table)) { + goto all_done; + } + /* For partitioned tables this could be already allocated from a previous partition invocation. For normal tables this is NULL. */ UT_DELETE(ctx->m_stage); ctx->m_stage = UT_NEW_NOKEY(ut_stage_alter_t(pk)); - if (m_prebuilt->table->ibd_file_missing - || dict_table_is_discarded(m_prebuilt->table)) { - goto all_done; - } - /* If we are doing a table rebuilding or having added virtual columns in the same clause, we will need to build a table template that carries translation information between MySQL TABLE and InnoDB @@ -7804,7 +7813,7 @@ commit_try_rebuild( /* The new table must inherit the flag from the "parent" table. */ if (dict_table_is_discarded(user_table)) { - rebuilt_table->ibd_file_missing = true; + rebuilt_table->file_unreadable = true; rebuilt_table->flags2 |= DICT_TF2_DISCARDED; } @@ -8316,15 +8325,15 @@ alter_stats_rebuild( DBUG_EXECUTE_IF( "ib_rename_index_fail2", - ibd_file_missing_orig = table->ibd_file_missing; - table->ibd_file_missing = TRUE; + ibd_file_missing_orig = table->file_unreadable; + table->file_unreadable = true; ); dberr_t ret = dict_stats_update(table, DICT_STATS_RECALC_PERSISTENT); DBUG_EXECUTE_IF( "ib_rename_index_fail2", - table->ibd_file_missing = ibd_file_missing_orig; + table->file_unreadable = ibd_file_missing_orig; ); if (ret != DB_SUCCESS) { @@ -8455,6 +8464,19 @@ ha_innobase::commit_inplace_alter_table( = static_cast<ha_innobase_inplace_ctx*>(*pctx); DBUG_ASSERT(ctx->prebuilt->trx == m_prebuilt->trx); + /* If decryption failed for old table or new table + fail here. */ + if ((!ctx->old_table->is_readable() && + fil_space_get(ctx->old_table->space) != NULL)|| + (!ctx->new_table->is_readable() && + fil_space_get(ctx->new_table->space) != NULL)) { + String str; + const char* engine= table_type(); + get_error_message(HA_ERR_DECRYPTION_FAILED, &str); + my_error(ER_GET_ERRMSG, MYF(0), HA_ERR_DECRYPTION_FAILED, str.c_ptr(), engine); + DBUG_RETURN(true); + } + /* Exclusively lock the table, to ensure that no other transaction is holding locks on the table while we change the table definition. The MySQL meta-data lock diff --git a/storage/innobase/include/btr0btr.ic b/storage/innobase/include/btr0btr.ic index 308fcfe9b03..bd4f2a40267 100644 --- a/storage/innobase/include/btr0btr.ic +++ b/storage/innobase/include/btr0btr.ic @@ -63,7 +63,7 @@ btr_block_get_func( if (err == DB_DECRYPTION_FAILED) { if (index && index->table) { - index->table->is_encrypted = true; + index->table->file_unreadable = true; } } diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 6832c133d58..3a6b9790cc6 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -835,17 +835,6 @@ buf_page_is_checksum_valid_none( ) MY_ATTRIBUTE((nonnull(1), warn_unused_result)); -/********************************************************************//** -Check if page is maybe compressed, encrypted or both when we encounter -corrupted page. Note that we can't be 100% sure if page is corrupted -or decrypt/decompress just failed. -@param[in] bpage Page -@return true if page corrupted, false if not */ -bool -buf_page_check_corrupt( - buf_page_t* bpage) /*!< in/out: buffer page read from disk */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); - /** Checks if a page contains only zeroes. @param[in] read_buf database page @param[in] page_size page size @@ -854,7 +843,6 @@ bool buf_page_is_zeroes( const byte* read_buf, const page_size_t& page_size); - /** Checks if a page is corrupt. @param[in] check_lsn true if we need to check and complain about the LSN @@ -1326,13 +1314,17 @@ buf_page_init_for_read( /********************************************************************//** Completes an asynchronous read or write request of a file page to or from the buffer pool. -@return true if successful */ -bool +@param[in,out] bpage pointer to the block in question +@param[in] evict true if page should be evicted from LRU +@return DB_SUCCESS if page has been read and is not corrupted, +DB_PAGE_CORRUPTED if page based on checksum check is corrupted, +DB_DECRYPTION_FAILED if page post encryption checksum matches but +after decryption normal page checksum does not match.*/ +dberr_t buf_page_io_complete( -/*=================*/ - buf_page_t* bpage, /*!< in: pointer to the block in question */ - bool evict = false);/*!< in: whether or not to evict - the page from LRU list. */ + buf_page_t* bpage, + bool evict = false); + /********************************************************************//** Calculates the index of a buffer pool to the buf_pool[] array. @return the position of the buffer pool in buf_pool[] */ @@ -1639,7 +1631,6 @@ public: if written again we check is TRIM operation needed. */ - unsigned key_version; /*!< key version for this block */ bool encrypted; /*!< page is still encrypted */ ulint real_size; /*!< Real size of the page diff --git a/storage/innobase/include/buf0rea.h b/storage/innobase/include/buf0rea.h index 9c97a5147c1..b22a7564bc3 100644 --- a/storage/innobase/include/buf0rea.h +++ b/storage/innobase/include/buf0rea.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -35,14 +35,19 @@ Created 11/5/1995 Heikki Tuuri buffer buf_pool if it is not already there. Sets the io_fix flag and sets an exclusive lock on the buffer frame. The flag is cleared and the x-lock released by the i/o-handler thread. -@param[in] page_id page id -@param[in] page_size page size -@return TRUE if page has been read in, FALSE in case of failure */ -ibool + +@param[in] space space_id +@param[in] zip_size compressed page size in bytes, or 0 +@param[in] offset page number +@return DB_SUCCESS if page has been read and is not corrupted, +@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted, +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but +after decryption normal page checksum does not match. +@retval DB_TABLESPACE_DELETED if tablespace .ibd file is missing */ +dberr_t buf_read_page( const page_id_t& page_id, - const page_size_t& page_size, - buf_page_t** bpage); + const page_size_t& page_size); /********************************************************************//** High-level function which reads a page asynchronously from a file to the @@ -51,9 +56,8 @@ an exclusive lock on the buffer frame. The flag is cleared and the x-lock released by the i/o-handler thread. @param[in] page_id page id @param[in] page_size page size -@param[in] sync true if synchronous aio is desired -@return TRUE if page has been read in, FALSE in case of failure */ -ibool +@param[in] sync true if synchronous aio is desired */ +void buf_read_page_background( const page_id_t& page_id, const page_size_t& page_size, diff --git a/storage/innobase/include/db0err.h b/storage/innobase/include/db0err.h index b0609991f61..984afa6e017 100644 --- a/storage/innobase/include/db0err.h +++ b/storage/innobase/include/db0err.h @@ -143,6 +143,8 @@ enum dberr_t { of missing key management plugin, or missing or incorrect key or incorret AES method or algorithm. */ + DB_PAGE_CORRUPTED, /* Page read from tablespace is + corrupted. */ DB_IO_ERROR = 100, /*!< Generic IO error */ diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index 18578388723..a8d517025a8 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -1889,6 +1889,15 @@ dict_set_corrupted_by_space( /*========================*/ ulint space_id); /*!< in: space ID */ +/**********************************************************************//** +Flags a table with specified space_id encrypted in the data dictionary +cache +@param[in] space_id Tablespace id */ +UNIV_INTERN +void +dict_set_encrypted_by_space( + ulint space_id); + /** Sets merge_threshold in the SYS_INDEXES @param[in,out] index index @param[in] merge_threshold value to set */ diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index 0630137bb4f..82db5fcca64 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -1004,6 +1004,13 @@ struct dict_index_t{ return(UNIV_LIKELY(!uncommitted)); } + /* Returns true if this is a single-table tablespace + and the .ibd file is missing or page decryption failed + and/or page is corrupted. + @return true if table is readable + @retval false if table is not readable */ + bool is_readable() const; + /** Flag an index committed or uncommitted. @param[in] committed whether the index is committed */ void set_committed(bool committed) @@ -1404,9 +1411,9 @@ struct dict_table_t { unsigned flags2:DICT_TF2_BITS; /** TRUE if this is in a single-table tablespace and the .ibd file is - missing. Then we must return in ha_innodb.cc an error if the user - tries to query such an orphaned table. */ - unsigned ibd_file_missing:1; + missing or page in file is encrypted. Then we must return in + ha_innodb.cc an error if the usertries to query such an table. */ + unsigned file_unreadable:1; /** TRUE if the table object has been added to the dictionary cache. */ unsigned cached:1; @@ -1727,7 +1734,19 @@ public: /** Timestamp of the last modification of this table. */ time_t update_time; - bool is_encrypted; + /** mysql_row_templ_t for base columns used for compute the virtual + columns */ + dict_vcol_templ_t* vc_templ; + + /* Returns true if this is a single-table tablespace + and the .ibd file is missing or page decryption failed + and/or page is corrupted. + @return true if table is readable + @retval false if table is not readable */ + inline bool is_readable() const + { + return(UNIV_LIKELY(!file_unreadable)); + } #ifdef UNIV_DEBUG /** Value of 'magic_n'. */ @@ -1736,9 +1755,6 @@ public: /** Magic number. */ ulint magic_n; #endif /* UNIV_DEBUG */ - /** mysql_row_templ_t for base columns used for compute the virtual - columns */ - dict_vcol_templ_t* vc_templ; }; /*******************************************************************//** @@ -1748,6 +1764,16 @@ lock_table_lock_list_init( /*======================*/ table_lock_list_t* locks); /*!< List to initialise */ +/* Returns true if this is a single-table tablespace +and the .ibd file is missing or page decryption failed +and/or page is corrupted. +@return true if table is readable +@retval false if table is not readable */ +inline bool dict_index_t::is_readable() const +{ + return(UNIV_LIKELY(!table->file_unreadable)); +} + /** A function object to add the foreign key constraint to the referenced set of the referenced table, if it exists in the dictionary cache. */ struct dict_foreign_add_to_referenced_table { diff --git a/storage/innobase/include/dict0stats.h b/storage/innobase/include/dict0stats.h index f02b8eb8eed..0cb6d84d58c 100644 --- a/storage/innobase/include/dict0stats.h +++ b/storage/innobase/include/dict0stats.h @@ -178,7 +178,7 @@ if the persistent stats do not exist. */ dberr_t dict_stats_rename_index( /*====================*/ - const dict_table_t* table, /*!< in: table whose index + dict_table_t* table, /*!< in: table whose index is renamed */ const char* old_index_name, /*!< in: old index name */ const char* new_index_name) /*!< in: new index name */ @@ -252,6 +252,19 @@ dict_stats_save_index_stat( const char* stat_description, trx_t* trx); +/** Report error if statistic update for a table failed because +.ibd file is missing, table decryption failed or table is corrupted. +@param[in,out] table Table +@param[in] defragment true if statistics is for defragment +@return DB_DECRYPTION_FAILED, DB_TABLESPACE_DELETED or DB_CORRUPTION +@retval DB_DECRYPTION_FAILED if decryption of the table failed +@retval DB_TABLESPACE_DELETED if .ibd file is missing +@retval DB_CORRUPTION if table is marked as corrupted */ +dberr_t +dict_stats_report_error( + dict_table_t* table, + bool defragment = false); + #include "dict0stats.ic" #ifdef UNIV_ENABLE_UNIT_TEST_DICT_STATS diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index 831d61445d8..755ebe3a0d7 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -116,10 +116,9 @@ struct fil_space_crypt_t : st_encryption_scheme min_key_version(new_min_key_version), page0_offset(0), encryption(new_encryption), - key_found(), + key_found(0), rotate_state() { - key_found = new_min_key_version; key_id = new_key_id; my_random_bytes(iv, sizeof(iv)); mutex_create(LATCH_ID_FIL_CRYPT_DATA_MUTEX, &mutex); @@ -134,6 +133,8 @@ struct fil_space_crypt_t : st_encryption_scheme type = CRYPT_SCHEME_1; min_key_version = key_get_latest_version(); } + + key_found = min_key_version; } /** Destructor */ @@ -290,16 +291,18 @@ fil_space_destroy_crypt_data( /****************************************************************** Parse a MLOG_FILE_WRITE_CRYPT_DATA log entry -@param[in] ptr Log entry start +@param[in,out] ptr Log entry start @param[in] end_ptr Log entry end @param[in] block buffer block +@param[out] err DB_SUCCESS or DB_DECRYPTION_FAILED @return position on log buffer */ UNIV_INTERN -const byte* +byte* fil_parse_write_crypt_data( - const byte* ptr, + byte* ptr, const byte* end_ptr, - const buf_block_t* block) + const buf_block_t* block, + dberr_t* err) MY_ATTRIBUTE((warn_unused_result)); /** Encrypt a buffer. @@ -484,7 +487,7 @@ encrypted, or corrupted. @param[in,out] page page frame (checksum is temporarily modified) @param[in] page_size page size -@param[in] space tablespace identifier +@param[in] space_id tablespace identifier @param[in] offset page number @return true if page is encrypted AND OK, false otherwise */ UNIV_INTERN @@ -496,7 +499,7 @@ fil_space_verify_crypt_checksum( bool strict_check, /*!< --strict-check */ FILE* log_file, /*!< --log */ #endif /* UNIV_INNOCHECKSUM */ - ulint space, + ulint space_id, ulint offset) MY_ATTRIBUTE((warn_unused_result)); diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 081362fb6f4..5b965fe791f 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -726,23 +726,45 @@ MY_ATTRIBUTE((warn_unused_result)); Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID +@param[in] silent whether to silently ignore missing tablespaces @param[in] for_io whether to look up the tablespace while performing I/O (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted or truncated */ +UNIV_INTERN fil_space_t* -fil_space_acquire(ulint id, bool for_io = false) +fil_space_acquire_low(ulint id, bool silent, bool for_io = false) MY_ATTRIBUTE((warn_unused_result)); +/** Acquire a tablespace when it could be dropped concurrently. +Used by background threads that do not necessarily hold proper locks +for concurrency control. +@param[in] id tablespace ID +@param[in] for_io whether to look up the tablespace while performing I/O + (possibly executing TRUNCATE) +@return the tablespace +@retval NULL if missing or being deleted or truncated */ +inline +fil_space_t* +fil_space_acquire(ulint id, bool for_io = false) +{ + return (fil_space_acquire_low(id, false, for_io)); +} + /** Acquire a tablespace that may not exist. Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID +@param[in] for_io whether to look up the tablespace while performing I/O + (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted */ +inline fil_space_t* -fil_space_acquire_silent(ulint id) - MY_ATTRIBUTE((warn_unused_result)); +fil_space_acquire_silent(ulint id, bool for_io = false) +{ + return (fil_space_acquire_low(id, true, for_io)); +} /** Release a tablespace acquired with fil_space_acquire(). @param[in,out] space tablespace to release */ @@ -786,11 +808,12 @@ public: /** Constructor: Look up the tablespace and increment the reference count if found. @param[in] space_id tablespace ID + @param[in] silent whether not print any errors @param[in] for_io whether to look up the tablespace while performing I/O (possibly executing TRUNCATE) */ - explicit FilSpace(ulint space_id, bool for_io = false) - : m_space(fil_space_acquire(space_id, for_io)) {} + explicit FilSpace(ulint space_id, bool silent = false, bool for_io = false) + : m_space(fil_space_acquire_low(space_id, silent, for_io)) {} /** Assignment operator: This assumes that fil_space_acquire() has already been done for the fil_space_t. The caller must @@ -1495,13 +1518,6 @@ fil_mtr_rename_log( /*******************************************************************//** Returns the table space by a given id, NULL if not found. */ fil_space_t* -fil_space_found_by_id( -/*==================*/ - ulint id); /*!< in: space id */ - -/*******************************************************************//** -Returns the table space by a given id, NULL if not found. */ -fil_space_t* fil_space_get_by_id( /*================*/ ulint id); /*!< in: space id */ @@ -1512,7 +1528,6 @@ by redo log. void fil_names_dirty( fil_space_t* space); - /** Write MLOG_FILE_NAME records when a non-predefined persistent tablespace was modified for the first time since the latest fil_names_clear(). @@ -1522,7 +1537,6 @@ void fil_names_dirty_and_write( fil_space_t* space, mtr_t* mtr); - /** Write MLOG_FILE_NAME records if a persistent tablespace was modified for the first time since the latest fil_names_clear(). @param[in,out] space tablespace @@ -1535,7 +1549,6 @@ fil_names_write_if_was_clean( mtr_t* mtr) { ut_ad(log_mutex_own()); - if (space == NULL) { return(false); } @@ -1591,8 +1604,6 @@ fil_names_clear( #ifdef UNIV_ENABLE_UNIT_TEST_MAKE_FILEPATH void test_make_filepath(); #endif /* UNIV_ENABLE_UNIT_TEST_MAKE_FILEPATH */ - - /** Determine the block size of the data file. @param[in] space tablespace @param[in] offset page number diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index 9e5248e7b86..bd07c20d46b 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -386,6 +386,23 @@ trx_print_latched( ulint max_query_len); /*!< in: max query length to print, or 0 to use the default max length */ +#ifdef WITH_WSREP +/**********************************************************************//** +Prints info about a transaction. +Transaction information may be retrieved without having trx_sys->mutex acquired +so it may not be completely accurate. The caller must own lock_sys->mutex +and the trx must have some locks to make sure that it does not escape +without locking lock_sys->mutex. */ +UNIV_INTERN +void +wsrep_trx_print_locking( +/*==============*/ + FILE* f, /*!< in: output stream */ + const trx_t* trx, /*!< in: transaction */ + ulint max_query_len) /*!< in: max query length to print, + or 0 to use the default max length */ + MY_ATTRIBUTE((nonnull)); +#endif /* WITH_WSREP */ /**********************************************************************//** Prints info about a transaction. Acquires and releases lock_sys->mutex and trx_sys->mutex. */ diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 800868bef94..0db929b7246 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -1381,14 +1381,13 @@ wsrep_kill_victim( is in the queue*/ } else if (lock->trx != trx) { if (wsrep_log_conflicts) { - mutex_enter(&trx_sys->mutex); if (bf_this) { ib::info() << "*** Priority TRANSACTION:"; } else { ib::info() << "*** Victim TRANSACTION:"; } - trx_print_latched(stderr, trx, 3000); + wsrep_trx_print_locking(stderr, trx, 3000); if (bf_other) { ib::info() << "*** Priority TRANSACTION:"; @@ -1396,9 +1395,7 @@ wsrep_kill_victim( ib::info() << "*** Victim TRANSACTION:"; } - trx_print_latched(stderr, lock->trx, 3000); - - mutex_exit(&trx_sys->mutex); + wsrep_trx_print_locking(stderr, lock->trx, 3000); ib::info() << "*** WAITING FOR THIS LOCK TO BE GRANTED:"; diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 0b46c92f00a..fb7f3911d3c 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1497,7 +1497,12 @@ parse_log: } break; case MLOG_FILE_WRITE_CRYPT_DATA: - ptr = const_cast<byte*>(fil_parse_write_crypt_data(ptr, end_ptr, block)); + dberr_t err; + ptr = const_cast<byte*>(fil_parse_write_crypt_data(ptr, end_ptr, block, &err)); + + if (err != DB_SUCCESS) { + recv_sys->found_corrupt_log = TRUE; + } break; default: ptr = NULL; @@ -1983,6 +1988,7 @@ recv_read_in_area( /** Apply the hash table of stored log records to persistent data pages. @param[in] last_batch whether the change buffer merge will be performed as part of the operation */ +UNIV_INTERN void recv_apply_hashed_log_recs(bool last_batch) { @@ -1993,6 +1999,11 @@ recv_apply_hashed_log_recs(bool last_batch) break; } + if (recv_sys->found_corrupt_log) { + mutex_exit(&recv_sys->mutex); + return; + } + mutex_exit(&recv_sys->mutex); os_thread_sleep(500000); } @@ -2076,6 +2087,10 @@ recv_apply_hashed_log_recs(bool last_batch) mutex_exit(&(recv_sys->mutex)); + if (recv_sys->found_corrupt_log) { + return; + } + os_thread_sleep(500000); mutex_enter(&(recv_sys->mutex)); diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index df7d26c63e8..abbe6cc1953 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -2063,7 +2063,7 @@ page_copy_rec_list_end_to_created_page( mtr_log_t log_mode; if (dict_table_is_temporary(index->table) - || index->table->ibd_file_missing /* IMPORT TABLESPACE */) { + || !index->table->is_readable() /* IMPORT TABLESPACE */) { log_mode = mtr_get_log_mode(mtr); } else { log_mode = mtr_set_log_mode(mtr, MTR_LOG_SHORT_INSERTS); diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index 01a55d0dc61..a2773baa34e 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -1959,16 +1959,13 @@ PageConverter::validate( buf_block_t* block) UNIV_NOTHROW { buf_frame_t* page = get_frame(block); - ulint space_id = mach_read_from_4( - page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - fil_space_t* space = fil_space_found_by_id(space_id); /* Check that the page number corresponds to the offset in the file. Flag as corrupt if it doesn't. Disable the check for LSN in buf_page_is_corrupted() */ if (buf_page_is_corrupted( - false, page, get_page_size(), space) + false, page, get_page_size(), NULL) || (page_get_page_no(page) != offset / m_page_size.physical() && page_get_page_no(page) != 0)) { @@ -2101,7 +2098,7 @@ row_import_discard_changes( index->space = FIL_NULL; } - table->ibd_file_missing = TRUE; + table->file_unreadable = true; fil_close_tablespace(trx, table->space); } @@ -3357,7 +3354,7 @@ row_import_for_mysql( ut_a(table->space); ut_ad(prebuilt->trx); - ut_a(table->ibd_file_missing); + ut_a(!table->is_readable()); ibuf_delete_for_discarded_space(table->space); @@ -3691,7 +3688,7 @@ row_import_for_mysql( return(row_import_error(prebuilt, trx, err)); } - table->ibd_file_missing = false; + table->file_unreadable = false; table->flags2 &= ~DICT_TF2_DISCARDED; /* Set autoinc value read from .cfg file, if one was specified. diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index 5803bc226cd..fb162592f75 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -1656,7 +1656,7 @@ row_ins_check_foreign_constraint( } if (check_table == NULL - || check_table->ibd_file_missing + || !check_table->is_readable() || check_index == NULL) { if (!srv_read_only_mode && check_ref) { @@ -2617,6 +2617,11 @@ row_ins_clust_index_entry_low( __FILE__, __LINE__, auto_inc, &mtr); cursor = btr_pcur_get_btr_cur(&pcur); cursor->thr = thr; + if (err != DB_SUCCESS) { + index->table->file_unreadable = true; + mtr_commit(&mtr); + goto func_exit; + } #ifdef UNIV_DEBUG { @@ -2950,7 +2955,7 @@ row_ins_sec_index_entry_low( " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } goto func_exit; } diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 82da61dc03b..4c30d0bfa49 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -26,6 +26,7 @@ Completed by Sunny Bains and Marko Makela *******************************************************/ #include <my_config.h> #include <log.h> +#include <sql_class.h> #include <math.h> @@ -1995,7 +1996,8 @@ row_merge_read_clustered_index( page_cur_t* cur = btr_pcur_get_page_cur(&pcur); /* Do not continue if table pages are still encrypted */ - if (old_table->is_encrypted || new_table->is_encrypted) { + if (!old_table->is_readable() || + !new_table->is_readable()) { err = DB_DECRYPTION_FAILED; trx->error_key_num = 0; goto func_exit; @@ -3300,7 +3302,11 @@ row_merge_sort( } #endif /* UNIV_SOLARIS */ - sql_print_information("InnoDB: Online DDL : merge-sorting has estimated %lu runs", num_runs); + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL : merge-sorting" + " has estimated " ULINTPF " runs", + num_runs); + } /* Merge the runs until we have one big run */ do { @@ -4295,7 +4301,7 @@ row_merge_rename_tables_dict( renamed along with the table. */ if (err == DB_SUCCESS && dict_table_is_file_per_table(old_table) - && !old_table->ibd_file_missing) { + && fil_space_get(old_table->space) != NULL) { /* Make pathname to update SYS_DATAFILES. */ char* tmp_path = row_make_new_pathname(old_table, tmp_name); @@ -4765,20 +4771,24 @@ row_merge_build_indexes( duplicate keys. */ innobase_rec_reset(table); - sql_print_information("InnoDB: Online DDL : Start"); - sql_print_information("InnoDB: Online DDL : Start reading clustered " - "index of the table and create temporary files"); + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL : Start reading" + " clustered index of the table" + " and create temporary files"); + } pct_cost = COST_READ_CLUSTERED_INDEX * 100 / (total_static_cost + total_dynamic_cost); /* Do not continue if we can't encrypt table pages */ - if (old_table->is_encrypted || new_table->is_encrypted) { + if (!old_table->is_readable() || + !new_table->is_readable()) { error = DB_DECRYPTION_FAILED; ib_push_warning(trx->mysql_thd, DB_DECRYPTION_FAILED, "Table %s is encrypted but encryption service or" " used key_id is not available. " " Can't continue reading table.", - old_table->is_encrypted ? old_table->name : new_table->name); + !old_table->is_readable() ? old_table->name : + new_table->name); goto func_exit; } @@ -4795,8 +4805,11 @@ row_merge_build_indexes( pct_progress += pct_cost; - sql_print_information("InnoDB: Online DDL : End of reading " - "clustered index of the table and create temporary files"); + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL : End of reading " + "clustered index of the table" + " and create temporary files"); + } for (i = 0; i < n_indexes; i++) { total_index_blocks += merge_files[i].offset; @@ -4895,8 +4908,7 @@ wait_again: DEBUG_FTS_SORT_PRINT("FTS_SORT: Complete Insert\n"); #endif } else if (merge_files[i].fd >= 0) { - char buf[3 * NAME_LEN]; - char *bufend; + char buf[NAME_LEN + 1]; row_merge_dup_t dup = { sort_idx, table, col_map, 0}; @@ -4905,17 +4917,24 @@ wait_again: total_index_blocks)) / (total_static_cost + total_dynamic_cost) * PCT_COST_MERGESORT_INDEX * 100; - - bufend = innobase_convert_name( + char* bufend = innobase_convert_name( buf, sizeof buf, - indexes[i]->name, strlen(indexes[i]->name), + indexes[i]->name, + strlen(indexes[i]->name), trx->mysql_thd); - buf[bufend - buf]='\0'; - sql_print_information("InnoDB: Online DDL : Start merge-sorting" - " index %s (%lu / %lu), estimated cost : %2.4f", - buf, (i+1), n_indexes, pct_cost); + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL :" + " Start merge-sorting" + " index %s" + " (" ULINTPF + " / " ULINTPF ")," + " estimated cost :" + " %2.4f", + buf, i + 1, n_indexes, + pct_cost); + } error = row_merge_sort( trx, &dup, &merge_files[i], @@ -4925,9 +4944,14 @@ wait_again: pct_progress += pct_cost; - sql_print_information("InnoDB: Online DDL : End of " - " merge-sorting index %s (%lu / %lu)", - buf, (i+1), n_indexes); + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL :" + " End of " + " merge-sorting index %s" + " (" ULINTPF + " / " ULINTPF ")", + buf, i + 1, n_indexes); + } DBUG_EXECUTE_IF( "ib_merge_wait_after_sort", @@ -4944,10 +4968,15 @@ wait_again: (total_static_cost + total_dynamic_cost) * PCT_COST_INSERT_INDEX * 100; - sql_print_information("InnoDB: Online DDL : Start " - "building index %s (%lu / %lu), estimated " - "cost : %2.4f", buf, (i+1), - n_indexes, pct_cost); + if (global_system_variables.log_warnings > 2) { + sql_print_information( + "InnoDB: Online DDL : Start " + "building index %s" + " (" ULINTPF + " / " ULINTPF "), estimated " + "cost : %2.4f", buf, i + 1, + n_indexes, pct_cost); + } error = row_merge_insert_index_tuples( trx->id, sort_idx, old_table, @@ -4960,9 +4989,13 @@ wait_again: pct_progress += pct_cost; - sql_print_information("InnoDB: Online DDL : " - "End of building index %s (%lu / %lu)", - buf, (i+1), n_indexes); + if (global_system_variables.log_warnings > 2) { + sql_print_information( + "InnoDB: Online DDL : " + "End of building index %s" + " (" ULINTPF " / " ULINTPF ")", + buf, i + 1, n_indexes); + } } } @@ -4988,11 +5021,8 @@ wait_again: DEBUG_SYNC_C("row_log_apply_before"); error = row_log_apply(trx, sort_idx, table, stage); DEBUG_SYNC_C("row_log_apply_after"); - sql_print_information("InnoDB: Online DDL : End of applying row log"); } - sql_print_information("InnoDB: Online DDL : Completed"); - if (error != DB_SUCCESS) { trx->error_key_num = key_numbers[i]; goto func_exit; diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 20a8777b71e..6160450d510 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -819,6 +819,7 @@ handle_new_error: break; case DB_CORRUPTION: + case DB_PAGE_CORRUPTED: ib::error() << "We detected index corruption in an InnoDB type" " table. You have to dump + drop + reimport the" " table or, in a case of widespread corruption," @@ -1402,6 +1403,63 @@ run_again: /** Does an insert for MySQL. @param[in] mysql_rec row in the MySQL format @param[in,out] prebuilt prebuilt struct in MySQL handle +@param[in] table Table +@param[in] trx Transaction +@param[in] push_warning true if we should push warning to user +@return DB_DECRYPTION_FAILED table is encrypted but decryption failed +DB_CORRUPTION table is corrupted +DB_TABLESPACE_NOT_FOUND tablespace .ibd file not found */ +static +dberr_t +row_mysql_get_table_status( + const dict_table_t* table, + trx_t* trx, + bool push_warning = true) +{ + dberr_t err = DB_SUCCESS; + if (fil_space_t* space = fil_space_acquire_silent(table->space)) { + if (space->crypt_data && space->crypt_data->is_encrypted()) { + // maybe we cannot access the table due to failing + // to decrypt + if (push_warning) { + ib_push_warning(trx, DB_DECRYPTION_FAILED, + "Table %s in tablespace %lu encrypted." + "However key management plugin or used key_id is not found or" + " used encryption algorithm or method does not match.", + table->name, table->space); + } + + err = DB_DECRYPTION_FAILED; + } else { + if (push_warning) { + ib_push_warning(trx, DB_CORRUPTION, + "Table %s in tablespace %lu corrupted.", + table->name, table->space); + } + + err = DB_CORRUPTION; + } + + fil_space_release(space); + } else { + ib::error() + << "InnoDB: MySQL is trying to use a table handle" + " but the .ibd file for" + " table " << table->name << " does not exist." + " Have you deleted the .ibd file" + " from the database directory under" + " the MySQL datadir, or have you" + " used DISCARD TABLESPACE?" + " Look from " REFMAN "innodb-troubleshooting.html" + " how you can resolve the problem."; + + err = DB_TABLESPACE_NOT_FOUND; + } + + return (err); +} + +/*********************************************************************//** @return error code or DB_SUCCESS */ dberr_t row_insert_for_mysql( @@ -1432,32 +1490,10 @@ row_insert_for_mysql( return(DB_TABLESPACE_DELETED); - } else if (prebuilt->table->ibd_file_missing) { - - ib::error() << ".ibd file is missing for table " - << prebuilt->table->name; - - return(DB_TABLESPACE_NOT_FOUND); - } else if (prebuilt->table->is_encrypted) { - ib_push_warning(trx, DB_DECRYPTION_FAILED, - "Table %s in tablespace " ULINTPF " encrypted." - "However key management plugin or used key_id is not found or" - " used encryption algorithm or method does not match.", - prebuilt->table->name, prebuilt->table->space); - return(DB_DECRYPTION_FAILED); + } else if (!prebuilt->table->is_readable()) { + return (row_mysql_get_table_status(prebuilt->table, trx, true)); } else if (srv_force_recovery) { - ib::error() << MODIFICATIONS_NOT_ALLOWED_MSG_FORCE_RECOVERY; - return(DB_READ_ONLY); - } - DBUG_EXECUTE_IF("mark_table_corrupted", { - /* Mark the table corrupted for the clustered index */ - dict_index_t* index = dict_table_get_first_index(table); - ut_ad(dict_index_is_clust(index)); - dict_set_corrupted(index, trx, "INSERT TABLE"); }); - - if (dict_table_is_corrupted(table)) { - ib::error() << "Table " << table->name << " is corrupt."; return(DB_TABLE_CORRUPT); } @@ -1852,27 +1888,17 @@ row_update_for_mysql_using_upd_graph( bool got_s_lock = false; DBUG_ENTER("row_update_for_mysql_using_upd_graph"); + if (!table->is_readable()) { + return (row_mysql_get_table_status(table, trx, true)); + } ut_ad(trx); ut_a(prebuilt->magic_n == ROW_PREBUILT_ALLOCATED); ut_a(prebuilt->magic_n2 == ROW_PREBUILT_ALLOCATED); UT_NOT_USED(mysql_rec); - if (prebuilt->table->ibd_file_missing) { - ib::error() << "MySQL is trying to use a table handle but the" - " .ibd file for table " << prebuilt->table->name - << " does not exist. Have you deleted" - " the .ibd file from the database directory under" - " the MySQL datadir, or have you used DISCARD" - " TABLESPACE? " << TROUBLESHOOTING_MSG; - DBUG_RETURN(DB_ERROR); - } else if (prebuilt->table->is_encrypted) { - ib_push_warning(trx, DB_DECRYPTION_FAILED, - "Table %s in tablespace " ULINTPF " encrypted." - "However key management plugin or used key_id is not found or" - " used encryption algorithm or method does not match.", - prebuilt->table->name, prebuilt->table->space); - return (DB_TABLE_NOT_FOUND); + if (UNIV_UNLIKELY(prebuilt->table->file_unreadable)) { + return (row_mysql_get_table_status(table, trx, true)); } if(srv_force_recovery) { @@ -3259,7 +3285,7 @@ row_discard_tablespace( /* All persistent operations successful, update the data dictionary memory cache. */ - table->ibd_file_missing = TRUE; + table->file_unreadable = true; table->flags2 |= DICT_TF2_DISCARDED; @@ -3315,8 +3341,6 @@ row_discard_tablespace_for_mysql( if (table == 0) { err = DB_TABLE_NOT_FOUND; - } else if (table->is_encrypted) { - err = DB_DECRYPTION_FAILED; } else if (dict_table_is_temporary(table)) { ib_senderrf(trx->mysql_thd, IB_LOG_LEVEL_ERROR, @@ -3487,7 +3511,6 @@ row_drop_ancillary_fts_tables( /* Drop ancillary FTS tables */ if (dict_table_has_fts_index(table) || DICT_TF2_FLAG_IS_SET(table, DICT_TF2_FTS_HAS_DOC_ID)) { - ut_ad(table->get_ref_count() == 0); ut_ad(trx_is_started(trx)); @@ -3655,18 +3678,6 @@ row_drop_table_for_mysql( err = DB_TABLE_NOT_FOUND; goto funct_exit; } - /* If table is encrypted and table page encryption failed - return error. */ - if (table->is_encrypted) { - - if (table->can_be_evicted) { - dict_table_move_from_lru_to_non_lru(table); - } - - dict_table_close(table, TRUE, FALSE); - err = DB_DECRYPTION_FAILED; - goto funct_exit; - } /* This function is called recursively via fts_drop_tables(). */ if (!trx_is_started(trx)) { @@ -4071,7 +4082,7 @@ row_drop_table_for_mysql( case DB_SUCCESS: space_id = table->space; - ibd_file_missing = table->ibd_file_missing; + ibd_file_missing = table->file_unreadable; is_discarded = dict_table_is_discarded(table); table_flags = table->flags; ut_ad(!dict_table_is_temporary(table)); @@ -4336,7 +4347,8 @@ loop: << table->name << ".frm' was lost."; } - if (table->ibd_file_missing) { + if (!table->is_readable() + && fil_space_get(table->space) == NULL) { ib::warn() << "Missing .ibd file for table " << table->name << "."; } @@ -4592,7 +4604,8 @@ row_rename_table_for_mysql( err = DB_TABLE_NOT_FOUND; goto funct_exit; - } else if (table->ibd_file_missing + } else if (!table->is_readable() + && fil_space_get(table->space) == NULL && !dict_table_is_discarded(table)) { err = DB_TABLE_NOT_FOUND; @@ -4659,7 +4672,7 @@ row_rename_table_for_mysql( the table is in a single-table tablespace. */ if (err == DB_SUCCESS && dict_table_is_file_per_table(table) - && !table->ibd_file_missing) { + && table->is_readable()) { /* Make a new pathname to update SYS_DATAFILES. */ char* new_path = row_make_new_pathname(table, new_name); diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc index 492d864ec96..37d3fd3ad89 100644 --- a/storage/innobase/row/row0purge.cc +++ b/storage/innobase/row/row0purge.cc @@ -880,6 +880,16 @@ try_again: innobase_init_vc_templ(node->table); } + if (!node->table->is_readable()) { + /* We skip purge of missing .ibd files */ + + dict_table_close(node->table, FALSE, FALSE); + + node->table = NULL; + + goto err_exit; + } + clust_index = dict_table_get_first_index(node->table); if (clust_index == NULL diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 229bd567c48..f7ba78c9f0b 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -4191,13 +4191,12 @@ row_search_mvcc( DBUG_RETURN(DB_TABLESPACE_DELETED); - } else if (prebuilt->table->ibd_file_missing) { - - DBUG_RETURN(DB_TABLESPACE_NOT_FOUND); - - } else if (prebuilt->table->is_encrypted) { - - return(DB_DECRYPTION_FAILED); + } else if (!prebuilt->table->is_readable()) { + if (fil_space_get(prebuilt->table->space) == NULL) { + return(DB_TABLESPACE_NOT_FOUND); + } else { + return(DB_DECRYPTION_FAILED); + } } else if (!prebuilt->index_usable) { DBUG_RETURN(DB_MISSING_HISTORY); @@ -4673,7 +4672,7 @@ wait_table_again: " used key_id is not available. " " Can't continue reading table.", prebuilt->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } rec = NULL; goto lock_wait_or_error; @@ -4695,7 +4694,7 @@ rec_loop: rec = btr_pcur_get_rec(pcur); - if (!rec) { + if (!index->table->is_readable()) { err = DB_DECRYPTION_FAILED; goto lock_wait_or_error; } @@ -4787,6 +4786,7 @@ wrong_offs: << ". Run CHECK TABLE. You may need to" " restore from a backup, or dump + drop +" " reimport the table."; + ut_ad(0); err = DB_CORRUPTION; diff --git a/storage/innobase/row/row0trunc.cc b/storage/innobase/row/row0trunc.cc index 1fc30e714f4..5724fad801f 100644 --- a/storage/innobase/row/row0trunc.cc +++ b/storage/innobase/row/row0trunc.cc @@ -1681,10 +1681,13 @@ row_truncate_sanity_checks( return(DB_TABLESPACE_DELETED); - } else if (table->ibd_file_missing) { - - return(DB_TABLESPACE_NOT_FOUND); + } else if (!table->is_readable()) { + if (fil_space_get(table->space) == NULL) { + return(DB_TABLESPACE_NOT_FOUND); + } else { + return(DB_DECRYPTION_FAILED); + } } else if (dict_table_is_corrupted(table)) { return(DB_TABLE_CORRUPT); diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 6d153ff1bc6..6be9c34ac5a 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -2210,6 +2210,11 @@ files_checked: recv_group_scan_log_recs(). */ recv_apply_hashed_log_recs(true); + + if (recv_sys->found_corrupt_log) { + return (DB_CORRUPTION); + } + DBUG_PRINT("ib_log", ("apply completed")); if (recv_needed_recovery) { diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index d6755fca7c0..38efcc3b154 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -773,7 +773,7 @@ trx_resurrect_table_locks( i != tables.end(); i++) { if (dict_table_t* table = dict_table_open_on_id( *i, FALSE, DICT_TABLE_OP_LOAD_TABLESPACE)) { - if (table->ibd_file_missing + if (!table->is_readable() || dict_table_is_temporary(table)) { mutex_enter(&dict_sys->mutex); dict_table_close(table, TRUE, FALSE); @@ -2456,6 +2456,121 @@ trx_print_latched( mem_heap_get_size(trx->lock.lock_heap)); } +#ifdef WITH_WSREP +/**********************************************************************//** +Prints info about a transaction. +Transaction information may be retrieved without having trx_sys->mutex acquired +so it may not be completely accurate. The caller must own lock_sys->mutex +and the trx must have some locks to make sure that it does not escape +without locking lock_sys->mutex. */ +UNIV_INTERN +void +wsrep_trx_print_locking( +/*==========*/ + FILE* f, + /*!< in: output stream */ + const trx_t* trx, + /*!< in: transaction */ + ulint max_query_len) + /*!< in: max query length to print, + or 0 to use the default max length */ +{ + ibool newline; + const char* op_info; + + ut_ad(lock_mutex_own()); + ut_ad(trx->lock.trx_locks.count > 0); + + fprintf(f, "TRANSACTION " TRX_ID_FMT, trx->id); + + /* trx->state may change since trx_sys->mutex is not required */ + switch (trx->state) { + case TRX_STATE_NOT_STARTED: + fputs(", not started", f); + goto state_ok; + case TRX_STATE_ACTIVE: + fprintf(f, ", ACTIVE %lu sec", + (ulong) difftime(time(NULL), trx->start_time)); + goto state_ok; + case TRX_STATE_PREPARED: + fprintf(f, ", ACTIVE (PREPARED) %lu sec", + (ulong) difftime(time(NULL), trx->start_time)); + goto state_ok; + case TRX_STATE_COMMITTED_IN_MEMORY: + fputs(", COMMITTED IN MEMORY", f); + goto state_ok; + case TRX_STATE_FORCED_ROLLBACK: + fputs(", FORCED ROLLBACK", f); + goto state_ok; + } + fprintf(f, ", state %lu", (ulong) trx->state); + ut_ad(0); +state_ok: + + /* prevent a race condition */ + op_info = trx->op_info; + + if (*op_info) { + putc(' ', f); + fputs(op_info, f); + } + + if (trx->is_recovered) { + fputs(" recovered trx", f); + } + + if (trx->declared_to_be_inside_innodb) { + fprintf(f, ", thread declared inside InnoDB %lu", + (ulong) trx->n_tickets_to_enter_innodb); + } + + putc('\n', f); + + if (trx->n_mysql_tables_in_use > 0 || trx->mysql_n_tables_locked > 0) { + fprintf(f, "mysql tables in use %lu, locked %lu\n", + (ulong) trx->n_mysql_tables_in_use, + (ulong) trx->mysql_n_tables_locked); + } + + newline = TRUE; + + /* trx->lock.que_state of an ACTIVE transaction may change + while we are not holding trx->mutex. We perform a dirty read + for performance reasons. */ + + switch (trx->lock.que_state) { + case TRX_QUE_RUNNING: + newline = FALSE; break; + case TRX_QUE_LOCK_WAIT: + fputs("LOCK WAIT ", f); break; + case TRX_QUE_ROLLING_BACK: + fputs("ROLLING BACK ", f); break; + case TRX_QUE_COMMITTING: + fputs("COMMITTING ", f); break; + default: + fprintf(f, "que state %lu ", (ulong) trx->lock.que_state); + } + + if (trx->has_search_latch) { + newline = TRUE; + fputs(", holds adaptive hash latch", f); + } + + if (trx->undo_no != 0) { + newline = TRUE; + fprintf(f, ", undo log entries " TRX_ID_FMT, trx->undo_no); + } + + if (newline) { + putc('\n', f); + } + + if (trx->mysql_thd != NULL) { + innobase_mysql_print_thd( + f, trx->mysql_thd, static_cast<uint>(max_query_len)); + } +} +#endif /* WITH_WSREP */ /**********************************************************************//** Prints info about a transaction. Acquires and releases lock_sys->mutex and trx_sys->mutex. */ diff --git a/storage/innobase/ut/ut0ut.cc b/storage/innobase/ut/ut0ut.cc index 2cae865cff2..862268fab0c 100644 --- a/storage/innobase/ut/ut0ut.cc +++ b/storage/innobase/ut/ut0ut.cc @@ -745,6 +745,8 @@ ut_strerr( return("Too many words in a FTS phrase or proximity search"); case DB_DECRYPTION_FAILED: return("Table is encrypted but decrypt failed."); + case DB_PAGE_CORRUPTED: + return("Page read from tablespace is corrupted."); case DB_IO_PARTIAL_FAILED: return("Partial IO failed"); case DB_FORCED_ABORT: |