diff options
Diffstat (limited to 'storage/xtradb')
41 files changed, 1141 insertions, 1137 deletions
diff --git a/storage/xtradb/api/api0api.cc b/storage/xtradb/api/api0api.cc index 739ea9f7572..bc83e98374f 100644 --- a/storage/xtradb/api/api0api.cc +++ b/storage/xtradb/api/api0api.cc @@ -247,7 +247,7 @@ ib_open_table_by_id( table = dict_table_open_on_id(table_id, TRUE, DICT_TABLE_OP_NORMAL); - if (table != NULL && table->ibd_file_missing) { + if (table != NULL && table->file_unreadable) { table = NULL; } @@ -272,7 +272,7 @@ ib_open_table_by_name( table = dict_table_open_on_name(name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); - if (table != NULL && table->ibd_file_missing) { + if (table != NULL && table->file_unreadable) { table = NULL; } @@ -292,7 +292,7 @@ ib_lookup_table_by_name( table = dict_table_get_low(name); - if (table != NULL && table->ibd_file_missing) { + if (table != NULL && table->file_unreadable) { table = NULL; } diff --git a/storage/xtradb/btr/btr0btr.cc b/storage/xtradb/btr/btr0btr.cc index 48411b6ff6a..c94b539c2c7 100644 --- a/storage/xtradb/btr/btr0btr.cc +++ b/storage/xtradb/btr/btr0btr.cc @@ -2,7 +2,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2014, 2016, MariaDB Corporation +Copyright (c) 2014, 2017, MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -744,8 +744,7 @@ btr_root_block_get( if (!block) { if (index && index->table) { - index->table->is_encrypted = TRUE; - index->table->corrupted = FALSE; + index->table->file_unreadable = true; ib_push_warning(index->table->thd, DB_DECRYPTION_FAILED, "Table %s in tablespace %lu is encrypted but encryption service or" @@ -760,6 +759,7 @@ btr_root_block_get( SRV_CORRUPT_TABLE_CHECK(block, return(0);); btr_assert_not_corrupted(block, index); + #ifdef UNIV_BTR_DEBUG if (!dict_index_is_ibuf(index)) { const page_t* root = buf_block_get_frame(block); @@ -5211,7 +5211,7 @@ btr_validate_index( page_t* root = btr_root_get(index, &mtr); - if (root == NULL && index->table->is_encrypted) { + if (root == NULL && index->table->file_unreadable) { err = DB_DECRYPTION_FAILED; mtr_commit(&mtr); return err; diff --git a/storage/xtradb/btr/btr0cur.cc b/storage/xtradb/btr/btr0cur.cc index ab1bcd37962..b5b3f1862f7 100644 --- a/storage/xtradb/btr/btr0cur.cc +++ b/storage/xtradb/btr/btr0cur.cc @@ -3,7 +3,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2015, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -658,7 +658,10 @@ retry_page_get: space, zip_size, page_no, rw_latch, guess, buf_mode, file, line, mtr, &err); + /* Note that block==NULL signifies either an error or change + buffering. */ if (err != DB_SUCCESS) { + ut_ad(block == NULL); if (err == DB_DECRYPTION_FAILED) { ib_push_warning((void *)NULL, DB_DECRYPTION_FAILED, @@ -666,7 +669,7 @@ retry_page_get: " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } goto func_exit; @@ -944,7 +947,7 @@ btr_cur_open_at_index_side_func( ulint offsets_[REC_OFFS_NORMAL_SIZE]; ulint* offsets = offsets_; dberr_t err = DB_SUCCESS; - + rec_offs_init(offsets_); estimate = latch_mode & BTR_ESTIMATE; @@ -988,6 +991,9 @@ btr_cur_open_at_index_side_func( block = buf_page_get_gen(space, zip_size, page_no, RW_NO_LATCH, NULL, BUF_GET, file, line, mtr, &err); + + ut_ad((block != NULL) == (err == DB_SUCCESS)); + if (err != DB_SUCCESS) { if (err == DB_DECRYPTION_FAILED) { ib_push_warning((void *)NULL, @@ -996,7 +1002,7 @@ btr_cur_open_at_index_side_func( " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } goto exit_loop; @@ -1161,6 +1167,8 @@ btr_cur_open_at_rnd_pos_func( RW_NO_LATCH, NULL, BUF_GET, file, line, mtr, &err); + ut_ad((block != NULL) == (err == DB_SUCCESS)); + if (err != DB_SUCCESS) { if (err == DB_DECRYPTION_FAILED) { ib_push_warning((void *)NULL, @@ -1169,8 +1177,9 @@ btr_cur_open_at_rnd_pos_func( " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } + goto exit_loop; } @@ -3853,6 +3862,8 @@ btr_estimate_n_rows_in_range_on_level( NULL, BUF_GET_POSSIBLY_FREED, __FILE__, __LINE__, &mtr, &err); + ut_ad((block != NULL) == (err == DB_SUCCESS)); + if (err != DB_SUCCESS) { if (err == DB_DECRYPTION_FAILED) { ib_push_warning((void *)NULL, @@ -3861,14 +3872,13 @@ btr_estimate_n_rows_in_range_on_level( " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } mtr_commit(&mtr); goto inexact; } - page = buf_block_get_frame(block); /* It is possible that the tree has been reorganized in the @@ -4010,6 +4020,10 @@ btr_estimate_n_rows_in_range_low( mtr_commit(&mtr); + if (index->table->file_unreadable) { + return (0); + } + mtr_start_trx(&mtr, trx); #ifdef UNIV_DEBUG @@ -4360,6 +4374,11 @@ btr_estimate_number_of_different_key_vals( because otherwise our algorithm would give a wrong estimate for an index where there is just one key value. */ + if (index->table->file_unreadable) { + mtr_commit(&mtr); + goto exit_loop; + } + page = btr_cur_get_page(&cursor); SRV_CORRUPT_TABLE_CHECK(page, goto exit_loop;); diff --git a/storage/xtradb/btr/btr0defragment.cc b/storage/xtradb/btr/btr0defragment.cc index ca00eed40a9..b3978722c1a 100644 --- a/storage/xtradb/btr/btr0defragment.cc +++ b/storage/xtradb/btr/btr0defragment.cc @@ -227,7 +227,7 @@ btr_defragment_add_index( page = buf_block_get_frame(block); } - if (page == NULL && index->table->is_encrypted) { + if (page == NULL && index->table->file_unreadable) { mtr_commit(&mtr); *err = DB_DECRYPTION_FAILED; return NULL; diff --git a/storage/xtradb/btr/btr0pcur.cc b/storage/xtradb/btr/btr0pcur.cc index dd6ef484fc3..0b970e1cf49 100644 --- a/storage/xtradb/btr/btr0pcur.cc +++ b/storage/xtradb/btr/btr0pcur.cc @@ -420,6 +420,11 @@ btr_pcur_move_to_next_page( cursor->old_stored = BTR_PCUR_OLD_NOT_STORED; page = btr_pcur_get_page(cursor); + + if (UNIV_UNLIKELY(!page)) { + return; + } + next_page_no = btr_page_get_next(page, mtr); space = buf_block_get_space(btr_pcur_get_block(cursor)); zip_size = buf_block_get_zip_size(btr_pcur_get_block(cursor)); @@ -429,6 +434,11 @@ btr_pcur_move_to_next_page( next_block = btr_block_get(space, zip_size, next_page_no, cursor->latch_mode, btr_pcur_get_btr_cur(cursor)->index, mtr); + + if (UNIV_UNLIKELY(!next_block)) { + return; + } + next_page = buf_block_get_frame(next_block); SRV_CORRUPT_TABLE_CHECK(next_page, diff --git a/storage/xtradb/btr/btr0scrub.cc b/storage/xtradb/btr/btr0scrub.cc index e9434c9f778..24c84ed301b 100644 --- a/storage/xtradb/btr/btr0scrub.cc +++ b/storage/xtradb/btr/btr0scrub.cc @@ -139,6 +139,7 @@ btr_scrub_lock_dict_func(ulint space_id, bool lock_to_close_table, } else { return false; } + os_thread_sleep(250000); time_t now = time(0); @@ -577,7 +578,7 @@ btr_scrub_table_needs_scrubbing( return false; } - if (table->corrupted) { + if (!table->is_readable()) { return false; } diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 394d2be6838..4593f84b43b 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -1181,7 +1181,6 @@ buf_block_init( block->page.buf_fix_count = 0; block->page.io_fix = BUF_IO_NONE; block->page.encrypted = false; - block->page.key_version = 0; block->page.real_size = 0; block->page.write_size = 0; block->modify_clock = 0; @@ -2351,7 +2350,17 @@ lookup: /* Page not in buf_pool: needs to be read from file */ ut_ad(!hash_lock); - buf_read_page(space, zip_size, offset, trx, NULL); + dberr_t err = buf_read_page(space, zip_size, offset, trx); + + if (err != DB_SUCCESS) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Reading compressed page " ULINTPF + ":" ULINTPF + " failed with error: %s.", + space, offset, ut_strerr(err)); + + goto err_exit; + } #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG ut_a(++buf_dbg_counter % 5771 || buf_validate()); @@ -2957,7 +2966,6 @@ loop: } if (block == NULL) { - buf_page_t* bpage=NULL; /* Page not in buf_pool: needs to be read from file */ @@ -2993,7 +3001,18 @@ loop: return(NULL); } - if (buf_read_page(space, zip_size, offset, trx, &bpage)) { + /* Call path is buf_read_page() -> buf_read_page_low() + (_fil_io()) -> buf_page_io_complete() -> + buf_decrypt_after_read() here fil_space_t* is used + and we decrypt -> buf_page_check_corrupt() where + page checksums are compared. Decryption/decompression + is handled lower level, error handling is handled on lower + level, here we need only to know is page really corrupted + or encrypted page with correct checksum. */ + + dberr_t local_err = buf_read_page(space, zip_size, offset, trx); + + if (local_err == DB_SUCCESS) { buf_read_ahead_random(space, zip_size, offset, ibuf_inside(mtr), trx); @@ -3001,89 +3020,47 @@ loop: } else if (retries < BUF_PAGE_READ_MAX_RETRIES) { ++retries; - bool corrupted = false; - - if (bpage) { - corrupted = buf_page_check_corrupt(bpage); - } - - /* Do not try again for encrypted pages */ - if (corrupted && bpage->encrypted) { - ib_mutex_t* pmutex = buf_page_get_mutex(bpage); - mutex_enter(&buf_pool->LRU_list_mutex); - mutex_enter(pmutex); - - ut_ad(buf_pool->n_pend_reads > 0); - os_atomic_decrement_ulint(&buf_pool->n_pend_reads, 1); - buf_page_set_io_fix(bpage, BUF_IO_NONE); - - if (!buf_LRU_free_page(bpage, true)) { - mutex_exit(&buf_pool->LRU_list_mutex); - } - - mutex_exit(pmutex); - rw_lock_x_unlock_gen(&((buf_block_t*) bpage)->lock, - BUF_IO_READ); - - if (err) { - *err = DB_DECRYPTION_FAILED; - } - - return (NULL); - } - DBUG_EXECUTE_IF( "innodb_page_corruption_retries", retries = BUF_PAGE_READ_MAX_RETRIES; ); } else { - bool corrupted = false; - - if (bpage) { - corrupted = buf_page_check_corrupt(bpage); + if (err) { + *err = local_err; } - if (corrupted && !bpage->encrypted) { - ib_logf(IB_LOG_LEVEL_ERROR, "Unable" - " to read tablespace %lu page no" - " %lu into the buffer pool after" - " %lu attempts\n" - "InnoDB: The most probable cause" - " of this error may be that the" - " table has been corrupted.\n" - "InnoDB: You can try to fix this" - " problem by using" - " innodb_force_recovery.\n" - "InnoDB: Please see reference manual" - " for more details.\n" - "InnoDB: Aborting...\n", - space, offset, - BUF_PAGE_READ_MAX_RETRIES); - - ut_error; - } else { - ib_mutex_t* pmutex = buf_page_get_mutex(bpage); - mutex_enter(&buf_pool->LRU_list_mutex); - mutex_enter(pmutex); - - ut_ad(buf_pool->n_pend_reads > 0); - os_atomic_decrement_ulint(&buf_pool->n_pend_reads, 1); - buf_page_set_io_fix(bpage, BUF_IO_NONE); - - if (!buf_LRU_free_page(bpage, true)) { - mutex_exit(&buf_pool->LRU_list_mutex); - } - - mutex_exit(pmutex); - rw_lock_x_unlock_gen(&((buf_block_t*) bpage)->lock, - BUF_IO_READ); - - if (err) { - *err = DB_DECRYPTION_FAILED; - } + /* Pages whose encryption key is unavailable or used + key, encryption algorithm or encryption method is + incorrect are marked as encrypted in + buf_page_check_corrupt(). Unencrypted page could be + corrupted in a way where the key_id field is + nonzero. There is no checksum on field + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION. */ + if (local_err == DB_DECRYPTION_FAILED) { + return (NULL); + } + /* Try to set table as corrupted instead of + asserting. */ + if (space > TRX_SYS_SPACE && + dict_set_corrupted_by_space(space)) { return (NULL); } + + ib_logf(IB_LOG_LEVEL_FATAL, "Unable" + " to read tablespace %lu page no" + " %lu into the buffer pool after" + " %lu attempts" + " The most probable cause" + " of this error may be that the" + " table has been corrupted." + " You can try to fix this" + " problem by using" + " innodb_force_recovery." + " Please see " REFMAN " for more" + " details. Aborting...", + space, offset, + BUF_PAGE_READ_MAX_RETRIES); } #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG @@ -3856,7 +3833,6 @@ buf_page_init_low( bpage->oldest_modification = 0; bpage->write_size = 0; bpage->encrypted = false; - bpage->key_version = 0; bpage->real_size = 0; HASH_INVALIDATE(bpage, hash); @@ -4499,61 +4475,55 @@ buf_page_monitor( /********************************************************************//** Mark a table with the specified space pointed by bpage->space corrupted. Also remove the bpage from LRU list. -@return TRUE if successful */ +@param[in,out] bpage Block */ static -ibool +void buf_mark_space_corrupt( -/*===================*/ - buf_page_t* bpage) /*!< in: pointer to the block in question */ + buf_page_t* bpage) { buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); const ibool uncompressed = (buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); ulint space = bpage->space; - ibool ret = TRUE; const ulint fold = buf_page_address_fold(bpage->space, bpage->offset); prio_rw_lock_t* hash_lock = buf_page_hash_lock_get(buf_pool, fold); + ib_mutex_t* block_mutex = buf_page_get_mutex(bpage); /* First unfix and release lock on the bpage */ ut_ad(!mutex_own(&buf_pool->LRU_list_mutex)); - if (!bpage->encrypted) { - mutex_enter(&buf_pool->LRU_list_mutex); - rw_lock_x_lock(hash_lock); - mutex_enter(buf_page_get_mutex(bpage)); - ut_ad(buf_page_get_io_fix(bpage) == BUF_IO_READ); - ut_ad(bpage->buf_fix_count == 0); + mutex_enter(&buf_pool->LRU_list_mutex); + rw_lock_x_lock(hash_lock); + mutex_enter(block_mutex); + ut_ad(buf_page_get_io_fix(bpage) == BUF_IO_READ); + ut_ad(bpage->buf_fix_count == 0); - /* Set BUF_IO_NONE before we remove the block from LRU list */ - buf_page_set_io_fix(bpage, BUF_IO_NONE); + /* Set BUF_IO_NONE before we remove the block from LRU list */ + buf_page_set_io_fix(bpage, BUF_IO_NONE); - if (uncompressed) { - rw_lock_x_unlock_gen( - &((buf_block_t*) bpage)->lock, - BUF_IO_READ); - } + if (uncompressed) { + rw_lock_x_unlock_gen( + &((buf_block_t*) bpage)->lock, + BUF_IO_READ); } - /* Find the table with specified space id, and mark it corrupted */ - if (dict_set_corrupted_by_space(space)) { - if (!bpage->encrypted) { - buf_LRU_free_one_page(bpage); - } + /* If block is not encrypted find the table with specified + space id, and mark it corrupted. Encrypted tables + are marked unusable later e.g. in ::open(). */ + if (!bpage->encrypted) { + dict_set_corrupted_by_space(space); } else { - if (!bpage->encrypted) { - mutex_exit(buf_page_get_mutex(bpage)); - } - ret = FALSE; + dict_set_encrypted_by_space(space); } - if(!bpage->encrypted) { - mutex_exit(&buf_pool->LRU_list_mutex); - ut_ad(buf_pool->n_pend_reads > 0); - os_atomic_decrement_ulint(&buf_pool->n_pend_reads, 1); - } + /* After this point bpage can't be referenced. This + function will release the hash_lock acquired above. */ + buf_LRU_free_one_page(bpage); - return(ret); + ut_ad(buf_pool->n_pend_reads > 0); + os_atomic_decrement_ulint(&buf_pool->n_pend_reads, 1); + mutex_exit(&buf_pool->LRU_list_mutex); } /********************************************************************//** @@ -4561,24 +4531,29 @@ Check if page is maybe compressed, encrypted or both when we encounter corrupted page. Note that we can't be 100% sure if page is corrupted or decrypt/decompress just failed. @param[in,out] bpage Page -@return true if page corrupted, false if not */ -UNIV_INTERN -bool -buf_page_check_corrupt( - buf_page_t* bpage) +@return DB_SUCCESS if page has been read and is not corrupted, +@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted, +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but +after decryption normal page checksum does not match. +@retval DB_TABLESPACE_DELETED if accessed tablespace is not found */ +static +dberr_t +buf_page_check_corrupt(buf_page_t* bpage) { ulint zip_size = buf_page_get_zip_size(bpage); byte* dst_frame = (zip_size) ? bpage->zip.data : ((buf_block_t*) bpage)->frame; - ulint space_id = bpage->space; - fil_space_t* space = fil_space_acquire_silent(space_id); + FilSpace space(bpage->space, true); bool still_encrypted = false; + dberr_t err = DB_SUCCESS; bool corrupted = false; - ulint page_type = mach_read_from_2(dst_frame + FIL_PAGE_TYPE); fil_space_crypt_t* crypt_data = NULL; - ut_ad(space); - crypt_data = space->crypt_data; + if (!space()) { + return(DB_TABLESPACE_DELETED); + } + + crypt_data = space()->crypt_data; /* In buf_decrypt_after_read we have either decrypted the page if page post encryption checksum matches and used key_id is found @@ -4590,62 +4565,65 @@ buf_page_check_corrupt( crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && !bpage->encrypted && fil_space_verify_crypt_checksum(dst_frame, zip_size, - space, bpage->offset)); + space(), bpage->offset)); if (!still_encrypted) { /* If traditional checksums match, we assume that page is not anymore encrypted. */ - corrupted = buf_page_is_corrupted(true, dst_frame, zip_size, space); + corrupted = buf_page_is_corrupted(true, dst_frame, zip_size, space()); if (!corrupted) { bpage->encrypted = false; + } else { + err = DB_PAGE_CORRUPTED; } } /* Pages that we think are unencrypted but do not match the checksum checks could be corrupted or encrypted or both. */ if (corrupted && !bpage->encrypted) { - ib_logf(IB_LOG_LEVEL_ERROR, - "%s: Block in space_id " ULINTPF " in file %s corrupted.", - page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED ? "Maybe corruption" : "Corruption", - space_id, (space && space->name) ? space->name : "NULL"); - ib_logf(IB_LOG_LEVEL_ERROR, - "Based on page type %s (" ULINTPF ")", - fil_get_page_type_name(page_type), page_type); + /* An error will be reported by + buf_page_io_complete(). */ } else if (still_encrypted || (bpage->encrypted && corrupted)) { bpage->encrypted = true; - corrupted = true; + err = DB_DECRYPTION_FAILED; ib_logf(IB_LOG_LEVEL_ERROR, - "Block in space_id " ULINTPF " in file %s encrypted.", - space_id, (space && space->name) ? space->name : "NULL"); - ib_logf(IB_LOG_LEVEL_ERROR, - "However key management plugin or used key_version %u is not found or" - " used encryption algorithm or method does not match.", - bpage->key_version); - if (space_id > TRX_SYS_SPACE) { - ib_logf(IB_LOG_LEVEL_ERROR, + "The page [page id: space=%u" + ", page number=%u]" + " in file %s cannot be decrypted.", + bpage->space, bpage->offset, + space()->name); + + ib_logf(IB_LOG_LEVEL_INFO, + "However key management plugin or used key_version " ULINTPF + " is not found or" + " used encryption algorithm or method does not match.", + mach_read_from_4(dst_frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION)); + + if (bpage->space > TRX_SYS_SPACE) { + ib_logf(IB_LOG_LEVEL_INFO, "Marking tablespace as missing. You may drop this table or" " install correct key management plugin and key file."); } } - if (space) { - fil_space_release(space); - } - - return corrupted; + return (err); } /********************************************************************//** Completes an asynchronous read or write request of a file page to or from the buffer pool. -@return true if successful */ +@param[in,out] bpage Page to complete +@return DB_SUCCESS if page has been read and is not corrupted, +DB_PAGE_CORRUPTED if page based on checksum check is corrupted, +DB_DECRYPTION_FAILED if page post encryption checksum matches but +after decryption normal page checksum does not match. +in write only DB_SUCCESS is possible. */ UNIV_INTERN -bool +dberr_t buf_page_io_complete( -/*=================*/ - buf_page_t* bpage) /*!< in: pointer to the block in question */ + buf_page_t* bpage) { enum buf_io_fix io_type; buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); @@ -4653,8 +4631,8 @@ buf_page_io_complete( == BUF_BLOCK_FILE_PAGE); bool have_LRU_mutex = false; fil_space_t* space = NULL; - byte* frame = NULL; - bool corrupted = false; + byte* frame = NULL; + dberr_t err = DB_SUCCESS; ut_a(buf_page_in_file(bpage)); @@ -4668,8 +4646,9 @@ buf_page_io_complete( ut_ad(io_type == BUF_IO_READ || io_type == BUF_IO_WRITE); if (io_type == BUF_IO_READ) { - ulint read_page_no; - ulint read_space_id; + ulint read_page_no = 0; + ulint read_space_id = 0; + uint key_version = 0; buf_page_decrypt_after_read(bpage); @@ -4693,10 +4672,11 @@ buf_page_io_complete( "Page %u in tablespace %u zip_decompress failure.", bpage->offset, bpage->space); - corrupted = true; + err = DB_PAGE_CORRUPTED; goto database_corrupted; } + os_atomic_decrement_ulint(&buf_pool->n_pend_unzip, 1); } else { ut_a(uncompressed); @@ -4709,6 +4689,8 @@ buf_page_io_complete( read_page_no = mach_read_from_4(frame + FIL_PAGE_OFFSET); read_space_id = mach_read_from_4( frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + key_version = mach_read_from_4( + frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); if (bpage->space == TRX_SYS_SPACE && buf_dblwr_page_inside(bpage->offset)) { @@ -4729,12 +4711,11 @@ buf_page_io_complete( page may contain garbage in MySQL < 4.1.1, which only supported bpage->space == 0. */ - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: space id and page n:o" - " stored in the page\n" - "InnoDB: read in are " ULINTPF ":" ULINTPF "," - " should be %u:%u!\n", + ib_logf(IB_LOG_LEVEL_ERROR, + "Space id and page n:o" + " stored in the page" + " read in are " ULINTPF ":" ULINTPF "," + " should be %u:%u!", read_space_id, read_page_no, bpage->space, @@ -4743,81 +4724,68 @@ buf_page_io_complete( if (UNIV_LIKELY(!bpage->is_corrupt || !srv_pass_corrupt_table)) { - corrupted = buf_page_check_corrupt(bpage); - + err = buf_page_check_corrupt(bpage); } database_corrupted: - if (corrupted) { + if (err != DB_SUCCESS) { /* Not a real corruption if it was triggered by error injection */ - DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", - if (bpage->space > TRX_SYS_SPACE - && buf_mark_space_corrupt(bpage)) { + if (bpage->space > TRX_SYS_SPACE) { + buf_mark_space_corrupt(bpage); ib_logf(IB_LOG_LEVEL_INFO, "Simulated page corruption"); - return(true); + return(err); } + err = DB_SUCCESS; goto page_not_corrupt; ); - if (!bpage->encrypted) { + if (err == DB_PAGE_CORRUPTED) { fil_system_enter(); space = fil_space_get_by_id(bpage->space); - fil_system_exit(); + ib_logf(IB_LOG_LEVEL_ERROR, "Database page corruption on disk" - " or a failed"); - ib_logf(IB_LOG_LEVEL_ERROR, - "Space %u file %s read of page %u.", - bpage->space, - space->name ? space->name : "NULL", - bpage->offset); - ib_logf(IB_LOG_LEVEL_ERROR, - "You may have to recover" - " from a backup."); + " or a failed file read of tablespace %s" + " page [page id: space=%u" + ", page number=%u]" + ". You may have to recover from " + "a backup.", + space->name, + bpage->space, bpage->offset); + + fil_system_exit(); buf_page_print(frame, buf_page_get_zip_size(bpage), BUF_PAGE_PRINT_NO_CRASH); - ib_logf(IB_LOG_LEVEL_ERROR, - "It is also possible that your operating" - "system has corrupted its own file cache."); - ib_logf(IB_LOG_LEVEL_ERROR, - "and rebooting your computer removes the error."); - ib_logf(IB_LOG_LEVEL_ERROR, - "If the corrupt page is an index page you can also try to"); - ib_logf(IB_LOG_LEVEL_ERROR, - "fix the corruption by dumping, dropping, and reimporting"); - ib_logf(IB_LOG_LEVEL_ERROR, - "the corrupt table. You can use CHECK"); - ib_logf(IB_LOG_LEVEL_ERROR, - "TABLE to scan your table for corruption."); - ib_logf(IB_LOG_LEVEL_ERROR, - "See also " - REFMAN "forcing-innodb-recovery.html" - " about forcing recovery."); + ib_logf(IB_LOG_LEVEL_INFO, + "It is also possible that your" + " operating system has corrupted" + " its own file cache and rebooting" + " your computer removes the error." + " If the corrupt page is an index page." + " You can also try to fix the" + " corruption by dumping, dropping," + " and reimporting the corrupt table." + " You can use CHECK TABLE to scan" + " your table for corruption. " + "Please refer to " REFMAN "forcing-innodb-recovery.html" + " for information about forcing recovery."); } if (srv_pass_corrupt_table && bpage->space != 0 && bpage->space < SRV_LOG_SPACE_FIRST_ID) { - trx_t* trx; - fprintf(stderr, - "InnoDB: space %u will be treated as corrupt.\n", + ib_logf(IB_LOG_LEVEL_INFO, + "InnoDB: space %u will be treated as corrupt.", bpage->space); fil_space_set_corrupt(bpage->space); - trx = innobase_get_trx(); - - if (trx && trx->dict_operation_lock_mode == RW_X_LATCH) { - dict_table_set_corrupt_by_space(bpage->space, FALSE); - } else { - dict_table_set_corrupt_by_space(bpage->space, TRUE); - } - + dict_set_corrupted_by_space(bpage->space); bpage->is_corrupt = TRUE; } @@ -4825,31 +4793,12 @@ database_corrupted: /* If page space id is larger than TRX_SYS_SPACE (0), we will attempt to mark the corresponding table as corrupted instead of crashing server */ - if (bpage->space > TRX_SYS_SPACE - && buf_mark_space_corrupt(bpage)) { - return(false); + if (bpage->space > TRX_SYS_SPACE) { + buf_mark_space_corrupt(bpage); + return(err); } else { - if (!bpage->encrypted) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Ending processing because of a corrupt database page."); - - ut_error; - } - - ib_push_warning(innobase_get_trx(), DB_DECRYPTION_FAILED, - "Table in tablespace %lu encrypted." - "However key management plugin or used key_id %lu is not found or" - " used encryption algorithm or method does not match." - " Can't continue opening the table.", - bpage->space, bpage->key_version); - - if (bpage->encrypted && bpage->space > TRX_SYS_SPACE) { - buf_mark_space_corrupt(bpage); - } else { - ut_error; - } - - return(false); + ib_logf(IB_LOG_LEVEL_FATAL, + "Ending processing because of a corrupt database page."); } } } @@ -4869,11 +4818,12 @@ database_corrupted: if (bpage && bpage->encrypted) { ib_logf(IB_LOG_LEVEL_WARN, - "Table in tablespace %lu encrypted." - "However key management plugin or used key_version %u is not found or" + "Table in tablespace " ULINTPF " encrypted." + "However key management plugin or used " + " key_version %u is not found or" " used encryption algorithm or method does not match." - " Can't continue opening the table.\n", - (ulint)bpage->space, bpage->key_version); + " Can't continue opening the table.", + read_space_id, key_version); } else { ibuf_merge_or_delete_for_page( @@ -4995,7 +4945,7 @@ retry_mutex: mutex_exit(block_mutex); - return(true); + return(err); } /*********************************************************************//** @@ -5023,7 +4973,7 @@ buf_all_freed_instance( mutex_exit(&buf_pool->LRU_list_mutex); - if (UNIV_LIKELY_NULL(block) && block->page.key_version == 0) { + if (UNIV_LIKELY_NULL(block)) { fil_space_t* space = fil_space_get(block->page.space); ib_logf(IB_LOG_LEVEL_ERROR, "Page %u %u still fixed or dirty.", @@ -6263,32 +6213,29 @@ buf_page_encrypt_before_write( switch (bpage->offset) { case 0: /* Page 0 of a tablespace is not encrypted/compressed */ - ut_ad(bpage->key_version == 0); return src_frame; case TRX_SYS_PAGE_NO: if (bpage->space == TRX_SYS_SPACE) { /* don't encrypt/compress page as it contains address to dblwr buffer */ - bpage->key_version = 0; return src_frame; } } fil_space_crypt_t* crypt_data = space->crypt_data; + const bool encrypted = crypt_data && !crypt_data->not_encrypted() && crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && (!crypt_data->is_default_encryption() || srv_encrypt_tables); - if (!encrypted) { - bpage->key_version = 0; - } - bool page_compressed = FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags); if (!encrypted && !page_compressed) { - /* No need to encrypt or page compress the page */ + /* No need to encrypt or page compress the page. + Clear key-version & crypt-checksum. */ + memset(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); return src_frame; } @@ -6310,9 +6257,6 @@ buf_page_encrypt_before_write( src_frame, dst_frame); - ulint key_version = mach_read_from_4(dst_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - ut_ad(key_version == 0 || key_version >= bpage->key_version); - bpage->key_version = key_version; bpage->real_size = page_size; slot->out_buf = dst_frame = tmp; @@ -6381,19 +6325,17 @@ buf_page_decrypt_after_read( buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); bool success = true; - bpage->key_version = key_version; - if (bpage->offset == 0) { /* File header pages are not encrypted/compressed */ return (true); } - fil_space_t* space = fil_space_acquire(bpage->space, true); + FilSpace space(bpage->space, false, true); /* Page is encrypted if encryption information is found from tablespace and page contains used key_version. This is true also for pages first compressed and then encrypted. */ - if (!space || !space->crypt_data) { + if (!space() || !space()->crypt_data) { key_version = 0; } @@ -6427,6 +6369,14 @@ buf_page_decrypt_after_read( decrypt. */ if (!fil_space_verify_crypt_checksum(dst_frame, zip_size, NULL, bpage->offset)) { + + /* Mark page encrypted in case it should + be. */ + if (key_version && space()->crypt_data && + space()->crypt_data->type != CRYPT_SCHEME_UNENCRYPTED) { + bpage->encrypted = true; + } + return (false); } @@ -6438,7 +6388,7 @@ buf_page_decrypt_after_read( #endif /* decrypt using crypt_buf to dst_frame */ - byte* res = fil_space_decrypt(space, + byte* res = fil_space_decrypt(space(), slot->crypt_buf, dst_frame, &bpage->encrypted); @@ -6474,8 +6424,5 @@ buf_page_decrypt_after_read( } } - if (space != NULL) { - fil_space_release(space); - } return (success); } diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index 55c5e4d543a..1c9646c0bd6 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -493,7 +493,9 @@ buf_dblwr_process() page_no = mach_read_from_4(page + FIL_PAGE_OFFSET); space_id = mach_read_from_4(page + FIL_PAGE_SPACE_ID); - if (!fil_tablespace_exists_in_mem(space_id)) { + FilSpace space(space_id, true); + + if (!space()) { /* Maybe we have dropped the single-table tablespace and this page once belonged to it: do nothing */ continue; @@ -508,8 +510,7 @@ buf_dblwr_process() continue; } - fil_space_t* space = fil_space_found_by_id(space_id); - ulint zip_size = fil_space_get_zip_size(space_id); + ulint zip_size = fsp_flags_get_zip_size(space()->flags); ut_ad(!buf_page_is_zeroes(page, zip_size)); /* Read in the actual page from the file */ @@ -545,7 +546,7 @@ buf_dblwr_process() if (fil_space_verify_crypt_checksum( read_buf, zip_size, NULL, page_no) || !buf_page_is_corrupted( - true, read_buf, zip_size, space)) { + true, read_buf, zip_size, space())) { /* The page is good; there is no need to consult the doublewrite buffer. */ continue; diff --git a/storage/xtradb/buf/buf0flu.cc b/storage/xtradb/buf/buf0flu.cc index f71bdb9f69a..dc47f6f486d 100644 --- a/storage/xtradb/buf/buf0flu.cc +++ b/storage/xtradb/buf/buf0flu.cc @@ -994,7 +994,13 @@ buf_flush_write_block_low( if (sync) { ut_ad(flush_type == BUF_FLUSH_SINGLE_PAGE); fil_flush(space); + +#ifdef UNIV_DEBUG + dberr_t err = +#endif buf_page_io_complete(bpage); + + ut_ad(err == DB_SUCCESS); } fil_space_release(space); diff --git a/storage/xtradb/buf/buf0rea.cc b/storage/xtradb/buf/buf0rea.cc index be2ee56504b..e275eead4cc 100644 --- a/storage/xtradb/buf/buf0rea.cc +++ b/storage/xtradb/buf/buf0rea.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2014, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -103,36 +103,46 @@ flag is cleared and the x-lock released by an i/o-handler thread. in buf_pool, or if the page is in the doublewrite buffer blocks in which case it is never read into the pool, or if the tablespace does not exist or is being dropped -@return 1 if read request is issued. 0 if it is not */ -UNIV_INTERN -ulint -buf_read_page_low( -/*==============*/ - dberr_t* err, /*!< out: DB_SUCCESS or DB_TABLESPACE_DELETED if we are - trying to read from a non-existent tablespace, or a - tablespace which is just now being dropped */ - bool sync, /*!< in: true if synchronous aio is desired */ - ulint mode, /*!< in: BUF_READ_IBUF_PAGES_ONLY, ..., - ORed to OS_AIO_SIMULATED_WAKE_LATER (see below - at read-ahead functions) */ - ulint space, /*!< in: space id */ - ulint zip_size,/*!< in: compressed page size, or 0 */ - ibool unzip, /*!< in: TRUE=request uncompressed page */ - ib_int64_t tablespace_version, /*!< in: if the space memory object has + +@param[out] err DB_SUCCESS, DB_TABLESPACE_DELETED if we are + trying to read from a non-existent tablespace, or a + tablespace which is just now being dropped, + DB_PAGE_CORRUPTED if page based on checksum + check is corrupted, or DB_DECRYPTION_FAILED + if page post encryption checksum matches but + after decryption normal page checksum does not match. +@param[in] sync true if synchronous aio is desired +@param[in] mode BUF_READ_IBUF_PAGES_ONLY, ..., + ORed to OS_AIO_SIMULATED_WAKE_LATER (see below + at read-ahead functions) +@param[in] space space id +@param[in] zip_size compressed page size, or 0 +@param[in] unzip TRUE=request uncompressed page +@param[in] tablespace_version if the space memory object has this timestamp different from what we are giving here, treat the tablespace as dropped; this is a timestamp we use to stop dangling page reads from a tablespace - which we have DISCARDed + IMPORTed back */ - ulint offset, /*!< in: page number */ - trx_t* trx, /*!< in: trx */ - buf_page_t** rbpage) /*!< out: page */ + which we have DISCARDed + IMPORTed back +@param[in] offset page number +@param[in] trx transaction +@return 1 if read request is issued. 0 if it is not */ +static +ulint +buf_read_page_low( + dberr_t* err, + bool sync, + ulint mode, + ulint space, + ulint zip_size, + ibool unzip, + ib_int64_t tablespace_version, + ulint offset, + trx_t* trx = NULL) { buf_page_t* bpage; ulint wake_later; ibool ignore_nonexistent_pages; - *err = DB_SUCCESS; - wake_later = mode & OS_AIO_SIMULATED_WAKE_LATER; mode = mode & ~OS_AIO_SIMULATED_WAKE_LATER; @@ -259,18 +269,13 @@ not_to_recover: if (sync) { /* The i/o is already completed when we arrive from fil_read */ - if (!buf_page_io_complete(bpage)) { - if (rbpage) { - *rbpage = bpage; - } + *err = buf_page_io_complete(bpage); + + if (*err != DB_SUCCESS) { return(0); } } - if (rbpage) { - *rbpage = bpage; - } - return(1); } @@ -307,7 +312,7 @@ buf_read_ahead_random( ulint ibuf_mode; ulint count; ulint low, high; - dberr_t err; + dberr_t err = DB_SUCCESS; ulint i; const ulint buf_read_ahead_random_area = BUF_READ_AHEAD_AREA(buf_pool); @@ -402,20 +407,33 @@ read_ahead: mode: hence FALSE as the first parameter */ if (!ibuf_bitmap_page(zip_size, i)) { + count += buf_read_page_low( &err, false, ibuf_mode | OS_AIO_SIMULATED_WAKE_LATER, space, zip_size, FALSE, - tablespace_version, i, trx, NULL); - if (err == DB_TABLESPACE_DELETED) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Warning: in random" - " readahead trying to access\n" - "InnoDB: tablespace %lu page %lu,\n" - "InnoDB: but the tablespace does not" - " exist or is just being dropped.\n", - (ulong) space, (ulong) i); + tablespace_version, i, trx); + + switch(err) { + case DB_SUCCESS: + break; + case DB_TABLESPACE_DELETED: + ib_logf(IB_LOG_LEVEL_WARN, + "In random" + " readahead trying to access" + " tablespace " ULINTPF ":" ULINTPF + " but the tablespace does not" + " exist or is just being dropped.", + space, i); + break; + case DB_DECRYPTION_FAILED: + ib_logf(IB_LOG_LEVEL_ERROR, + "Random readahead failed to decrypt page " + ULINTPF ":" ULINTPF ".", + i, space); + break; + default: + ut_error; } } } @@ -449,44 +467,60 @@ High-level function which reads a page asynchronously from a file to the buffer buf_pool if it is not already there. Sets the io_fix flag and sets an exclusive lock on the buffer frame. The flag is cleared and the x-lock released by the i/o-handler thread. -@return TRUE if page has been read in, FALSE in case of failure */ + +@param[in] space_id space_id +@param[in] zip_size compressed page size in bytes, or 0 +@param[in] offset page number +@param[in] trx transaction +@param[out] encrypted true if page encrypted +@return DB_SUCCESS if page has been read and is not corrupted, +@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted, +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but +after decryption normal page checksum does not match. +@retval DB_TABLESPACE_DELETED if tablespace .ibd file is missing */ UNIV_INTERN -ibool +dberr_t buf_read_page( -/*==========*/ - ulint space, /*!< in: space id */ - ulint zip_size, /*!< in: compressed page size in bytes, or 0 */ - ulint offset, /*!< in: page number */ - trx_t* trx, /*!< in: trx */ - buf_page_t** bpage) /*!< out: page */ + ulint space_id, + ulint zip_size, + ulint offset, + trx_t* trx) { ib_int64_t tablespace_version; ulint count; - dberr_t err; + dberr_t err = DB_SUCCESS; - tablespace_version = fil_space_get_version(space); + tablespace_version = fil_space_get_version(space_id); - /* We do the i/o in the synchronous aio mode to save thread - switches: hence TRUE */ + FilSpace space(space_id, true); - count = buf_read_page_low(&err, true, BUF_READ_ANY_PAGE, space, + if (space()) { + + /* We do the i/o in the synchronous aio mode to save thread + switches: hence TRUE */ + count = buf_read_page_low(&err, true, BUF_READ_ANY_PAGE, space_id, zip_size, FALSE, - tablespace_version, offset, trx, bpage); - srv_stats.buf_pool_reads.add(count); - if (err == DB_TABLESPACE_DELETED) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: trying to access" - " tablespace %lu page no. %lu,\n" - "InnoDB: but the tablespace does not exist" - " or is just being dropped.\n", - (ulong) space, (ulong) offset); + tablespace_version, offset, trx); + + srv_stats.buf_pool_reads.add(count); + } + + /* Page corruption and decryption failures are already reported + in above function. */ + if (!space() || err == DB_TABLESPACE_DELETED) { + err = DB_TABLESPACE_DELETED; + ib_logf(IB_LOG_LEVEL_ERROR, + "Trying to access" + " tablespace [space=" ULINTPF ": page=" ULINTPF + "] but the tablespace does not exist" + " or is just being dropped.", + space_id, offset); } /* Increment number of I/O operations used for LRU policy. */ buf_LRU_stat_inc_io(); - return(count > 0); + return(err); } /********************************************************************//** @@ -494,23 +528,23 @@ High-level function which reads a page asynchronously from a file to the buffer buf_pool if it is not already there. Sets the io_fix flag and sets an exclusive lock on the buffer frame. The flag is cleared and the x-lock released by the i/o-handler thread. -@return TRUE if page has been read in, FALSE in case of failure */ +@param[in] space Tablespace id +@param[in] offset Page no */ UNIV_INTERN -ibool +void buf_read_page_async( -/*================*/ - ulint space, /*!< in: space id */ - ulint offset) /*!< in: page number */ + ulint space, + ulint offset) { ulint zip_size; ib_int64_t tablespace_version; ulint count; - dberr_t err; + dberr_t err = DB_SUCCESS; zip_size = fil_space_get_zip_size(space); if (zip_size == ULINT_UNDEFINED) { - return(FALSE); + return; } tablespace_version = fil_space_get_version(space); @@ -519,7 +553,30 @@ buf_read_page_async( | OS_AIO_SIMULATED_WAKE_LATER | BUF_READ_IGNORE_NONEXISTENT_PAGES, space, zip_size, FALSE, - tablespace_version, offset, NULL,NULL); + tablespace_version, offset); + + switch(err) { + case DB_SUCCESS: + break; + case DB_TABLESPACE_DELETED: + ib_logf(IB_LOG_LEVEL_ERROR, + "In async page read " + "trying to access " + "page " ULINTPF ":" ULINTPF + " in nonexisting or being-dropped tablespace", + space, offset); + break; + + case DB_DECRYPTION_FAILED: + ib_logf(IB_LOG_LEVEL_ERROR, + "Async page read failed to decrypt page " + ULINTPF ":" ULINTPF ".", + space, offset); + break; + default: + ut_error; + } + srv_stats.buf_pool_reads.add(count); /* We do not increment number of I/O operations used for LRU policy @@ -528,8 +585,6 @@ buf_read_page_async( buffer pool. Since this function is called from buffer pool load these IOs are deliberate and are not part of normal workload we can ignore these in our heuristics. */ - - return(count > 0); } /********************************************************************//** @@ -580,7 +635,7 @@ buf_read_ahead_linear( ulint fail_count; ulint ibuf_mode; ulint low, high; - dberr_t err; + dberr_t err = DB_SUCCESS; ulint i; const ulint buf_read_ahead_linear_area = BUF_READ_AHEAD_AREA(buf_pool); @@ -784,19 +839,34 @@ buf_read_ahead_linear( aio mode: hence FALSE as the first parameter */ if (!ibuf_bitmap_page(zip_size, i)) { + count += buf_read_page_low( &err, false, ibuf_mode, - space, zip_size, FALSE, tablespace_version, i, trx, NULL); - if (err == DB_TABLESPACE_DELETED) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Warning: in" - " linear readahead trying to access\n" - "InnoDB: tablespace %lu page %lu,\n" - "InnoDB: but the tablespace does not" - " exist or is just being dropped.\n", - (ulong) space, (ulong) i); + space, zip_size, FALSE, tablespace_version, + i, trx); + + switch(err) { + case DB_SUCCESS: + break; + case DB_TABLESPACE_DELETED: + ib_logf(IB_LOG_LEVEL_WARN, + "In linear" + " readahead trying to access" + " tablespace " ULINTPF ":" ULINTPF + " but the tablespace does not" + " exist or is just being dropped.", + space, i); + break; + + case DB_DECRYPTION_FAILED: + ib_logf(IB_LOG_LEVEL_ERROR, + "Linear readahead failed to decrypt page " + ULINTPF ":" ULINTPF ".", + i, space); + break; + default: + ut_error; } } } @@ -858,9 +928,9 @@ buf_read_ibuf_merge_pages( #endif for (i = 0; i < n_stored; i++) { - dberr_t err; buf_pool_t* buf_pool; ulint zip_size = fil_space_get_zip_size(space_ids[i]); + dberr_t err = DB_SUCCESS; buf_pool = buf_pool_get(space_ids[i], page_nos[i]); @@ -870,16 +940,19 @@ buf_read_ibuf_merge_pages( } if (UNIV_UNLIKELY(zip_size == ULINT_UNDEFINED)) { - goto tablespace_deleted; } buf_read_page_low(&err, sync && (i + 1 == n_stored), BUF_READ_ANY_PAGE, space_ids[i], zip_size, TRUE, space_versions[i], - page_nos[i], NULL, NULL); + page_nos[i], NULL); + + switch(err) { + case DB_SUCCESS: + break; + case DB_TABLESPACE_DELETED: - if (UNIV_UNLIKELY(err == DB_TABLESPACE_DELETED)) { tablespace_deleted: /* We have deleted or are deleting the single-table tablespace: remove the entries for that page */ @@ -887,6 +960,15 @@ tablespace_deleted: ibuf_merge_or_delete_for_page(NULL, space_ids[i], page_nos[i], zip_size, FALSE); + break; + case DB_DECRYPTION_FAILED: + ib_logf(IB_LOG_LEVEL_ERROR, + "Failed to decrypt insert buffer page " + ULINTPF ":" ULINTPF ".", + space_ids[i], page_nos[i]); + break; + default: + ut_error; } } @@ -924,7 +1006,7 @@ buf_read_recv_pages( { ib_int64_t tablespace_version; ulint count; - dberr_t err; + dberr_t err = DB_SUCCESS; ulint i; zip_size = fil_space_get_zip_size(space); @@ -1013,12 +1095,20 @@ not_to_recover: if ((i + 1 == n_stored) && sync) { buf_read_page_low(&err, true, BUF_READ_ANY_PAGE, space, zip_size, TRUE, tablespace_version, - page_nos[i], NULL, NULL); + page_nos[i], NULL); } else { buf_read_page_low(&err, false, BUF_READ_ANY_PAGE | OS_AIO_SIMULATED_WAKE_LATER, space, zip_size, TRUE, - tablespace_version, page_nos[i], NULL, NULL); + tablespace_version, page_nos[i], + NULL); + } + + if (err == DB_DECRYPTION_FAILED) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Recovery failed to decrypt read page " + ULINTPF ":" ULINTPF ".", + space, page_nos[i]); } } diff --git a/storage/xtradb/dict/dict0crea.cc b/storage/xtradb/dict/dict0crea.cc index 1ada35a89a2..6d5b12474eb 100644 --- a/storage/xtradb/dict/dict0crea.cc +++ b/storage/xtradb/dict/dict0crea.cc @@ -692,7 +692,7 @@ dict_create_index_tree_step( dberr_t err = DB_SUCCESS; ulint zip_size = dict_table_zip_size(index->table); - if (node->index->table->ibd_file_missing + if (node->index->table->file_unreadable || dict_table_is_discarded(node->index->table)) { node->page_no = FIL_NULL; diff --git a/storage/xtradb/dict/dict0dict.cc b/storage/xtradb/dict/dict0dict.cc index 49de1cf7ef8..bc46bcab63b 100644 --- a/storage/xtradb/dict/dict0dict.cc +++ b/storage/xtradb/dict/dict0dict.cc @@ -1185,47 +1185,42 @@ dict_table_open_on_name( if (table != NULL) { - /* If table is encrypted return table */ + /* If table is encrypted or corrupted */ if (ignore_err == DICT_ERR_IGNORE_NONE - && table->is_encrypted) { + && !table->is_readable()) { /* Make life easy for drop table. */ if (table->can_be_evicted) { dict_table_move_from_lru_to_non_lru(table); } - if (table->can_be_evicted) { - dict_move_to_mru(table); - } + if (table->corrupted) { - ++table->n_ref_count; + if (!dict_locked) { + mutex_exit(&dict_sys->mutex); + } - if (!dict_locked) { - mutex_exit(&dict_sys->mutex); - } + char buf[MAX_FULL_NAME_LEN]; + ut_format_name(table->name, TRUE, buf, sizeof(buf)); - return (table); - } - /* If table is corrupted, return NULL */ - else if (ignore_err == DICT_ERR_IGNORE_NONE - && table->corrupted) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Table %s is corrupted. Please " + "drop the table and recreate.", + buf); + + return(NULL); + } - /* Make life easy for drop table. */ if (table->can_be_evicted) { - dict_table_move_from_lru_to_non_lru(table); + dict_move_to_mru(table); } + ++table->n_ref_count; + if (!dict_locked) { mutex_exit(&dict_sys->mutex); } - ut_print_timestamp(stderr); - - fprintf(stderr, " InnoDB: table "); - ut_print_name(stderr, NULL, TRUE, table->name); - fprintf(stderr, "is corrupted. Please drop the table " - "and recreate\n"); - - return(NULL); + return (table); } if (table->can_be_evicted) { @@ -6183,11 +6178,29 @@ dict_set_corrupted_by_space( /* mark the table->corrupted bit only, since the caller could be too deep in the stack for SYS_INDEXES update */ - table->corrupted = TRUE; + table->corrupted = true; + table->file_unreadable = true; return(TRUE); } + +/** Flags a table with specified space_id encrypted in the data dictionary +cache +@param[in] space_id Tablespace id */ +UNIV_INTERN +void +dict_set_encrypted_by_space(ulint space_id) +{ + dict_table_t* table; + + table = dict_find_table_by_space(space_id); + + if (table) { + table->file_unreadable = true; + } +} + /**********************************************************************//** Flags an index corrupted both in the data dictionary cache and in the SYS_INDEXES */ @@ -6325,43 +6338,6 @@ dict_set_corrupted_index_cache_only( index->type |= DICT_CORRUPT; } -/************************************************************************* -set is_corrupt flag by space_id*/ - -void -dict_table_set_corrupt_by_space( -/*============================*/ - ulint space_id, - ibool need_mutex) -{ - dict_table_t* table; - ibool found = FALSE; - - ut_a(space_id != 0 && space_id < SRV_LOG_SPACE_FIRST_ID); - - if (need_mutex) - mutex_enter(&(dict_sys->mutex)); - - table = UT_LIST_GET_FIRST(dict_sys->table_LRU); - - while (table) { - if (table->space == space_id) { - table->is_corrupt = TRUE; - found = TRUE; - } - - table = UT_LIST_GET_NEXT(table_LRU, table); - } - - if (need_mutex) - mutex_exit(&(dict_sys->mutex)); - - if (!found) { - fprintf(stderr, "InnoDB: space to be marked as " - "crashed was not found for id " ULINTPF ".\n", - space_id); - } -} #endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** @@ -6683,7 +6659,8 @@ dict_table_schema_check( } } - if (table->ibd_file_missing) { + if (!table->is_readable() && + fil_space_get(table->space) == NULL) { /* missing tablespace */ ut_snprintf(errstr, errstr_sz, diff --git a/storage/xtradb/dict/dict0load.cc b/storage/xtradb/dict/dict0load.cc index 776bb873c5c..83f7bdafb35 100644 --- a/storage/xtradb/dict/dict0load.cc +++ b/storage/xtradb/dict/dict0load.cc @@ -1964,7 +1964,7 @@ dict_load_indexes( dict_mem_index_free(index); goto func_exit; } else if (index->page == FIL_NULL - && !table->ibd_file_missing + && !table->file_unreadable && (!(index->type & DICT_FTS))) { fprintf(stderr, @@ -2193,7 +2193,7 @@ err_len: (*table)->id = mach_read_from_8(field); - (*table)->ibd_file_missing = FALSE; + (*table)->file_unreadable = false; return(NULL); } @@ -2380,7 +2380,7 @@ err_exit: "Table '%s' tablespace is set as discarded.", table_name); - table->ibd_file_missing = TRUE; + table->file_unreadable = true; } else if (!fil_space_for_table_exists_in_mem( table->space, name, false, true, heap, @@ -2388,7 +2388,7 @@ err_exit: if (DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY)) { /* Do not bother to retry opening temporary tables. */ - table->ibd_file_missing = TRUE; + table->file_unreadable = true; } else { if (!(ignore_err & DICT_ERR_IGNORE_RECOVER_LOCK)) { @@ -2423,8 +2423,9 @@ err_exit: /* We failed to find a sensible tablespace file */ - table->ibd_file_missing = TRUE; + table->file_unreadable = true; } + if (filepath) { mem_free(filepath); } @@ -2448,9 +2449,10 @@ err_exit: were not allowed while the table is being locked by a transaction. */ dict_err_ignore_t index_load_err = !(ignore_err & DICT_ERR_IGNORE_RECOVER_LOCK) - && table->ibd_file_missing + && table->file_unreadable ? DICT_ERR_IGNORE_ALL : ignore_err; + err = dict_load_indexes(table, heap, index_load_err); if (err == DB_INDEX_CORRUPT) { @@ -2485,7 +2487,7 @@ err_exit: of the error condition, since the user may want to dump data from the clustered index. However we load the foreign key information only if all indexes were loaded. */ - if (!cached || table->ibd_file_missing) { + if (!cached || table->file_unreadable) { /* Don't attempt to load the indexes from disk. */ } else if (err == DB_SUCCESS) { err = dict_load_foreigns(table->name, NULL, true, true, @@ -2518,12 +2520,12 @@ err_exit: table = NULL; } else if (dict_index_is_corrupted(index) - && !table->ibd_file_missing) { + && !table->file_unreadable) { /* It is possible we force to load a corrupted clustered index if srv_load_corrupted is set. Mark the table as corrupted in this case */ - table->corrupted = TRUE; + table->corrupted = true; } } @@ -2532,7 +2534,7 @@ func_exit: ut_ad(!table || ignore_err != DICT_ERR_IGNORE_NONE - || table->ibd_file_missing + || table->file_unreadable || !table->corrupted); if (table && table->fts) { diff --git a/storage/xtradb/dict/dict0mem.cc b/storage/xtradb/dict/dict0mem.cc index bde72b83c54..fa7177c5137 100644 --- a/storage/xtradb/dict/dict0mem.cc +++ b/storage/xtradb/dict/dict0mem.cc @@ -2,6 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -137,8 +138,6 @@ dict_mem_table_create( table->fts = NULL; } - table->is_corrupt = FALSE; - #endif /* !UNIV_HOTBACKUP */ new(&table->foreign_set) dict_foreign_set(); diff --git a/storage/xtradb/dict/dict0stats.cc b/storage/xtradb/dict/dict0stats.cc index 33b6c2e23e0..d2e9a2618c0 100644 --- a/storage/xtradb/dict/dict0stats.cc +++ b/storage/xtradb/dict/dict0stats.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2009, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -920,7 +921,11 @@ dict_stats_update_transient_for_index( index->stat_n_leaf_pages = size; - btr_estimate_number_of_different_key_vals(index); + /* Do not continue if table decryption has failed or + table is already marked as corrupted. */ + if (index->is_readable()) { + btr_estimate_number_of_different_key_vals(index); + } } } @@ -974,8 +979,9 @@ dict_stats_update_transient( continue; } - /* Do not continue if table decryption has failed. */ - if (index->table->is_encrypted) { + /* Do not continue if table decryption has failed or + table is already marked as corrupted. */ + if (!index->is_readable()) { break; } @@ -2437,6 +2443,61 @@ dict_stats_save_index_stat( return(ret); } +/** Report error if statistic update for a table failed because +.ibd file is missing, table decryption failed or table is corrupted. +@param[in,out] table Table +@param[in] defragment true if statistics is for defragment +@return DB_DECRYPTION_FAILED, DB_TABLESPACE_DELETED or DB_CORRUPTION +@retval DB_DECRYPTION_FAILED if decryption of the table failed +@retval DB_TABLESPACE_DELETED if .ibd file is missing +@retval DB_CORRUPTION if table is marked as corrupted */ +static +dberr_t +dict_stats_report_error( + dict_table_t* table, + bool defragment = false) +{ + char buf[3 * NAME_LEN]; + dberr_t err; + + innobase_format_name(buf, sizeof buf, + table->name, + true); + + FilSpace space(table->space); + + if (space()) { + if (table->corrupted) { + ib_logf(IB_LOG_LEVEL_INFO, + "Cannot save%s statistics because " + " table %s in file %s is corrupted.", + defragment ? " defragment" : " ", + buf, space()->chain.start->name); + err = DB_CORRUPTION; + } else { + ib_logf(IB_LOG_LEVEL_INFO, + "Cannot save%s statistics because " + " table %s in file %s can't be decrypted.", + defragment ? " defragment" : " ", + buf, space()->chain.start->name); + err = DB_DECRYPTION_FAILED; + } + } else { + ib_logf(IB_LOG_LEVEL_INFO, + "Cannot save%s statistics for " + " table %s because .ibd file is missing." + " For help, please " + "refer to " REFMAN "innodb-troubleshooting.html.", + defragment ? " defragment" : " ", + buf); + err = DB_TABLESPACE_DELETED; + } + + dict_stats_empty_table(table, defragment); + + return (err); +} + /** Save the table's statistics into the persistent statistics storage. @param[in] table_orig table whose stats to save @param[in] only_for_index if this is non-NULL, then stats for indexes @@ -2457,6 +2518,11 @@ dict_stats_save( char db_utf8[MAX_DB_UTF8_LEN]; char table_utf8[MAX_TABLE_UTF8_LEN]; + if (table_orig->is_readable()) { + } else { + return (dict_stats_report_error(table_orig)); + } + table = dict_stats_snapshot_create(table_orig); dict_fs2utf8(table->name, db_utf8, sizeof(db_utf8), @@ -3192,15 +3258,8 @@ dict_stats_update( ut_ad(!mutex_own(&dict_sys->mutex)); - if (table->ibd_file_missing) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: cannot calculate statistics for table %s " - "because the .ibd file is missing. For help, please " - "refer to " REFMAN "innodb-troubleshooting.html\n", - ut_format_name(table->name, TRUE, buf, sizeof(buf))); - dict_stats_empty_table(table, true); - return(DB_TABLESPACE_DELETED); + if (!table->is_readable()) { + return (dict_stats_report_error(table)); } else if (srv_force_recovery >= SRV_FORCE_NO_IBUF_MERGE) { /* If we have set a high innodb_force_recovery level, do not calculate statistics, as a badly corrupted index can @@ -3946,19 +4005,10 @@ dict_stats_save_defrag_stats( { dberr_t ret; - if (index->table->ibd_file_missing) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Cannot save defragment stats because " - ".ibd file is missing.\n"); - return (DB_TABLESPACE_DELETED); - } - if (dict_index_is_corrupted(index)) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Cannot save defragment stats because " - "index is corrupted.\n"); - return(DB_CORRUPTION); + + if (index->is_readable()) { + } else { + return (dict_stats_report_error(index->table, true)); } if (dict_index_is_univ(index)) { diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index da8e538022d..72a4e713e60 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -479,11 +479,12 @@ Parse a MLOG_FILE_WRITE_CRYPT_DATA log entry @param[in] block buffer block @return position on log buffer */ UNIV_INTERN -const byte* +byte* fil_parse_write_crypt_data( - const byte* ptr, + byte* ptr, const byte* end_ptr, - const buf_block_t* block) + const buf_block_t* block, + dberr_t* err) { /* check that redo log entry is complete */ uint entry_size = @@ -495,6 +496,8 @@ fil_parse_write_crypt_data( 4 + // size of key_id 1; // fil_encryption_t + *err = DB_SUCCESS; + if (ptr + entry_size > end_ptr) { return NULL; } @@ -541,6 +544,11 @@ fil_parse_write_crypt_data( fil_space_release(space); } + /* Check is used key found from encryption plugin */ + if (crypt_data->should_encrypt() && !crypt_data->is_key_found()) { + *err = DB_DECRYPTION_FAILED; + } + return ptr; } @@ -710,10 +718,12 @@ fil_space_encrypt( } bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size, space); + memcpy(tmp_mem+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, src+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 8); bool different = memcmp(src, tmp_mem, size); if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) { - fprintf(stderr, "ok %d corrupted %d corrupted1 %d err %d different %d\n", ok , corrupted, corrupted1, err, different); + fprintf(stderr, "ok %d corrupted %d corrupted1 %d err %d different %d\n", + ok , corrupted, corrupted1, err, different); fprintf(stderr, "src_frame\n"); buf_page_print(src_frame, zip_size, BUF_PAGE_PRINT_NO_CRASH); fprintf(stderr, "encrypted_frame\n"); @@ -768,25 +778,6 @@ fil_space_decrypt( return false; } - if (crypt_data == NULL) { - if (!(space == 0 && offset == 0) && key_version != 0) { - /* FIL_PAGE_FILE_FLUSH_LSN field i.e. - FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION - should be only defined for the - first page in a system tablespace - data file (ibdata*, not *.ibd), if not - clear it. */ - - DBUG_PRINT("ib_crypt", - ("Page on space %lu offset %lu has key_version %u" - " when it shoud be undefined.", - space, offset, key_version)); - - mach_write_to_4(src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0); - } - return false; - } - ut_a(crypt_data != NULL && crypt_data->is_encrypted()); /* read space & lsn */ @@ -838,9 +829,6 @@ fil_space_decrypt( memcpy(tmp_frame + page_size - FIL_PAGE_DATA_END, src_frame + page_size - FIL_PAGE_DATA_END, FIL_PAGE_DATA_END); - - // clear key-version & crypt-checksum from dst - memset(tmp_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 0, 8); } srv_stats.pages_decrypted.inc(); @@ -962,11 +950,12 @@ fil_space_verify_crypt_checksum( /* If page is not encrypted, return false */ if (key_version == 0) { - return false; + return(false); } srv_checksum_algorithm_t algorithm = static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm); + /* If no checksum is used, can't continue checking. */ if (algorithm == SRV_CHECKSUM_ALGORITHM_NONE) { return(true); @@ -990,28 +979,31 @@ fil_space_verify_crypt_checksum( return (true); } - /* Compressed pages use different checksum method. We first store - the post encryption checksum on checksum location and after function - restore the original. */ - if (zip_size) { - ib_uint32_t old = static_cast<ib_uint32_t>(mach_read_from_4( - page + FIL_PAGE_SPACE_OR_CHKSUM)); - - mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum); + ib_uint32_t cchecksum1 = 0; + ib_uint32_t cchecksum2 = 0; - bool valid = page_zip_verify_checksum(page, zip_size); + /* Calculate checksums */ + if (zip_size) { + cchecksum1 = page_zip_calc_checksum( + page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32); - mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, old); + if(cchecksum1 != checksum) { + cchecksum2 = page_zip_calc_checksum( + page, zip_size, + SRV_CHECKSUM_ALGORITHM_INNODB); + } + } else { + cchecksum1 = buf_calc_page_crc32(page); - return (valid); + if (cchecksum1 != checksum) { + cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum( + page); + } } /* If stored checksum matches one of the calculated checksums page is not corrupted. */ - ib_uint32_t cchecksum1 = buf_calc_page_crc32(page); - ib_uint32_t cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum( - page); bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2 || checksum == BUF_NO_CHECKSUM_MAGIC); @@ -1042,13 +1034,19 @@ fil_space_verify_crypt_checksum( ulint checksum1 = mach_read_from_4( page + FIL_PAGE_SPACE_OR_CHKSUM); - ulint checksum2 = mach_read_from_4( - page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); + ulint checksum2 = checksum1; + bool valid; - bool valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2) + if (zip_size) { + valid = (checksum1 == cchecksum1); + } else { + checksum1 = mach_read_from_4( + page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); + valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2) || buf_page_is_checksum_valid_none(page,checksum1,checksum2) || buf_page_is_checksum_valid_innodb(page,checksum1, checksum2)); + } if (encrypted && valid) { /* If page is encrypted and traditional checksums match, @@ -1964,11 +1962,11 @@ fil_crypt_rotate_page( bool modified = false; int needs_scrubbing = BTR_SCRUB_SKIP_PAGE; lsn_t block_lsn = block->page.newest_modification; - uint kv = block->page.key_version; + byte* frame = buf_block_get_frame(block); + uint kv = mach_read_from_4(frame+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); /* check if tablespace is closing after reading page */ - if (space->is_stopping()) { - byte* frame = buf_block_get_frame(block); + if (!space->is_stopping()) { if (kv == 0 && fil_crypt_is_page_uninitialized(frame, zip_size)) { @@ -1990,9 +1988,6 @@ fil_crypt_rotate_page( FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id, MLOG_4BYTES, &mtr); - /* update block */ - block->page.key_version = key_state->key_version; - /* statistics */ state->crypt_stat.pages_modified++; } else { @@ -2083,6 +2078,11 @@ fil_crypt_rotate_page( state->end_lsn = block_lsn; } } + } else { + /* If block read failed mtr memo and log should be empty. */ + ut_ad(dyn_array_get_data_size(&mtr.memo) == 0); + ut_ad(dyn_array_get_data_size(&mtr.log) == 0); + mtr_commit(&mtr); } if (sleeptime_ms) { diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index e504ab3947e..7ac08cc0e97 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -330,7 +330,11 @@ fil_write( } /*******************************************************************//** -Returns the table space by a given id, NULL if not found. */ +Returns the table space by a given id, NULL if not found. +It is unsafe to dereference the returned pointer. It is fine to check +for NULL. +@param[in] id Tablespace id +@return table space or NULL */ fil_space_t* fil_space_get_by_id( /*================*/ @@ -350,26 +354,6 @@ fil_space_get_by_id( return(space); } -/*******************************************************************//** -Returns the table space by a given id, NULL if not found. */ -fil_space_t* -fil_space_found_by_id( -/*==================*/ - ulint id) /*!< in: space id */ -{ - fil_space_t* space = NULL; - mutex_enter(&fil_system->mutex); - space = fil_space_get_by_id(id); - - /* Not found if space is being deleted */ - if (space && space->stop_new_ops) { - space = NULL; - } - - mutex_exit(&fil_system->mutex); - return space; -} - /****************************************************************//** Get space id from fil node */ ulint @@ -2516,74 +2500,6 @@ fil_read_first_page( /*================ SINGLE-TABLE TABLESPACES ==========================*/ -#ifndef UNIV_HOTBACKUP -/*******************************************************************//** -Increments the count of pending operation, if space is not being deleted. -@return TRUE if being deleted, and operation should be skipped */ -UNIV_INTERN -ibool -fil_inc_pending_ops( -/*================*/ - ulint id, /*!< in: space id */ - ibool print_err) /*!< in: need to print error or not */ -{ - fil_space_t* space; - - mutex_enter(&fil_system->mutex); - - space = fil_space_get_by_id(id); - - if (space == NULL) { - if (print_err) { - fprintf(stderr, - "InnoDB: Error: trying to do an operation on a" - " dropped tablespace %lu\n", - (ulong) id); - } - } - - if (space == NULL || space->stop_new_ops) { - mutex_exit(&fil_system->mutex); - - return(TRUE); - } - - space->n_pending_ops++; - - mutex_exit(&fil_system->mutex); - - return(FALSE); -} - -/*******************************************************************//** -Decrements the count of pending operations. */ -UNIV_INTERN -void -fil_decr_pending_ops( -/*=================*/ - ulint id) /*!< in: space id */ -{ - fil_space_t* space; - - mutex_enter(&fil_system->mutex); - - space = fil_space_get_by_id(id); - - if (space == NULL) { - fprintf(stderr, - "InnoDB: Error: decrementing pending operation" - " of a dropped tablespace %lu\n", - (ulong) id); - } - - if (space != NULL) { - space->n_pending_ops--; - } - - mutex_exit(&fil_system->mutex); -} -#endif /* !UNIV_HOTBACKUP */ - /********************************************************//** Creates the database directory for a table if it does not exist yet. */ static @@ -2975,7 +2891,7 @@ fil_check_pending_operations( fil_space_t* sp = fil_space_get_by_id(id); if (sp) { - sp->stop_new_ops = TRUE; + sp->stop_new_ops = true; /* space could be freed by other threads as soon as n_pending_ops reaches 0, thus increment pending ops here. */ @@ -5390,7 +5306,7 @@ fil_tablespace_deleted_or_being_deleted_in_mem( space = fil_space_get_by_id(id); - if (space == NULL || space->stop_new_ops) { + if (space == NULL || space->is_stopping()) { mutex_exit(&fil_system->mutex); return(TRUE); @@ -6143,15 +6059,19 @@ _fil_io( /* If we are deleting a tablespace we don't allow async read operations on that. However, we do allow write and sync read operations */ if (space == 0 - || (type == OS_FILE_READ && !sync && space->stop_new_ops)) { + || (type == OS_FILE_READ + && !sync + && space->stop_new_ops)) { mutex_exit(&fil_system->mutex); ib_logf(IB_LOG_LEVEL_ERROR, "Trying to do i/o to a tablespace which does " - "not exist. i/o type %lu, space id %lu, " - "page no. %lu, i/o length %lu bytes", - (ulong) type, (ulong) space_id, (ulong) block_offset, - (ulong) len); + "not exist. i/o type " ULINTPF + ", space id " ULINTPF " , " + "page no. " ULINTPF + ", i/o length " ULINTPF " bytes", + type, space_id, block_offset, + len); return(DB_TABLESPACE_DELETED); } @@ -6255,8 +6175,15 @@ _fil_io( mutex_exit(&fil_system->mutex); if (mode == OS_AIO_NORMAL) { ut_a(space->purpose == FIL_TABLESPACE); - buf_page_io_complete(static_cast<buf_page_t *> - (message)); + dberr_t err = buf_page_io_complete(static_cast<buf_page_t *> + (message)); + + if (err != DB_SUCCESS) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Write operation failed for tablespace %s (" + ULINTPF ") offset " ULINTPF " error=%d.", + space->name, space->id, byte_offset, err); + } } } @@ -6359,6 +6286,8 @@ fil_aio_wait( mutex_enter(&fil_system->mutex); fil_node_complete_io(fil_node, fil_system, type); + ulint purpose = fil_node->space->purpose; + space_id = fil_node->space->id; mutex_exit(&fil_system->mutex); @@ -6370,9 +6299,27 @@ fil_aio_wait( deadlocks in the i/o system. We keep tablespace 0 data files always open, and use a special i/o thread to serve insert buffer requests. */ - if (fil_node->space->purpose == FIL_TABLESPACE) { + if (purpose == FIL_TABLESPACE) { srv_set_io_thread_op_info(segment, "complete io for buf page"); - buf_page_io_complete(static_cast<buf_page_t*>(message)); + buf_page_t* bpage = static_cast<buf_page_t*>(message); + ulint offset = bpage->offset; + dberr_t err = buf_page_io_complete(bpage); + + if (err != DB_SUCCESS) { + ut_ad(type == OS_FILE_READ); + /* In crash recovery set log corruption on + and produce only an error to fail InnoDB startup. */ + if (recv_recovery_is_on() && !srv_force_recovery) { + recv_sys->found_corrupt_log = true; + } + + ib_logf(IB_LOG_LEVEL_ERROR, + "Read operation failed for tablespace %s" + " offset " ULINTPF " with error %s", + fil_node->name, + offset, + ut_strerr(err)); + } } else { srv_set_io_thread_op_info(segment, "complete io for log"); log_io_complete(static_cast<log_group_t*>(message)); @@ -6393,7 +6340,8 @@ fil_flush( mutex_enter(&fil_system->mutex); if (fil_space_t* space = fil_space_get_by_id(space_id)) { - if (!space->is_stopping()) { + if (!space->stop_new_ops) { + fil_flush_low(space); } } @@ -7331,76 +7279,6 @@ fil_space_set_corrupt( mutex_exit(&fil_system->mutex); } -/****************************************************************** -Get id of first tablespace or ULINT_UNDEFINED if none */ -UNIV_INTERN -ulint -fil_get_first_space() -/*=================*/ -{ - ulint out_id = ULINT_UNDEFINED; - fil_space_t* space; - - mutex_enter(&fil_system->mutex); - - space = UT_LIST_GET_FIRST(fil_system->space_list); - if (space != NULL) { - do - { - if (!space->stop_new_ops) { - out_id = space->id; - break; - } - space = UT_LIST_GET_NEXT(space_list, space); - } while (space != NULL); - } - - mutex_exit(&fil_system->mutex); - - return out_id; -} - -/****************************************************************** -Get id of next tablespace or ULINT_UNDEFINED if none */ -UNIV_INTERN -ulint -fil_get_next_space( -/*===============*/ - ulint id) /*!< in: previous space id */ -{ - bool found; - fil_space_t* space; - ulint out_id = ULINT_UNDEFINED; - - mutex_enter(&fil_system->mutex); - - space = fil_space_get_by_id(id); - if (space == NULL) { - /* we didn't find it...search for space with space->id > id */ - found = false; - space = UT_LIST_GET_FIRST(fil_system->space_list); - } else { - /* we found it, take next available space */ - found = true; - } - - while ((space = UT_LIST_GET_NEXT(space_list, space)) != NULL) { - - if (!found && space->id <= id) - continue; - - if (!space->stop_new_ops && UT_LIST_GET_LEN(space->chain) > 0) { - /* inc reference to prevent drop */ - out_id = space->id; - break; - } - } - - mutex_exit(&fil_system->mutex); - - return out_id; -} - /** Acquire a tablespace when it could be dropped concurrently. Used by background threads that do not necessarily hold proper locks for concurrency control. @@ -7410,9 +7288,9 @@ for concurrency control. (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted or truncated */ -inline +UNIV_INTERN fil_space_t* -fil_space_acquire_low(ulint id, bool silent, bool for_io = false) +fil_space_acquire_low(ulint id, bool silent, bool for_io) { fil_space_t* space; @@ -7436,32 +7314,6 @@ fil_space_acquire_low(ulint id, bool silent, bool for_io = false) return(space); } -/** Acquire a tablespace when it could be dropped concurrently. -Used by background threads that do not necessarily hold proper locks -for concurrency control. -@param[in] id tablespace ID -@param[in] for_io whether to look up the tablespace while performing I/O - (possibly executing TRUNCATE) -@return the tablespace -@retval NULL if missing or being deleted or truncated */ -fil_space_t* -fil_space_acquire(ulint id, bool for_io) -{ - return(fil_space_acquire_low(id, false, for_io)); -} - -/** Acquire a tablespace that may not exist. -Used by background threads that do not necessarily hold proper locks -for concurrency control. -@param[in] id tablespace ID -@return the tablespace -@retval NULL if missing or being deleted */ -fil_space_t* -fil_space_acquire_silent(ulint id) -{ - return(fil_space_acquire_low(id, true)); -} - /** Release a tablespace acquired with fil_space_acquire(). @param[in,out] space tablespace to release */ void @@ -7590,7 +7442,7 @@ fil_space_keyrotate_next( space->purpose == FIL_TABLESPACE. */ while (space != NULL && (UT_LIST_GET_LEN(space->chain) == 0 - || space->stop_new_ops)) { + || space->is_stopping())) { old = space; space = UT_LIST_GET_NEXT(rotation_list, space); diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index d2cd1951bd7..433ba46956f 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -32,7 +32,7 @@ this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA *****************************************************************************/ - + #define MYSQL_SERVER #include <sql_table.h> // explain_filename, nz2, EXPLAIN_PARTITIONS_AS_COMMENT, @@ -6228,20 +6228,23 @@ ha_innobase::innobase_initialize_autoinc() break; } case DB_RECORD_NOT_FOUND: - ut_print_timestamp(stderr); - fprintf(stderr, " InnoDB: MySQL and InnoDB data " - "dictionaries are out of sync.\n" - "InnoDB: Unable to find the AUTOINC column " - "%s in the InnoDB table %s.\n" - "InnoDB: We set the next AUTOINC column " - "value to 0,\n" - "InnoDB: in effect disabling the AUTOINC " - "next value generation.\n" - "InnoDB: You can either set the next " - "AUTOINC value explicitly using ALTER TABLE\n" - "InnoDB: or fix the data dictionary by " - "recreating the table.\n", - col_name, index->table->name); + char buf[MAX_FULL_NAME_LEN]; + ut_format_name(index->table->name, TRUE, buf, sizeof(buf)); + + ib_logf(IB_LOG_LEVEL_ERROR, + "MySQL and InnoDB data " + "dictionaries are out of sync." + " Unable to find the AUTOINC column " + " %s in the InnoDB table %s." + " We set the next AUTOINC column " + "value to 0" + " in effect disabling the AUTOINC " + "next value generation." + " You can either set the next " + "AUTOINC value explicitly using ALTER TABLE " + " or fix the data dictionary by " + "recreating the table.", + col_name, buf); /* This will disable the AUTOINC generation. */ auto_inc = 0; @@ -6300,7 +6303,7 @@ ha_innobase::open( DBUG_RETURN(1); } - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt && + if (UNIV_UNLIKELY(share->ib_table && share->ib_table->corrupted && srv_pass_corrupt_table <= 1)) { free_share(share); @@ -6336,23 +6339,24 @@ ha_innobase::open( && (table->s->fields != dict_table_get_n_user_cols(ib_table) - 1)))) { ib_logf(IB_LOG_LEVEL_WARN, - "table %s contains %lu user defined columns " - "in InnoDB, but %lu columns in MySQL. Please " + "table %s contains " ULINTPF " user defined columns " + "in InnoDB, but %u columns in MySQL. Please " "check INFORMATION_SCHEMA.INNODB_SYS_COLUMNS and " REFMAN "innodb-troubleshooting.html " "for how to resolve it", - norm_name, (ulong) dict_table_get_n_user_cols(ib_table), - (ulong) table->s->fields); + norm_name, dict_table_get_n_user_cols(ib_table), + table->s->fields); /* Mark this table as corrupted, so the drop table or force recovery can still use it, but not others. */ ib_table->corrupted = true; + ib_table->file_unreadable = true; dict_table_close(ib_table, FALSE, FALSE); ib_table = NULL; is_part = NULL; } - if (UNIV_UNLIKELY(ib_table && ib_table->is_corrupt && + if (UNIV_UNLIKELY(ib_table && ib_table->corrupted && srv_pass_corrupt_table <= 1)) { free_share(share); my_free(upd_buf); @@ -6462,7 +6466,7 @@ table_opened: ib_table->thd = (void*)thd; /* No point to init any statistics if tablespace is still encrypted. */ - if (!ib_table->is_encrypted) { + if (ib_table->is_readable()) { dict_stats_init(ib_table); } else { ib_table->stat_initialized = 1; @@ -6470,7 +6474,9 @@ table_opened: MONITOR_INC(MONITOR_TABLE_OPEN); - bool no_tablespace; + bool no_tablespace = false; + bool encrypted = false; + FilSpace space; if (dict_table_is_discarded(ib_table)) { @@ -6485,23 +6491,28 @@ table_opened: no_tablespace = false; - } else if (ib_table->ibd_file_missing) { + } else if (!ib_table->is_readable()) { + space = fil_space_acquire_silent(ib_table->space); - ib_senderrf( - thd, IB_LOG_LEVEL_WARN, - ER_TABLESPACE_MISSING, norm_name); - - /* This means we have no idea what happened to the tablespace - file, best to play it safe. */ - - no_tablespace = true; - } else if (ib_table->is_encrypted) { - /* This means that tablespace was found but we could not - decrypt encrypted page. */ - no_tablespace = true; - ib_table->ibd_file_missing = true; - } else { - no_tablespace = false; + if (space()) { + if (space()->crypt_data && space()->crypt_data->is_encrypted()) { + /* This means that tablespace was found but we could not + decrypt encrypted page. */ + no_tablespace = true; + encrypted = true; + } else { + no_tablespace = true; + } + } else { + ib_senderrf( + thd, IB_LOG_LEVEL_WARN, + ER_TABLESPACE_MISSING, norm_name); + + /* This means we have no idea what happened to the tablespace + file, best to play it safe. */ + + no_tablespace = true; + } } if (!thd_tablespace_op(thd) && no_tablespace) { @@ -6512,27 +6523,33 @@ table_opened: /* If table has no talespace but it has crypt data, check is tablespace made unaccessible because encryption service or used key_id is not available. */ - if (ib_table) { - fil_space_crypt_t* crypt_data = ib_table->crypt_data; + if (encrypted) { + bool warning_pushed = false; + char buf[MAX_FULL_NAME_LEN]; + ut_format_name(ib_table->name, TRUE, buf, sizeof(buf)); - if (crypt_data && crypt_data->should_encrypt()) { + if (!encryption_key_id_exists(space()->crypt_data->key_id)) { + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + HA_ERR_DECRYPTION_FAILED, + "Table %s in file %s is encrypted but encryption service or" + " used key_id %u is not available. " + " Can't continue reading table.", + buf, space()->chain.start->name, + space()->crypt_data->key_id); + ret_err = HA_ERR_DECRYPTION_FAILED; + warning_pushed = true; + } - if (!encryption_key_id_exists(crypt_data->key_id)) { - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - HA_ERR_DECRYPTION_FAILED, - "Table %s is encrypted but encryption service or" - " used key_id %u is not available. " - " Can't continue reading table.", - ib_table->name, crypt_data->key_id); - ret_err = HA_ERR_DECRYPTION_FAILED; - } - } else if (ib_table->is_encrypted) { + /* If table is marked as encrypted then we push + warning if it has not been already done as used + key_id might be found but it is incorrect. */ + if (!warning_pushed) { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, HA_ERR_DECRYPTION_FAILED, - "Table %s is encrypted but encryption service or" + "Table %s in file %s is encrypted but encryption service or" " used key_id is not available. " " Can't continue reading table.", - ib_table->name); + buf, space()->chain.start->name); ret_err = HA_ERR_DECRYPTION_FAILED; } } @@ -6668,7 +6685,7 @@ table_opened: /* Only if the table has an AUTOINC column. */ if (prebuilt->table != NULL - && !prebuilt->table->ibd_file_missing + && prebuilt->table->is_readable() && table->found_next_number_field != NULL) { dict_table_autoinc_lock(prebuilt->table); @@ -8653,16 +8670,6 @@ ha_innobase::write_row( ha_statistic_increment(&SSV::ha_write_count); - if (share->ib_table != prebuilt->table) { - fprintf(stderr, - "InnoDB: Warning: share->ib_table %p prebuilt->table %p table %s is_corrupt %lu.", - share->ib_table, prebuilt->table, prebuilt->table->name, prebuilt->table->is_corrupt); - } - - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - sql_command = thd_sql_command(user_thd); if ((sql_command == SQLCOM_ALTER_TABLE @@ -9000,16 +9007,6 @@ wsrep_error: func_exit: innobase_active_small(); - if (share->ib_table != prebuilt->table) { - fprintf(stderr, - "InnoDB: Warning: share->ib_table %p prebuilt->table %p table %s is_corrupt %lu.", - share->ib_table, prebuilt->table, prebuilt->table->name, prebuilt->table->is_corrupt); - } - - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - DBUG_RETURN(error_result); } @@ -9420,16 +9417,6 @@ ha_innobase::update_row( ha_statistic_increment(&SSV::ha_update_count); - if (share->ib_table != prebuilt->table) { - fprintf(stderr, - "InnoDB: Warning: share->ib_table %p prebuilt->table %p table %s is_corrupt %lu.", - share->ib_table, prebuilt->table, prebuilt->table->name, prebuilt->table->is_corrupt); - } - - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - if (prebuilt->upd_node) { uvect = prebuilt->upd_node->update; } else { @@ -9535,16 +9522,6 @@ func_exit: wsrep_error: #endif /* WITH_WSREP */ - if (share->ib_table != prebuilt->table) { - fprintf(stderr, - "InnoDB: Warning: share->ib_table %p prebuilt->table %p table %s is_corrupt %lu.", - share->ib_table, prebuilt->table, prebuilt->table->name, prebuilt->table->is_corrupt); - } - - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - DBUG_RETURN(err); } @@ -9573,11 +9550,6 @@ ha_innobase::delete_row( ha_statistic_increment(&SSV::ha_delete_count); - if (UNIV_UNLIKELY(share && share->ib_table - && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - if (!prebuilt->upd_node) { row_get_prebuilt_update_vector(prebuilt); } @@ -9612,11 +9584,6 @@ ha_innobase::delete_row( wsrep_error: #endif /* WITH_WSREP */ - if (UNIV_UNLIKELY(share && share->ib_table - && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - DBUG_RETURN(convert_error_code_to_mysql( error, prebuilt->table->flags, user_thd)); } @@ -9867,17 +9834,13 @@ ha_innobase::index_read( ha_statistic_increment(&SSV::ha_read_key_count); - if (UNIV_UNLIKELY(srv_pass_corrupt_table <= 1 && share - && share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - index = prebuilt->index; if (UNIV_UNLIKELY(index == NULL) || dict_index_is_corrupted(index)) { prebuilt->index_usable = FALSE; DBUG_RETURN(HA_ERR_CRASHED); } + if (UNIV_UNLIKELY(!prebuilt->index_usable)) { DBUG_RETURN(dict_index_is_corrupted(index) ? HA_ERR_INDEX_CORRUPT @@ -9944,11 +9907,6 @@ ha_innobase::index_read( ret = DB_UNSUPPORTED; } - if (UNIV_UNLIKELY(srv_pass_corrupt_table <= 1 && share - && share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - switch (ret) { case DB_SUCCESS: error = 0; @@ -10108,11 +10066,6 @@ ha_innobase::change_active_index( { DBUG_ENTER("change_active_index"); - if (UNIV_UNLIKELY(srv_pass_corrupt_table <= 1 && share - && share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - ut_ad(user_thd == ha_thd()); ut_a(prebuilt->trx == thd_to_trx(user_thd)); @@ -10232,13 +10185,23 @@ ha_innobase::general_fetch( DBUG_RETURN(HA_ERR_END_OF_FILE); } - if (UNIV_UNLIKELY(srv_pass_corrupt_table <= 1 && share - && share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - ut_a(prebuilt->trx == thd_to_trx(user_thd)); + if (prebuilt->table->is_readable()) { + } else { + if (prebuilt->table->corrupted) { + DBUG_RETURN(HA_ERR_CRASHED); + } else { + FilSpace space(prebuilt->table->space, true); + + if (space()) { + DBUG_RETURN(HA_ERR_DECRYPTION_FAILED); + } else { + DBUG_RETURN(HA_ERR_NO_SUCH_TABLE); + } + } + } + innobase_srv_conc_enter_innodb(prebuilt->trx); ret = row_search_for_mysql( @@ -10246,11 +10209,6 @@ ha_innobase::general_fetch( innobase_srv_conc_exit_innodb(prebuilt->trx); - if (UNIV_UNLIKELY(srv_pass_corrupt_table <= 1 && share - && share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - switch (ret) { case DB_SUCCESS: error = 0; @@ -12979,7 +12937,7 @@ ha_innobase::discard_or_import_tablespace( user may want to set the DISCARD flag in order to IMPORT a new tablespace. */ - if (dict_table->ibd_file_missing) { + if (!dict_table->is_readable()) { ib_senderrf( prebuilt->trx->mysql_thd, IB_LOG_LEVEL_WARN, ER_TABLESPACE_MISSING, @@ -12989,7 +12947,7 @@ ha_innobase::discard_or_import_tablespace( err = row_discard_tablespace_for_mysql( dict_table->name, prebuilt->trx); - } else if (!dict_table->ibd_file_missing) { + } else if (dict_table->is_readable()) { /* Commit the transaction in order to release the table lock. */ trx_commit_for_mysql(prebuilt->trx); @@ -13066,15 +13024,7 @@ ha_innobase::truncate() update_thd(ha_thd()); - if (share->ib_table != prebuilt->table) { - fprintf(stderr, - "InnoDB: Warning: share->ib_table %p prebuilt->table %p table %s is_corrupt %lu.", - share->ib_table, prebuilt->table, prebuilt->table->name, prebuilt->table->is_corrupt); - } - - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } + DBUG_ASSERT(share->ib_table == prebuilt->table); if (UNIV_UNLIKELY(prebuilt->trx->fake_changes)) { DBUG_RETURN(HA_ERR_WRONG_COMMAND); @@ -13087,16 +13037,6 @@ ha_innobase::truncate() err = row_truncate_table_for_mysql(prebuilt->table, prebuilt->trx); - if (share->ib_table != prebuilt->table) { - fprintf(stderr, - "InnoDB: Warning: share->ib_table %p prebuilt->table %p table %s is_corrupt %lu.", - share->ib_table, prebuilt->table, prebuilt->table->name, prebuilt->table->is_corrupt); - } - - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } - switch (err) { case DB_TABLESPACE_DELETED: @@ -14497,15 +14437,7 @@ ha_innobase::analyze( { int ret; - if (share->ib_table != prebuilt->table) { - fprintf(stderr, - "InnoDB: Warning: share->ib_table %p prebuilt->table %p table %s is_corrupt %lu.", - share->ib_table, prebuilt->table, prebuilt->table->name, prebuilt->table->is_corrupt); - } - - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt)) { - return(HA_ADMIN_CORRUPT); - } + DBUG_ASSERT(share->ib_table == prebuilt->table); /* Simply call this->info_low() with all the flags and request recalculation of the statistics */ @@ -14513,15 +14445,7 @@ ha_innobase::analyze( HA_STATUS_TIME | HA_STATUS_CONST | HA_STATUS_VARIABLE, true /* this is ANALYZE */); - if (share->ib_table != prebuilt->table) { - fprintf(stderr, - "InnoDB: Warning: share->ib_table %p prebuilt->table %p table %s is_corrupt %lu.", - share->ib_table, prebuilt->table, prebuilt->table->name, prebuilt->table->is_corrupt); - } - - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt)) { - return(HA_ADMIN_CORRUPT); - } + DBUG_ASSERT(share->ib_table == prebuilt->table); if (ret != 0) { return(HA_ADMIN_FAILED); @@ -14625,7 +14549,8 @@ ha_innobase::check( DBUG_RETURN(HA_ADMIN_CORRUPT); - } else if (prebuilt->table->ibd_file_missing) { + } else if (!prebuilt->table->is_readable() && + fil_space_get(prebuilt->table->space) == NULL) { ib_senderrf( thd, IB_LOG_LEVEL_ERROR, @@ -14699,6 +14624,7 @@ ha_innobase::check( server_mutex, srv_fatal_semaphore_wait_threshold, SRV_SEMAPHORE_WAIT_EXTENSION); + dberr_t err = btr_validate_index(index, prebuilt->trx); /* Restore the fatal lock wait timeout after @@ -14797,6 +14723,7 @@ ha_innobase::check( if (!(index->type & DICT_CLUSTERED)) { check_result = false; }); + if (!check_result) { innobase_format_name( index_name, sizeof index_name, @@ -15826,15 +15753,7 @@ ha_innobase::transactional_table_lock( update_thd(thd); - if (share->ib_table != prebuilt->table) { - fprintf(stderr, - "InnoDB: Warning: share->ib_table %p prebuilt->table %p table %s is_corrupt %lu.", - share->ib_table, prebuilt->table, prebuilt->table->name, prebuilt->table->is_corrupt); - } - - if (UNIV_UNLIKELY(share->ib_table && share->ib_table->is_corrupt)) { - DBUG_RETURN(HA_ERR_CRASHED); - } + DBUG_ASSERT(share->ib_table == prebuilt->table); if (!thd_tablespace_op(thd)) { @@ -15845,7 +15764,7 @@ ha_innobase::transactional_table_lock( ER_TABLESPACE_DISCARDED, table->s->table_name.str); - } else if (prebuilt->table->ibd_file_missing) { + } else if (!prebuilt->table->is_readable()) { ib_senderrf( thd, IB_LOG_LEVEL_ERROR, diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc index b3a684deb91..67807dafa6b 100644 --- a/storage/xtradb/handler/handler0alter.cc +++ b/storage/xtradb/handler/handler0alter.cc @@ -3137,7 +3137,7 @@ prepare_inplace_alter_table_dict( clustered index of the old table, later. */ if (new_clustered || !ctx->online - || user_table->ibd_file_missing + || !user_table->is_readable() || dict_table_is_discarded(user_table)) { /* No need to allocate a modification log. */ ut_ad(!ctx->add_index[a]->online_log); @@ -3557,6 +3557,43 @@ ha_innobase::prepare_inplace_alter_table( goto func_exit; } + indexed_table = prebuilt->table; + + if (indexed_table->is_readable()) { + } else { + if (indexed_table->corrupted) { + /* Handled below */ + } else { + FilSpace space(indexed_table->space, true); + + if (space()) { + String str; + const char* engine= table_type(); + char buf[MAX_FULL_NAME_LEN]; + ut_format_name(indexed_table->name, TRUE, buf, sizeof(buf)); + + push_warning_printf(user_thd, Sql_condition::WARN_LEVEL_WARN, + HA_ERR_DECRYPTION_FAILED, + "Table %s in file %s is encrypted but encryption service or" + " used key_id is not available. " + " Can't continue reading table.", + buf, space()->chain.start->name); + + my_error(ER_GET_ERRMSG, MYF(0), HA_ERR_DECRYPTION_FAILED, str.c_ptr(), engine); + DBUG_RETURN(true); + } + } + } + + if (indexed_table->corrupted + || dict_table_get_first_index(indexed_table) == NULL + || dict_index_is_corrupted( + dict_table_get_first_index(indexed_table))) { + /* The clustered index is corrupted. */ + my_error(ER_CHECK_NO_SUCH_TABLE, MYF(0)); + DBUG_RETURN(true); + } + if (ha_alter_info->handler_flags & Alter_inplace_info::CHANGE_CREATE_OPTION) { /* Check engine specific table options */ @@ -4209,7 +4246,7 @@ ok_exit: DBUG_ASSERT(ctx->trx); DBUG_ASSERT(ctx->prebuilt == prebuilt); - if (prebuilt->table->ibd_file_missing + if (prebuilt->table->file_unreadable || dict_table_is_discarded(prebuilt->table)) { goto all_done; } @@ -5231,7 +5268,7 @@ commit_try_rebuild( /* The new table must inherit the flag from the "parent" table. */ if (dict_table_is_discarded(user_table)) { - rebuilt_table->ibd_file_missing = true; + rebuilt_table->file_unreadable = true; rebuilt_table->flags2 |= DICT_TF2_DISCARDED; } @@ -5763,9 +5800,9 @@ ha_innobase::commit_inplace_alter_table( if (ha_alter_info->group_commit_ctx) { ctx_array = ha_alter_info->group_commit_ctx; } else { - ctx_single[0] = ctx0; - ctx_single[1] = NULL; - ctx_array = ctx_single; + ctx_single[0] = ctx0; + ctx_single[1] = NULL; + ctx_array = ctx_single; } DBUG_ASSERT(ctx0 == ctx_array[0]); @@ -5794,6 +5831,19 @@ ha_innobase::commit_inplace_alter_table( = static_cast<ha_innobase_inplace_ctx*>(*pctx); DBUG_ASSERT(ctx->prebuilt->trx == prebuilt->trx); + /* If decryption failed for old table or new table + fail here. */ + if ((ctx->old_table->file_unreadable && + fil_space_get(ctx->old_table->space) != NULL)|| + (ctx->new_table->file_unreadable && + fil_space_get(ctx->new_table->space) != NULL)) { + String str; + const char* engine= table_type(); + get_error_message(HA_ERR_DECRYPTION_FAILED, &str); + my_error(ER_GET_ERRMSG, MYF(0), HA_ERR_DECRYPTION_FAILED, str.c_ptr(), engine); + DBUG_RETURN(true); + } + /* Exclusively lock the table, to ensure that no other transaction is holding locks on the table while we change the table definition. The MySQL meta-data lock diff --git a/storage/xtradb/include/btr0btr.ic b/storage/xtradb/include/btr0btr.ic index 62a24873482..0f5f025d6a3 100644 --- a/storage/xtradb/include/btr0btr.ic +++ b/storage/xtradb/include/btr0btr.ic @@ -61,7 +61,7 @@ btr_block_get_func( if (err == DB_DECRYPTION_FAILED) { if (index && index->table) { - index->table->is_encrypted = true; + index->table->file_unreadable = true; } } diff --git a/storage/xtradb/include/buf0buf.h b/storage/xtradb/include/buf0buf.h index 4a632e2345f..08c3a765a8b 100644 --- a/storage/xtradb/include/buf0buf.h +++ b/storage/xtradb/include/buf0buf.h @@ -690,17 +690,6 @@ buf_page_is_corrupted( const fil_space_t* space) MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** -Check if page is maybe compressed, encrypted or both when we encounter -corrupted page. Note that we can't be 100% sure if page is corrupted -or decrypt/decompress just failed. -@param[in] bpage Page -@return true if page corrupted, false if not */ -bool -buf_page_check_corrupt( - buf_page_t* bpage) /*!< in/out: buffer page read from disk */ - MY_ATTRIBUTE(( warn_unused_result)); - -/********************************************************************//** Checks if a page is all zeroes. @return TRUE if the page is all zeroes */ bool @@ -1273,12 +1262,15 @@ buf_page_init_for_read( /********************************************************************//** Completes an asynchronous read or write request of a file page to or from the buffer pool. -@return true if successful */ +@param[in,out] bpage pointer to the block in question +@return DB_SUCCESS if page has been read and is not corrupted, +DB_PAGE_CORRUPTED if page based on checksum check is corrupted, +DB_DECRYPTION_FAILED if page post encryption checksum matches but +after decryption normal page checksum does not match.*/ UNIV_INTERN -bool +dberr_t buf_page_io_complete( -/*=================*/ - buf_page_t* bpage); /*!< in: pointer to the block in question */ + buf_page_t* bpage); /********************************************************************//** Calculates a folded value of a file page address to use in the page hash table. @@ -1680,7 +1672,6 @@ struct buf_page_t{ if written again we check is TRIM operation needed. */ - unsigned key_version; /*!< key version for this block */ bool encrypted; /*!< page is still encrypted */ ulint real_size; /*!< Real size of the page diff --git a/storage/xtradb/include/buf0rea.h b/storage/xtradb/include/buf0rea.h index f0652b5d2cd..ab73108a71e 100644 --- a/storage/xtradb/include/buf0rea.h +++ b/storage/xtradb/include/buf0rea.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -35,29 +35,37 @@ High-level function which reads a page asynchronously from a file to the buffer buf_pool if it is not already there. Sets the io_fix flag and sets an exclusive lock on the buffer frame. The flag is cleared and the x-lock released by the i/o-handler thread. -@return TRUE if page has been read in, FALSE in case of failure */ + +@param[in] space space_id +@param[in] zip_size compressed page size in bytes, or 0 +@param[in] offset page number +@param[in] trx transaction +@return DB_SUCCESS if page has been read and is not corrupted, +@retval DB_PAGE_CORRUPTED if page based on checksum check is corrupted, +@retval DB_DECRYPTION_FAILED if page post encryption checksum matches but +after decryption normal page checksum does not match. +@retval DB_TABLESPACE_DELETED if tablespace .ibd file is missing */ UNIV_INTERN -ibool +dberr_t buf_read_page( -/*==========*/ - ulint space, /*!< in: space id */ - ulint zip_size, /*!< in: compressed page size in bytes, or 0 */ - ulint offset, /*!< in: page number */ - trx_t* trx, /*!< in: trx */ - buf_page_t** bpage /*!< out: page */ -); + ulint space, + ulint zip_size, + ulint offset, + trx_t* trx); + /********************************************************************//** High-level function which reads a page asynchronously from a file to the buffer buf_pool if it is not already there. Sets the io_fix flag and sets an exclusive lock on the buffer frame. The flag is cleared and the x-lock released by the i/o-handler thread. -@return TRUE if page has been read in, FALSE in case of failure */ +@param[in] space Tablespace id +@param[in] offset Page number */ UNIV_INTERN -ibool +void buf_read_page_async( -/*================*/ - ulint space, /*!< in: space id */ - ulint offset);/*!< in: page number */ + ulint space, + ulint offset); + /********************************************************************//** Applies a random read-ahead in buf_pool if there are at least a threshold value of accessed pages from the random read-ahead area. Does not read any diff --git a/storage/xtradb/include/db0err.h b/storage/xtradb/include/db0err.h index b11c5f4ea1a..8bd3beda110 100644 --- a/storage/xtradb/include/db0err.h +++ b/storage/xtradb/include/db0err.h @@ -138,6 +138,8 @@ enum dberr_t { of missing key management plugin, or missing or incorrect key or incorret AES method or algorithm. */ + DB_PAGE_CORRUPTED, /* Page read from tablespace is + corrupted. */ /* The following are partial failure codes */ DB_FAIL = 1000, DB_OVERFLOW, diff --git a/storage/xtradb/include/dict0dict.h b/storage/xtradb/include/dict0dict.h index 1622b927a76..6da8eb892d9 100644 --- a/storage/xtradb/include/dict0dict.h +++ b/storage/xtradb/include/dict0dict.h @@ -1767,16 +1767,6 @@ dict_close(void); /*============*/ #ifndef UNIV_HOTBACKUP /**********************************************************************//** -Check whether the table is corrupted. -@return nonzero for corrupted table, zero for valid tables */ -UNIV_INLINE -ulint -dict_table_is_corrupted( -/*====================*/ - const dict_table_t* table) /*!< in: table */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); - -/**********************************************************************//** Check whether the index is corrupted. @return nonzero for corrupted index, zero for valid indexes */ UNIV_INLINE @@ -1820,6 +1810,15 @@ dict_set_corrupted_by_space( /*========================*/ ulint space_id); /*!< in: space ID */ +/**********************************************************************//** +Flags a table with specified space_id encrypted in the data dictionary +cache +@param[in] space_id Tablespace id */ +UNIV_INTERN +void +dict_set_encrypted_by_space( + ulint space_id); + /********************************************************************//** Validate the table flags. @return true if valid. */ @@ -1900,14 +1899,6 @@ dict_table_get_index_on_first_col( in table */ #endif /* !UNIV_HOTBACKUP */ -/************************************************************************* -set is_corrupt flag by space_id*/ - -void -dict_table_set_corrupt_by_space( -/*============================*/ - ulint space_id, - ibool need_mutex); #ifndef UNIV_NONINL #include "dict0dict.ic" diff --git a/storage/xtradb/include/dict0dict.ic b/storage/xtradb/include/dict0dict.ic index e3de7a33123..4ed1afc8094 100644 --- a/storage/xtradb/include/dict0dict.ic +++ b/storage/xtradb/include/dict0dict.ic @@ -1538,21 +1538,6 @@ dict_max_field_len_store_undo( } /********************************************************************//** -Check whether the table is corrupted. -@return nonzero for corrupted table, zero for valid tables */ -UNIV_INLINE -ulint -dict_table_is_corrupted( -/*====================*/ - const dict_table_t* table) /*!< in: table */ -{ - ut_ad(table); - ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); - - return(table->corrupted); -} - -/********************************************************************//** Check whether the index is corrupted. @return nonzero for corrupted index, zero for valid indexes */ UNIV_INLINE @@ -1564,7 +1549,7 @@ dict_index_is_corrupted( ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); return((index->type & DICT_CORRUPT) - || (index->table && index->table->corrupted)); + || (index->table && index->table->corrupted)); } /********************************************************************//** diff --git a/storage/xtradb/include/dict0mem.h b/storage/xtradb/include/dict0mem.h index 96c85cd8a99..a32581a0e90 100644 --- a/storage/xtradb/include/dict0mem.h +++ b/storage/xtradb/include/dict0mem.h @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -795,6 +795,9 @@ struct dict_index_t{ to first_blob_page_no; protected by blobs_mutex; @see btr_blob_dbg_t */ #endif /* UNIV_BLOB_DEBUG */ + + bool is_readable() const; + #ifdef UNIV_DEBUG ulint magic_n;/*!< magic number */ /** Value of dict_index_t::magic_n */ @@ -1062,11 +1065,13 @@ struct dict_table_t{ table is placed */ unsigned flags:DICT_TF_BITS; /*!< DICT_TF_... */ unsigned flags2:DICT_TF2_BITS; /*!< DICT_TF2_... */ - unsigned ibd_file_missing:1; - /*!< TRUE if this is in a single-table - tablespace and the .ibd file is missing; then - we must return in ha_innodb.cc an error if the - user tries to query such an orphaned table */ + unsigned file_unreadable:1; + /*!< true if this is in a single-table + tablespace and the .ibd file is missing + or page decryption failed and page is + corrupted; then we must return in + ha_innodb.cc an error if the + user tries to query such table */ unsigned cached:1;/*!< TRUE if the table object has been added to the dictionary cache */ unsigned to_be_dropped:1; @@ -1364,10 +1369,19 @@ struct dict_table_t{ UT_LIST_BASE_NODE_T(lock_t) locks; /*!< list of locks on the table; protected by lock_sys->mutex */ - ibool is_corrupt; - ibool is_encrypted; + #endif /* !UNIV_HOTBACKUP */ + /* Returns true if this is a single-table tablespace + and the .ibd file is missing or page decryption failed + and/or page is corrupted. + @return true if table is readable + @retval false if table is not readable */ + inline bool is_readable() const + { + return(UNIV_LIKELY(!file_unreadable)); + } + #ifdef UNIV_DEBUG ulint magic_n;/*!< magic number */ /** Value of dict_table_t::magic_n */ @@ -1375,6 +1389,16 @@ struct dict_table_t{ #endif /* UNIV_DEBUG */ }; +/* Returns true if this is a single-table tablespace +and the .ibd file is missing or page decryption failed +and/or page is corrupted. +@return true if table is readable +@retval false if table is not readable */ +inline bool dict_index_t::is_readable() const +{ + return(UNIV_LIKELY(!table->file_unreadable)); +} + /** A function object to add the foreign key constraint to the referenced set of the referenced table, if it exists in the dictionary cache. */ struct dict_foreign_add_to_referenced_table { diff --git a/storage/xtradb/include/fil0crypt.h b/storage/xtradb/include/fil0crypt.h index e7e9676aa3a..228dfb895fe 100644 --- a/storage/xtradb/include/fil0crypt.h +++ b/storage/xtradb/include/fil0crypt.h @@ -117,8 +117,7 @@ struct fil_space_crypt_t : st_encryption_scheme min_key_version(new_min_key_version), page0_offset(0), encryption(new_encryption), - mutex(), - key_found(new_min_key_version), + key_found(0), rotate_state() { key_id = new_key_id; @@ -136,6 +135,8 @@ struct fil_space_crypt_t : st_encryption_scheme type = CRYPT_SCHEME_1; min_key_version = key_get_latest_version(); } + + key_found = min_key_version; } /** Destructor */ @@ -298,13 +299,15 @@ Parse a MLOG_FILE_WRITE_CRYPT_DATA log entry @param[in] ptr Log entry start @param[in] end_ptr Log entry end @param[in] block buffer block +@param[out] err DB_SUCCESS or DB_DECRYPTION_FAILED @return position on log buffer */ UNIV_INTERN -const byte* +byte* fil_parse_write_crypt_data( - const byte* ptr, + byte* ptr, const byte* end_ptr, - const buf_block_t* block) + const buf_block_t* block, + dberr_t* err) MY_ATTRIBUTE((warn_unused_result)); /****************************************************************** diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index d73a68d9d34..0f7526a8e5e 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -650,23 +650,45 @@ fil_write_flushed_lsn_to_data_files( Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID +@param[in] silent whether to silently ignore missing tablespaces @param[in] for_io whether to look up the tablespace while performing I/O (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted or truncated */ +UNIV_INTERN fil_space_t* -fil_space_acquire(ulint id, bool for_io = false) +fil_space_acquire_low(ulint id, bool silent, bool for_io = false) MY_ATTRIBUTE((warn_unused_result)); +/** Acquire a tablespace when it could be dropped concurrently. +Used by background threads that do not necessarily hold proper locks +for concurrency control. +@param[in] id tablespace ID +@param[in] for_io whether to look up the tablespace while performing I/O + (possibly executing TRUNCATE) +@return the tablespace +@retval NULL if missing or being deleted or truncated */ +inline +fil_space_t* +fil_space_acquire(ulint id, bool for_io = false) +{ + return (fil_space_acquire_low(id, false, for_io)); +} + /** Acquire a tablespace that may not exist. Used by background threads that do not necessarily hold proper locks for concurrency control. @param[in] id tablespace ID +@param[in] for_io whether to look up the tablespace while performing I/O + (possibly executing TRUNCATE) @return the tablespace @retval NULL if missing or being deleted */ +inline fil_space_t* -fil_space_acquire_silent(ulint id) - MY_ATTRIBUTE((warn_unused_result)); +fil_space_acquire_silent(ulint id, bool for_io = false) +{ + return (fil_space_acquire_low(id, true, for_io)); +} /** Release a tablespace acquired with fil_space_acquire(). @param[in,out] space tablespace to release */ @@ -699,6 +721,64 @@ fil_space_keyrotate_next( fil_space_t* prev_space) MY_ATTRIBUTE((warn_unused_result)); +/** Wrapper with reference-counting for a fil_space_t. */ +class FilSpace +{ +public: + /** Default constructor: Use this when reference counting + is done outside this wrapper. */ + FilSpace() : m_space(NULL) {} + + /** Constructor: Look up the tablespace and increment the + reference count if found. + @param[in] space_id tablespace ID + @param[in] silent whether not print any errors + @param[in] for_io whether to look up the tablespace + while performing I/O + (possibly executing TRUNCATE) */ + explicit FilSpace(ulint space_id, bool silent = false, bool for_io = false) + : m_space(fil_space_acquire_low(space_id, silent, for_io)) {} + + /** Assignment operator: This assumes that fil_space_acquire() + has already been done for the fil_space_t. The caller must + assign NULL if it calls fil_space_release(). + @param[in] space tablespace to assign */ + class FilSpace& operator=(fil_space_t* space) + { + /* fil_space_acquire() must have been invoked. */ + ut_ad(space == NULL || space->n_pending_ops > 0); + m_space = space; + return(*this); + } + + /** Destructor - Decrement the reference count if a fil_space_t + is still assigned. */ + ~FilSpace() + { + if (m_space != NULL) { + fil_space_release(m_space); + } + } + + /** Implicit type conversion + @return the wrapped object */ + operator const fil_space_t*() const + { + return(m_space); + } + + /** Explicit type conversion + @return the wrapped object */ + const fil_space_t* operator()() const + { + return(m_space); + } + +private: + /** The wrapped pointer */ + fil_space_t* m_space; +}; + /*******************************************************************//** Reads the flushed lsn, arch no, and tablespace flag fields from a data file at database startup. @@ -721,22 +801,6 @@ fil_read_first_page( fil_space_crypt_t** crypt_data) /*!< out: crypt data */ __attribute__((warn_unused_result)); -/*******************************************************************//** -Increments the count of pending operation, if space is not being deleted. -@return TRUE if being deleted, and operation should be skipped */ -UNIV_INTERN -ibool -fil_inc_pending_ops( -/*================*/ - ulint id, /*!< in: space id */ - ibool print_err); /*!< in: need to print error or not */ -/*******************************************************************//** -Decrements the count of pending operations. */ -UNIV_INTERN -void -fil_decr_pending_ops( -/*=================*/ - ulint id); /*!< in: space id */ #endif /* !UNIV_HOTBACKUP */ /*******************************************************************//** Parses the body of a log record written about an .ibd file operation. That is, @@ -1428,36 +1492,6 @@ fil_space_get_by_id( /*================*/ ulint id); /*!< in: space id */ -/****************************************************************** -Get id of first tablespace or ULINT_UNDEFINED if none */ -UNIV_INTERN -ulint -fil_get_first_space(); -/*=================*/ - -/****************************************************************** -Get id of next tablespace or ULINT_UNDEFINED if none */ -UNIV_INTERN -ulint -fil_get_next_space( -/*===============*/ - ulint id); /*!< in: space id */ - -/****************************************************************** -Get id of first tablespace that has node or ULINT_UNDEFINED if none */ -UNIV_INTERN -ulint -fil_get_first_space_safe(); -/*======================*/ - -/****************************************************************** -Get id of next tablespace that has node or ULINT_UNDEFINED if none */ -UNIV_INTERN -ulint -fil_get_next_space_safe( -/*====================*/ - ulint id); /*!< in: previous space id */ - #endif /* UNIV_INNOCHECKSUM */ /****************************************************************//** diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index 120f1432ccf..d6d31b0c572 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -1383,7 +1383,12 @@ recv_parse_or_apply_log_rec_body( } break; case MLOG_FILE_WRITE_CRYPT_DATA: - ptr = const_cast<byte*>(fil_parse_write_crypt_data(ptr, end_ptr, block)); + dberr_t err; + ptr = const_cast<byte*>(fil_parse_write_crypt_data(ptr, end_ptr, block, &err)); + + if (err != DB_SUCCESS) { + recv_sys->found_corrupt_log = TRUE; + } break; default: ptr = NULL; @@ -1868,6 +1873,11 @@ recv_apply_hashed_log_recs(bool last_batch) break; } + if (recv_sys->found_corrupt_log) { + mutex_exit(&recv_sys->mutex); + return; + } + mutex_exit(&recv_sys->mutex); os_thread_sleep(500000); } @@ -1932,6 +1942,10 @@ recv_apply_hashed_log_recs(bool last_batch) mutex_exit(&(recv_sys->mutex)); + if (recv_sys->found_corrupt_log) { + return; + } + os_thread_sleep(500000); mutex_enter(&(recv_sys->mutex)); diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index baa7bcbea09..81d6fda9e53 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -2030,15 +2030,12 @@ PageConverter::validate( buf_block_t* block) UNIV_NOTHROW { buf_frame_t* page = get_frame(block); - ulint space_id = mach_read_from_4( - page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - fil_space_t* space = fil_space_found_by_id(space_id); /* Check that the page number corresponds to the offset in the file. Flag as corrupt if it doesn't. Disable the check for LSN in buf_page_is_corrupted() */ - if (buf_page_is_corrupted(false, page, get_zip_size(), space) + if (buf_page_is_corrupted(false, page, get_zip_size(), NULL) || (page_get_page_no(page) != offset / m_page_size && page_get_page_no(page) != 0)) { @@ -2201,7 +2198,7 @@ row_import_discard_changes( index->space = FIL_NULL; } - table->ibd_file_missing = TRUE; + table->file_unreadable = true; fil_close_tablespace(trx, table->space); } @@ -3447,7 +3444,7 @@ row_import_for_mysql( ut_a(table->space); ut_ad(prebuilt->trx); - ut_a(table->ibd_file_missing); + ut_a(table->file_unreadable); trx_start_if_not_started(prebuilt->trx); @@ -3752,7 +3749,7 @@ row_import_for_mysql( return(row_import_error(prebuilt, trx, err)); } - table->ibd_file_missing = false; + table->file_unreadable = false; table->flags2 &= ~DICT_TF2_DISCARDED; if (autoinc != 0) { diff --git a/storage/xtradb/row/row0ins.cc b/storage/xtradb/row/row0ins.cc index 8dbf093a17d..f4f96d32c50 100644 --- a/storage/xtradb/row/row0ins.cc +++ b/storage/xtradb/row/row0ins.cc @@ -1518,7 +1518,7 @@ run_again: } if (check_table == NULL - || check_table->ibd_file_missing + || check_table->file_unreadable || check_index == NULL) { if (!srv_read_only_mode && check_ref) { @@ -2410,8 +2410,7 @@ row_ins_clust_index_entry_low( &cursor, 0, __FILE__, __LINE__, &mtr); if (err != DB_SUCCESS) { - index->table->is_encrypted = true; - index->table->ibd_file_missing = true; + index->table->file_unreadable = true; mtr_commit(&mtr); goto func_exit; } @@ -2771,7 +2770,7 @@ row_ins_sec_index_entry_low( " used key_id is not available. " " Can't continue reading table.", index->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } goto func_exit; } diff --git a/storage/xtradb/row/row0log.cc b/storage/xtradb/row/row0log.cc index a17ee405720..666b59b42db 100644 --- a/storage/xtradb/row/row0log.cc +++ b/storage/xtradb/row/row0log.cc @@ -3674,7 +3674,7 @@ row_log_apply( rw_lock_x_lock(dict_index_get_lock(index)); - if (!dict_table_is_corrupted(index->table)) { + if (!index->table->corrupted) { error = row_log_apply_ops(trx, index, &dup); } else { error = DB_SUCCESS; diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index bf2d317f135..f0c804eb4bc 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -1620,7 +1620,8 @@ row_merge_read_clustered_index( page_cur_t* cur = btr_pcur_get_page_cur(&pcur); /* Do not continue if table pages are still encrypted */ - if (old_table->is_encrypted || new_table->is_encrypted) { + if (!old_table->is_readable() || + !new_table->is_readable()) { err = DB_DECRYPTION_FAILED; trx->error_key_num = 0; goto func_exit; @@ -3695,7 +3696,7 @@ row_merge_rename_tables_dict( table is in a non-system tablespace where space > 0. */ if (err == DB_SUCCESS && old_table->space != TRX_SYS_SPACE - && !old_table->ibd_file_missing) { + && fil_space_get(old_table->space) != NULL) { /* Make pathname to update SYS_DATAFILES. */ char* tmp_path = row_make_new_pathname(old_table, tmp_name); @@ -4005,6 +4006,7 @@ row_merge_build_indexes( os_mem_alloc_large(&block_size)); if (crypt_block == NULL) { + fil_space_release(space); DBUG_RETURN(DB_OUT_OF_MEMORY); } } else { @@ -4070,13 +4072,15 @@ row_merge_build_indexes( pct_cost = COST_READ_CLUSTERED_INDEX * 100 / (total_static_cost + total_dynamic_cost); /* Do not continue if we can't encrypt table pages */ - if (old_table->is_encrypted || new_table->is_encrypted) { + if (!old_table->is_readable() || + !new_table->is_readable()) { error = DB_DECRYPTION_FAILED; ib_push_warning(trx->mysql_thd, DB_DECRYPTION_FAILED, "Table %s is encrypted but encryption service or" " used key_id is not available. " " Can't continue reading table.", - old_table->is_encrypted ? old_table->name : new_table->name); + !old_table->is_readable() ? old_table->name : + new_table->name); goto func_exit; } diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index 7c8636d354f..a4141c36eda 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -663,6 +663,7 @@ handle_new_error: abort(); case DB_CORRUPTION: + case DB_PAGE_CORRUPTED: fputs("InnoDB: We detected index corruption" " in an InnoDB type table.\n" "InnoDB: You have to dump + drop + reimport" @@ -1280,6 +1281,71 @@ run_again: } /*********************************************************************//** +Determine is tablespace encrypted but decryption failed, is table corrupted +or is tablespace .ibd file missing. +@param[in] table Table +@param[in] trx Transaction +@param[in] push_warning true if we should push warning to user +@return DB_DECRYPTION_FAILED table is encrypted but decryption failed +DB_CORRUPTION table is corrupted +DB_TABLESPACE_NOT_FOUND tablespace .ibd file not found */ +static +dberr_t +row_mysql_get_table_status( + const dict_table_t* table, + trx_t* trx, + bool push_warning = true) +{ + dberr_t err = DB_SUCCESS; + FilSpace space(table->space, true); + char buf[MAX_FULL_NAME_LEN]; + ut_format_name(table->name, TRUE, buf, sizeof(buf)); + + if (space()) { + + if (space()->crypt_data && space()->crypt_data->is_encrypted()) { + // maybe we cannot access the table due to failing + // to decrypt + if (push_warning) { + ib_push_warning(trx,HA_ERR_DECRYPTION_FAILED, + "Table %s in file %s is encrypted but encryption service or" + " used key_id %u is not available. " + " Can't continue reading table.", + buf, space()->chain.start->name, + space()->crypt_data->key_id); + } + + err = DB_DECRYPTION_FAILED; + } else { + if (push_warning) { + ib_push_warning(trx, DB_CORRUPTION, + "Table %s in file %s corrupted.", + buf, space()->chain.start->name); + } + + err = DB_CORRUPTION; + } + + } else { + ib_logf(IB_LOG_LEVEL_ERROR, + "InnoDB: MySQL is trying to use a table handle" + " but the .ibd file for" + " table %s does not exist." + " Have you deleted the .ibd file" + " from the database directory under" + " the MySQL datadir, or have you" + " used DISCARD TABLESPACE?" + " Look from " REFMAN "innodb-troubleshooting.html" + " how you can resolve the problem.", + buf); + + err = DB_TABLESPACE_NOT_FOUND; + } + + return (err); +} + +/*********************************************************************//** Does an insert for MySQL. @return error code or DB_SUCCESS */ UNIV_INTERN @@ -1308,20 +1374,8 @@ row_insert_for_mysql( return(DB_TABLESPACE_DELETED); - } else if (prebuilt->table->ibd_file_missing) { - - ib_logf(IB_LOG_LEVEL_ERROR, - ".ibd file is missing for table %s", - prebuilt->table->name); - - return(DB_TABLESPACE_NOT_FOUND); - } else if (prebuilt->table->is_encrypted) { - ib_push_warning(trx, DB_DECRYPTION_FAILED, - "Table %s in tablespace %lu encrypted." - "However key management plugin or used key_id is not found or" - " used encryption algorithm or method does not match.", - prebuilt->table->name, prebuilt->table->space); - return(DB_DECRYPTION_FAILED); + } else if (!prebuilt->table->is_readable()) { + return (row_mysql_get_table_status(prebuilt->table, trx, true)); } else if (prebuilt->magic_n != ROW_PREBUILT_ALLOCATED) { fprintf(stderr, "InnoDB: Error: trying to free a corrupt\n" @@ -1712,28 +1766,8 @@ row_update_for_mysql( ut_ad(trx != NULL); UT_NOT_USED(mysql_rec); - if (prebuilt->table->ibd_file_missing) { - ut_print_timestamp(stderr); - fprintf(stderr, " InnoDB: Error:\n" - "InnoDB: MySQL is trying to use a table handle" - " but the .ibd file for\n" - "InnoDB: table %s does not exist.\n" - "InnoDB: Have you deleted the .ibd file" - " from the database directory under\n" - "InnoDB: the MySQL datadir, or have you" - " used DISCARD TABLESPACE?\n" - "InnoDB: Look from\n" - "InnoDB: " REFMAN "innodb-troubleshooting.html\n" - "InnoDB: how you can resolve the problem.\n", - prebuilt->table->name); - return(DB_ERROR); - } else if (prebuilt->table->is_encrypted) { - ib_push_warning(trx, DB_DECRYPTION_FAILED, - "Table %s in tablespace %lu encrypted." - "However key management plugin or used key_id is not found or" - " used encryption algorithm or method does not match.", - prebuilt->table->name, prebuilt->table->space); - return (DB_TABLE_NOT_FOUND); + if (!table->is_readable()) { + return (row_mysql_get_table_status(table, trx, true)); } if (UNIV_UNLIKELY(prebuilt->magic_n != ROW_PREBUILT_ALLOCATED)) { @@ -3128,7 +3162,7 @@ row_discard_tablespace( /* All persistent operations successful, update the data dictionary memory cache. */ - table->ibd_file_missing = TRUE; + table->file_unreadable = true; table->flags2 |= DICT_TF2_DISCARDED; @@ -3185,8 +3219,6 @@ row_discard_tablespace_for_mysql( if (table == 0) { err = DB_TABLE_NOT_FOUND; - } else if (table->is_encrypted) { - err = DB_DECRYPTION_FAILED; } else if (table->space == TRX_SYS_SPACE) { char table_name[MAX_FULL_NAME_LEN + 1]; @@ -3401,10 +3433,8 @@ row_truncate_table_for_mysql( if (dict_table_is_discarded(table)) { return(DB_TABLESPACE_DELETED); - } else if (table->is_encrypted) { - return(DB_DECRYPTION_FAILED); - } else if (table->ibd_file_missing) { - return(DB_TABLESPACE_NOT_FOUND); + } else if (!table->is_readable()) { + return (row_mysql_get_table_status(table, trx, true)); } trx_start_for_ddl(trx, TRX_DICT_OP_TABLE); @@ -3570,7 +3600,7 @@ row_truncate_table_for_mysql( "create a new tablespace", table->name); - table->ibd_file_missing = 1; + table->file_unreadable = true; err = DB_ERROR; goto funct_exit; } @@ -3994,19 +4024,6 @@ row_drop_table_for_mysql( goto funct_exit; } - /* If table is encrypted and table page encryption failed - return error. */ - if (table->is_encrypted) { - - if (table->can_be_evicted) { - dict_table_move_from_lru_to_non_lru(table); - } - - dict_table_close(table, TRUE, FALSE); - err = DB_DECRYPTION_FAILED; - goto funct_exit; - } - /* Turn on this drop bit before we could release the dictionary latch */ table->to_be_dropped = true; @@ -4378,7 +4395,7 @@ row_drop_table_for_mysql( from table->heap, which will be freed by dict_table_remove_from_cache(table) below. */ space_id = table->space; - ibd_file_missing = table->ibd_file_missing; + ibd_file_missing = table->file_unreadable; table_flags = table->flags; is_temp = DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY); @@ -4389,7 +4406,8 @@ row_drop_table_for_mysql( have a temp flag but not know the temp path */ ut_a(table->dir_path_of_temp_table == NULL || is_temp); if (dict_table_is_discarded(table) - || table->ibd_file_missing) { + || (!table->is_readable() + && fil_space_get(table->space) == NULL)) { /* Do not attempt to drop known-to-be-missing tablespaces. */ space_id = 0; @@ -4801,7 +4819,8 @@ loop: "'%s.frm' was lost.", table->name); } - if (table->ibd_file_missing) { + if (!table->is_readable() + && fil_space_get(table->space) == NULL) { ib_logf(IB_LOG_LEVEL_WARN, "Missing %s.ibd file for table %s.", table->name, table->name); @@ -5075,7 +5094,8 @@ row_rename_table_for_mysql( stderr); goto funct_exit; - } else if (table->ibd_file_missing + } else if (!table->is_readable() + && fil_space_get(table->space) == NULL && !dict_table_is_discarded(table)) { err = DB_TABLE_NOT_FOUND; @@ -5145,7 +5165,7 @@ row_rename_table_for_mysql( which have space IDs > 0. */ if (err == DB_SUCCESS && table->space != TRX_SYS_SPACE - && !table->ibd_file_missing) { + && table->is_readable()) { /* Make a new pathname to update SYS_DATAFILES. */ char* new_path = row_make_new_pathname(table, new_name); diff --git a/storage/xtradb/row/row0purge.cc b/storage/xtradb/row/row0purge.cc index 35b3520749b..8a1dbd6f69f 100644 --- a/storage/xtradb/row/row0purge.cc +++ b/storage/xtradb/row/row0purge.cc @@ -777,7 +777,7 @@ row_purge_parse_undo_rec( goto err_exit; } - if (node->table->ibd_file_missing) { + if (node->table->file_unreadable) { /* We skip purge of missing .ibd files */ dict_table_close(node->table, FALSE, FALSE); diff --git a/storage/xtradb/row/row0sel.cc b/storage/xtradb/row/row0sel.cc index 29569bb31e7..9a583e2023a 100644 --- a/storage/xtradb/row/row0sel.cc +++ b/storage/xtradb/row/row0sel.cc @@ -62,6 +62,7 @@ Created 12/19/1997 Heikki Tuuri #include "srv0start.h" #include "m_string.h" /* for my_sys.h */ #include "my_sys.h" /* DEBUG_SYNC_C */ +#include "fil0fil.h" #include "my_compare.h" /* enum icp_result */ @@ -3728,13 +3729,12 @@ row_search_for_mysql( return(DB_TABLESPACE_DELETED); - } else if (prebuilt->table->ibd_file_missing) { - - return(DB_TABLESPACE_NOT_FOUND); - - } else if (prebuilt->table->is_encrypted) { - - return(DB_DECRYPTION_FAILED); + } else if (!prebuilt->table->is_readable()) { + if (fil_space_get(prebuilt->table->space) == NULL) { + return(DB_TABLESPACE_NOT_FOUND); + } else { + return(DB_DECRYPTION_FAILED); + } } else if (!prebuilt->index_usable) { return(DB_MISSING_HISTORY); @@ -4193,7 +4193,7 @@ wait_table_again: " used key_id is not available. " " Can't continue reading table.", prebuilt->table->name); - index->table->is_encrypted = true; + index->table->file_unreadable = true; } rec = NULL; goto lock_wait_or_error; @@ -4213,7 +4213,7 @@ rec_loop: rec = btr_pcur_get_rec(pcur); - if (!rec) { + if (!index->table->is_readable()) { err = DB_DECRYPTION_FAILED; goto lock_wait_or_error; } @@ -4305,7 +4305,7 @@ rec_loop: wrong_offs: if (srv_pass_corrupt_table && index->table->space != 0 && index->table->space < SRV_LOG_SPACE_FIRST_ID) { - index->table->is_corrupt = TRUE; + index->table->file_unreadable = TRUE; fil_space_set_corrupt(index->table->space); } @@ -4366,20 +4366,22 @@ wrong_offs: offsets = rec_get_offsets(rec, index, offsets, ULINT_UNDEFINED, &heap); if (UNIV_UNLIKELY(srv_force_recovery > 0 - || (index->table->is_corrupt && + || (!index->table->is_readable() && srv_pass_corrupt_table == 2))) { if (!rec_validate(rec, offsets) || !btr_index_rec_validate(rec, index, FALSE)) { - fprintf(stderr, - "InnoDB: Index corruption: rec offs %lu" - " next offs %lu, page no %lu,\n" - "InnoDB: ", - (ulong) page_offset(rec), - (ulong) next_offs, - (ulong) page_get_page_no(page_align(rec))); - dict_index_name_print(stderr, trx, index); - fputs(". We try to skip the record.\n", - stderr); + char buf[MAX_FULL_NAME_LEN]; + ut_format_name(index->table->name, FALSE, buf, sizeof(buf)); + + ib_logf(IB_LOG_LEVEL_ERROR, + "Index %s corrupted: rec offs " ULINTPF + " next offs " ULINTPF + ", page no " ULINTPF " ." + " We try to skip the record.", + buf, + page_offset(rec), + next_offs, + page_get_page_no(page_align(rec))); goto next_rec; } diff --git a/storage/xtradb/row/row0uins.cc b/storage/xtradb/row/row0uins.cc index 651042fb820..f14a4ef9bcf 100644 --- a/storage/xtradb/row/row0uins.cc +++ b/storage/xtradb/row/row0uins.cc @@ -320,7 +320,7 @@ row_undo_ins_parse_undo_rec( /* Skip the UNDO if we can't find the table or the .ibd file. */ if (UNIV_UNLIKELY(node->table == NULL)) { - } else if (UNIV_UNLIKELY(node->table->ibd_file_missing)) { + } else if (UNIV_UNLIKELY(node->table->file_unreadable)) { close_table: dict_table_close(node->table, dict_locked, FALSE); node->table = NULL; diff --git a/storage/xtradb/row/row0umod.cc b/storage/xtradb/row/row0umod.cc index 19576d976bd..8deba4f00a5 100644 --- a/storage/xtradb/row/row0umod.cc +++ b/storage/xtradb/row/row0umod.cc @@ -1068,7 +1068,7 @@ row_undo_mod_parse_undo_rec( return; } - if (node->table->ibd_file_missing) { + if (node->table->file_unreadable) { dict_table_close(node->table, dict_locked, FALSE); /* We skip undo operations to missing .ibd files */ diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index 18cde4e416f..bc1d5acfd4d 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -2562,6 +2562,11 @@ files_checked: recv_group_scan_log_recs(). */ recv_apply_hashed_log_recs(true); + + if (recv_sys->found_corrupt_log) { + return (DB_CORRUPTION); + } + DBUG_PRINT("ib_log", ("apply completed")); } diff --git a/storage/xtradb/trx/trx0trx.cc b/storage/xtradb/trx/trx0trx.cc index 92d7525ea84..ed80bafa6c6 100644 --- a/storage/xtradb/trx/trx0trx.cc +++ b/storage/xtradb/trx/trx0trx.cc @@ -661,7 +661,7 @@ trx_resurrect_table_locks( i != tables.end(); i++) { if (dict_table_t* table = dict_table_open_on_id( *i, FALSE, DICT_TABLE_OP_LOAD_TABLESPACE)) { - if (table->ibd_file_missing + if (table->file_unreadable || dict_table_is_temporary(table)) { mutex_enter(&dict_sys->mutex); dict_table_close(table, TRUE, FALSE); diff --git a/storage/xtradb/ut/ut0ut.cc b/storage/xtradb/ut/ut0ut.cc index acedb56879a..fd52537ae11 100644 --- a/storage/xtradb/ut/ut0ut.cc +++ b/storage/xtradb/ut/ut0ut.cc @@ -852,6 +852,8 @@ ut_strerr( return("BLOB record length is greater than 10%% of redo log"); case DB_DECRYPTION_FAILED: return("Table is encrypted but decrypt failed."); + case DB_PAGE_CORRUPTED: + return("Page read from tablespace is corrupted."); /* do not add default: in order to produce a warning if new code is added to the enum but not added here */ |