diff options
Diffstat (limited to 'storage/innobase')
211 files changed, 7938 insertions, 27200 deletions
diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 799200ae43f..64daf36ffd6 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright (c) 2006, 2016, Oracle and/or its affiliates. All rights reserved. -# Copyright (c) 2014, 2016, MariaDB Corporation +# Copyright (c) 2014, 2017, MariaDB Corporation # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -36,8 +36,6 @@ MYSQL_CHECK_NUMA() INCLUDE(innodb.cmake) SET(INNOBASE_SOURCES - api/api0api.cc - api/api0misc.cc btr/btr0btr.cc btr/btr0bulk.cc btr/btr0cur.cc diff --git a/storage/innobase/api/api0api.cc b/storage/innobase/api/api0api.cc deleted file mode 100644 index c6947b6f054..00000000000 --- a/storage/innobase/api/api0api.cc +++ /dev/null @@ -1,3438 +0,0 @@ -/***************************************************************************** - -Copyright (c) 2008, 2016, Oracle and/or its affiliates. All Rights Reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/**************************************************//** -@file api/api0api.cc -InnoDB Native API - -2008-08-01 Created Sunny Bains -3/20/2011 Jimmy Yang extracted from Embedded InnoDB -*******************************************************/ - -#include "ha_prototypes.h" - -#include "api0api.h" -#include "api0misc.h" -#include "srv0start.h" -#include "dict0dict.h" -#include "btr0pcur.h" -#include "row0ins.h" -#include "row0upd.h" -#include "row0vers.h" -#include "trx0roll.h" -#include "dict0crea.h" -#include "row0merge.h" -#include "pars0pars.h" -#include "lock0types.h" -#include "row0sel.h" -#include "lock0lock.h" -#include "rem0cmp.h" -#include "dict0priv.h" -#include "trx0roll.h" -#include "row0trunc.h" - -/** configure variable for binlog option with InnoDB APIs */ -my_bool ib_binlog_enabled = FALSE; - -/** configure variable for MDL option with InnoDB APIs */ -my_bool ib_mdl_enabled = FALSE; - -/** configure variable for disable rowlock with InnoDB APIs */ -my_bool ib_disable_row_lock = FALSE; - -/** configure variable for Transaction isolation levels */ -ulong ib_trx_level_setting = IB_TRX_READ_UNCOMMITTED; - -/** configure variable for background commit interval in seconds */ -ulong ib_bk_commit_interval = 0; - -/** InnoDB tuple types. */ -enum ib_tuple_type_t{ - TPL_TYPE_ROW, /*!< Data row tuple */ - TPL_TYPE_KEY /*!< Index key tuple */ -}; - -/** Query types supported. */ -enum ib_qry_type_t{ - QRY_NON, /*!< None/Sentinel */ - QRY_INS, /*!< Insert operation */ - QRY_UPD, /*!< Update operation */ - QRY_SEL /*!< Select operation */ -}; - -/** Query graph types. */ -struct ib_qry_grph_t { - que_fork_t* ins; /*!< Innobase SQL query graph used - in inserts */ - que_fork_t* upd; /*!< Innobase SQL query graph used - in updates or deletes */ - que_fork_t* sel; /*!< dummy query graph used in - selects */ -}; - -/** Query node types. */ -struct ib_qry_node_t { - ins_node_t* ins; /*!< Innobase SQL insert node - used to perform inserts to the table */ - upd_node_t* upd; /*!< Innobase SQL update node - used to perform updates and deletes */ - sel_node_t* sel; /*!< Innobase SQL select node - used to perform selects on the table */ -}; - -/** Query processing fields. */ -struct ib_qry_proc_t { - - ib_qry_node_t node; /*!< Query node*/ - - ib_qry_grph_t grph; /*!< Query graph */ -}; - -/** Cursor instance for traversing tables/indexes. This will eventually -become row_prebuilt_t. */ -struct ib_cursor_t { - mem_heap_t* heap; /*!< Instance heap */ - - mem_heap_t* query_heap; /*!< Heap to use for query graphs */ - - ib_qry_proc_t q_proc; /*!< Query processing info */ - - ib_match_mode_t match_mode; /*!< ib_cursor_moveto match mode */ - - row_prebuilt_t* prebuilt; /*!< For reading rows */ - - bool valid_trx; /*!< Valid transaction attached */ -}; - -/** InnoDB table columns used during table and index schema creation. */ -struct ib_col_t { - const char* name; /*!< Name of column */ - - ib_col_type_t ib_col_type; /*!< Main type of the column */ - - ulint len; /*!< Length of the column */ - - ib_col_attr_t ib_col_attr; /*!< Column attributes */ - -}; - -/** InnoDB index columns used during index and index schema creation. */ -struct ib_key_col_t { - const char* name; /*!< Name of column */ - - ulint prefix_len; /*!< Column index prefix len or 0 */ -}; - -struct ib_table_def_t; - -/** InnoDB index schema used during index creation */ -struct ib_index_def_t { - mem_heap_t* heap; /*!< Heap used to build this and all - its columns in the list */ - - const char* name; /*!< Index name */ - - dict_table_t* table; /*!< Parent InnoDB table */ - - ib_table_def_t* schema; /*!< Parent table schema that owns - this instance */ - - ibool clustered; /*!< True if clustered index */ - - ibool unique; /*!< True if unique index */ - - ib_vector_t* cols; /*!< Vector of columns */ - - trx_t* usr_trx; /*!< User transacton covering the - DDL operations */ -}; - -/** InnoDB table schema used during table creation */ -struct ib_table_def_t { - mem_heap_t* heap; /*!< Heap used to build this and all - its columns in the list */ - const char* name; /*!< Table name */ - - ib_tbl_fmt_t ib_tbl_fmt; /*!< Row format */ - - ulint page_size; /*!< Page size */ - - ib_vector_t* cols; /*!< Vector of columns */ - - ib_vector_t* indexes; /*!< Vector of indexes */ - - dict_table_t* table; /* Table read from or NULL */ -}; - -/** InnoDB tuple used for key operations. */ -struct ib_tuple_t { - mem_heap_t* heap; /*!< Heap used to build - this and for copying - the column values. */ - - ib_tuple_type_t type; /*!< Tuple discriminitor. */ - - const dict_index_t* index; /*!< Index for tuple can be either - secondary or cluster index. */ - - dtuple_t* ptr; /*!< The internal tuple - instance */ -}; - -/** The following counter is used to convey information to InnoDB -about server activity: in case of normal DML ops it is not -sensible to call srv_active_wake_master_thread after each -operation, we only do it every INNOBASE_WAKE_INTERVAL'th step. */ - -#define INNOBASE_WAKE_INTERVAL 32 - -/*****************************************************************//** -Check whether the InnoDB persistent cursor is positioned. -@return IB_TRUE if positioned */ -UNIV_INLINE -ib_bool_t -ib_btr_cursor_is_positioned( -/*========================*/ - btr_pcur_t* pcur) /*!< in: InnoDB persistent cursor */ -{ - return(pcur->old_stored - && (pcur->pos_state == BTR_PCUR_IS_POSITIONED - || pcur->pos_state == BTR_PCUR_WAS_POSITIONED)); -} - - -/********************************************************************//** -Open a table using the table id, if found then increment table ref count. -@return table instance if found */ -static -dict_table_t* -ib_open_table_by_id( -/*================*/ - ib_id_u64_t tid, /*!< in: table id to lookup */ - ib_bool_t locked) /*!< in: TRUE if own dict mutex */ -{ - dict_table_t* table; - table_id_t table_id; - - table_id = tid; - - if (!locked) { - dict_mutex_enter_for_mysql(); - } - - table = dict_table_open_on_id(table_id, TRUE, DICT_TABLE_OP_NORMAL); - - if (table != NULL && table->ibd_file_missing) { - table = NULL; - } - - if (!locked) { - dict_mutex_exit_for_mysql(); - } - - return(table); -} - -/********************************************************************//** -Open a table using the table name, if found then increment table ref count. -@return table instance if found */ -static -dict_table_t* -ib_open_table_by_name( -/*==================*/ - const char* name) /*!< in: table name to lookup */ -{ - dict_table_t* table; - - table = dict_table_open_on_name(name, FALSE, FALSE, - DICT_ERR_IGNORE_NONE); - - if (table != NULL && table->ibd_file_missing) { - table = NULL; - } - - return(table); -} - -/********************************************************************//** -Find table using table name. -@return table instance if found */ -static -dict_table_t* -ib_lookup_table_by_name( -/*====================*/ - const char* name) /*!< in: table name to lookup */ -{ - dict_table_t* table; - - table = dict_table_get_low(name); - - if (table != NULL && table->ibd_file_missing) { - table = NULL; - } - - return(table); -} - -/********************************************************************//** -Increments innobase_active_counter and every INNOBASE_WAKE_INTERVALth -time calls srv_active_wake_master_thread. This function should be used -when a single database operation may introduce a small need for -server utility activity, like checkpointing. */ -UNIV_INLINE -void -ib_wake_master_thread(void) -/*=======================*/ -{ - static ulint ib_signal_counter = 0; - - ++ib_signal_counter; - - if ((ib_signal_counter % INNOBASE_WAKE_INTERVAL) == 0) { - srv_active_wake_master_thread(); - } -} - -/*****************************************************************//** -Read the columns from a rec into a tuple. */ -static -void -ib_read_tuple( -/*==========*/ - const rec_t* rec, /*!< in: Record to read */ - ib_bool_t page_format, /*!< in: IB_TRUE if compressed format */ - ib_tuple_t* tuple, /*!< in: tuple to read into */ - void** rec_buf, /*!< in/out: row buffer */ - ulint* len) /*!< in/out: buffer len */ -{ - ulint i; - void* ptr; - rec_t* copy; - ulint rec_meta_data; - ulint n_index_fields; - ulint offsets_[REC_OFFS_NORMAL_SIZE]; - ulint* offsets = offsets_; - dtuple_t* dtuple = tuple->ptr; - const dict_index_t* index = tuple->index; - ulint offset_size; - - rec_offs_init(offsets_); - - offsets = rec_get_offsets( - rec, index, offsets, ULINT_UNDEFINED, &tuple->heap); - - rec_meta_data = rec_get_info_bits(rec, page_format); - dtuple_set_info_bits(dtuple, rec_meta_data); - - offset_size = rec_offs_size(offsets); - - if (rec_buf && *rec_buf) { - if (*len < offset_size) { - free(*rec_buf); - *rec_buf = malloc(offset_size); - *len = offset_size; - } - ptr = *rec_buf; - } else { - /* Make a copy of the rec. */ - ptr = mem_heap_alloc(tuple->heap, offset_size); - } - - copy = rec_copy(ptr, rec, offsets); - - n_index_fields = ut_min( - rec_offs_n_fields(offsets), dtuple_get_n_fields(dtuple)); - - for (i = 0; i < n_index_fields; ++i) { - ulint len; - const byte* data; - dfield_t* dfield; - - if (tuple->type == TPL_TYPE_ROW) { - const dict_col_t* col; - ulint col_no; - const dict_field_t* index_field; - - index_field = dict_index_get_nth_field(index, i); - col = dict_field_get_col(index_field); - col_no = dict_col_get_no(col); - - dfield = dtuple_get_nth_field(dtuple, col_no); - } else { - dfield = dtuple_get_nth_field(dtuple, i); - } - - data = rec_get_nth_field(copy, offsets, i, &len); - - /* Fetch and copy any externally stored column. */ - if (rec_offs_nth_extern(offsets, i)) { - - const page_size_t page_size( - dict_table_page_size(index->table)); - - data = btr_rec_copy_externally_stored_field( - copy, offsets, page_size, i, &len, - tuple->heap); - - ut_a(len != UNIV_SQL_NULL); - } - - dfield_set_data(dfield, data, len); - } -} - -/*****************************************************************//** -Create an InnoDB key tuple. -@return tuple instance created, or NULL */ -static -ib_tpl_t -ib_key_tuple_new_low( -/*=================*/ - const dict_index_t* index, /*!< in: index for which tuple - required */ - ulint n_cols, /*!< in: no. of user defined cols */ - mem_heap_t* heap) /*!< in: memory heap */ -{ - ib_tuple_t* tuple; - ulint i; - ulint n_cmp_cols; - - tuple = static_cast<ib_tuple_t*>( - mem_heap_alloc(heap, sizeof(*tuple))); - - if (tuple == NULL) { - mem_heap_free(heap); - return(NULL); - } - - tuple->heap = heap; - tuple->index = index; - tuple->type = TPL_TYPE_KEY; - - /* Is it a generated clustered index ? */ - if (n_cols == 0) { - ++n_cols; - } - - tuple->ptr = dtuple_create(heap, n_cols); - - /* Copy types and set to SQL_NULL. */ - dict_index_copy_types(tuple->ptr, index, n_cols); - - for (i = 0; i < n_cols; i++) { - - dfield_t* dfield; - - dfield = dtuple_get_nth_field(tuple->ptr, i); - dfield_set_null(dfield); - } - - n_cmp_cols = dict_index_get_n_ordering_defined_by_user(index); - - dtuple_set_n_fields_cmp(tuple->ptr, n_cmp_cols); - - return((ib_tpl_t) tuple); -} - -/*****************************************************************//** -Create an InnoDB key tuple. -@return tuple instance created, or NULL */ -static -ib_tpl_t -ib_key_tuple_new( -/*=============*/ - const dict_index_t* index, /*!< in: index of tuple */ - ulint n_cols) /*!< in: no. of user defined cols */ -{ - mem_heap_t* heap; - - heap = mem_heap_create(64); - - if (heap == NULL) { - return(NULL); - } - - return(ib_key_tuple_new_low(index, n_cols, heap)); -} - -/*****************************************************************//** -Create an InnoDB row tuple. -@return tuple instance, or NULL */ -static -ib_tpl_t -ib_row_tuple_new_low( -/*=================*/ - const dict_index_t* index, /*!< in: index of tuple */ - ulint n_cols, /*!< in: no. of cols in tuple */ - mem_heap_t* heap) /*!< in: memory heap */ -{ - ib_tuple_t* tuple; - - tuple = static_cast<ib_tuple_t*>(mem_heap_alloc(heap, sizeof(*tuple))); - - if (tuple == NULL) { - mem_heap_free(heap); - return(NULL); - } - - tuple->heap = heap; - tuple->index = index; - tuple->type = TPL_TYPE_ROW; - - tuple->ptr = dtuple_create(heap, n_cols); - - /* Copy types and set to SQL_NULL. */ - dict_table_copy_types(tuple->ptr, index->table); - - return((ib_tpl_t) tuple); -} - -/*****************************************************************//** -Create an InnoDB row tuple. -@return tuple instance, or NULL */ -static -ib_tpl_t -ib_row_tuple_new( -/*=============*/ - const dict_index_t* index, /*!< in: index of tuple */ - ulint n_cols) /*!< in: no. of cols in tuple */ -{ - mem_heap_t* heap; - - heap = mem_heap_create(64); - - if (heap == NULL) { - return(NULL); - } - - return(ib_row_tuple_new_low(index, n_cols, heap)); -} - -/*****************************************************************//** -Begin a transaction. -@return innobase txn handle */ -ib_err_t -ib_trx_start( -/*=========*/ - ib_trx_t ib_trx, /*!< in: transaction to restart */ - ib_trx_level_t ib_trx_level, /*!< in: trx isolation level */ - ib_bool_t read_write, /*!< in: true if read write - transaction */ - ib_bool_t auto_commit, /*!< in: auto commit after each - single DML */ - void* thd) /*!< in: THD */ -{ - ib_err_t err = DB_SUCCESS; - trx_t* trx = (trx_t*) ib_trx; - - ut_a(ib_trx_level <= IB_TRX_SERIALIZABLE); - - trx->api_trx = true; - trx->api_auto_commit = auto_commit; - trx->read_write = read_write; - - trx_start_if_not_started(trx, read_write); - - trx->isolation_level = ib_trx_level; - - /* FIXME: This is a place holder, we should add an arg that comes - from the client. */ - trx->mysql_thd = static_cast<THD*>(thd); - - return(err); -} - -/*****************************************************************//** -Begin a transaction. This will allocate a new transaction handle. -put the transaction in the active state. -@return innobase txn handle */ -ib_trx_t -ib_trx_begin( -/*=========*/ - ib_trx_level_t ib_trx_level, /*!< in: trx isolation level */ - ib_bool_t read_write, /*!< in: true if read write - transaction */ - ib_bool_t auto_commit) /*!< in: auto commit after each - single DML */ -{ - trx_t* trx; - ib_bool_t started; - - trx = trx_allocate_for_mysql(); - - started = ib_trx_start(static_cast<ib_trx_t>(trx), ib_trx_level, - read_write, auto_commit, NULL); - ut_a(started); - - return(static_cast<ib_trx_t>(trx)); -} - -/*****************************************************************//** -Check if transaction is read_only -@return transaction read_only status */ -ib_u32_t -ib_trx_read_only( -/*=============*/ - ib_trx_t ib_trx) /*!< in: trx handle */ -{ - trx_t* trx = (trx_t*) ib_trx; - - return(trx->read_only); -} -/*****************************************************************//** -Get a trx start time. -@return trx start_time */ -ib_u64_t -ib_trx_get_start_time( -/*==================*/ - ib_trx_t ib_trx) /*!< in: transaction */ -{ - trx_t* trx = (trx_t*) ib_trx; - return(static_cast<ib_u64_t>(trx->start_time)); -} -/*****************************************************************//** -Release the resources of the transaction. -@return DB_SUCCESS or err code */ -ib_err_t -ib_trx_release( -/*===========*/ - ib_trx_t ib_trx) /*!< in: trx handle */ -{ - trx_t* trx = (trx_t*) ib_trx; - - ut_ad(trx != NULL); - trx_free_for_mysql(trx); - - return(DB_SUCCESS); -} - -/*****************************************************************//** -Commit a transaction. This function will also release the schema -latches too. -@return DB_SUCCESS or err code */ -ib_err_t -ib_trx_commit( -/*==========*/ - ib_trx_t ib_trx) /*!< in: trx handle */ -{ - ib_err_t err = DB_SUCCESS; - trx_t* trx = reinterpret_cast<trx_t*>(ib_trx); - - if (!trx_is_started(trx)) { - - return(err); - } - - trx_commit(trx); - - return(DB_SUCCESS); -} - -/*****************************************************************//** -Rollback a transaction. This function will also release the schema -latches too. -@return DB_SUCCESS or err code */ -ib_err_t -ib_trx_rollback( -/*============*/ - ib_trx_t ib_trx) /*!< in: trx handle */ -{ - ib_err_t err; - trx_t* trx = (trx_t*) ib_trx; - - err = static_cast<ib_err_t>(trx_rollback_for_mysql(trx)); - - /* It should always succeed */ - ut_a(err == DB_SUCCESS); - - return(err); -} - -#ifdef _WIN32 -/*****************************************************************//** -Convert a string to lower case. */ -static -void -ib_to_lower_case( -/*=============*/ - char* ptr) /*!< string to convert to lower case */ -{ - while (*ptr) { - *ptr = tolower(*ptr); - ++ptr; - } -} -#endif /* _WIN32 */ - -/*****************************************************************//** -Normalizes a table name string. A normalized name consists of the -database name catenated to '/' and table name. An example: -test/mytable. On Windows normalization puts both the database name and the -table name always to lower case. This function can be called for system -tables and they don't have a database component. For tables that don't have -a database component, we don't normalize them to lower case on Windows. -The assumption is that they are system tables that reside in the system -table space. */ -static -void -ib_normalize_table_name( -/*====================*/ - char* norm_name, /*!< out: normalized name as a - null-terminated string */ - const char* name) /*!< in: table name string */ -{ - const char* ptr = name; - - /* Scan name from the end */ - - ptr += ut_strlen(name) - 1; - - /* Find the start of the table name. */ - while (ptr >= name && *ptr != '\\' && *ptr != '/' && ptr > name) { - --ptr; - } - - - /* For system tables there is no '/' or dbname. */ - ut_a(ptr >= name); - - if (ptr > name) { - const char* db_name; - const char* table_name; - - table_name = ptr + 1; - - --ptr; - - while (ptr >= name && *ptr != '\\' && *ptr != '/') { - ptr--; - } - - db_name = ptr + 1; - - memcpy(norm_name, db_name, - ut_strlen(name) + 1 - (db_name - name)); - - norm_name[table_name - db_name - 1] = '/'; -#ifdef _WIN32 - ib_to_lower_case(norm_name); -#endif - } else { - ut_strcpy(norm_name, name); - } -} - -/*****************************************************************//** -Get a table id. The caller must have acquired the dictionary mutex. -@return DB_SUCCESS if found */ -static -ib_err_t -ib_table_get_id_low( -/*================*/ - const char* table_name, /*!< in: table to find */ - ib_id_u64_t* table_id) /*!< out: table id if found */ -{ - dict_table_t* table; - ib_err_t err = DB_TABLE_NOT_FOUND; - - *table_id = 0; - - table = ib_lookup_table_by_name(table_name); - - if (table != NULL) { - *table_id = (table->id); - - err = DB_SUCCESS; - } - - return(err); -} - -/*****************************************************************//** -Create an internal cursor instance. -@return DB_SUCCESS or err code */ -static -ib_err_t -ib_create_cursor( -/*=============*/ - ib_crsr_t* ib_crsr, /*!< out: InnoDB cursor */ - dict_table_t* table, /*!< in: table instance */ - dict_index_t* index, /*!< in: index to use */ - trx_t* trx) /*!< in: transaction */ -{ - mem_heap_t* heap; - ib_cursor_t* cursor; - ib_err_t err = DB_SUCCESS; - - heap = mem_heap_create(sizeof(*cursor) * 2); - - if (heap != NULL) { - row_prebuilt_t* prebuilt; - - cursor = static_cast<ib_cursor_t*>( - mem_heap_zalloc(heap, sizeof(*cursor))); - - cursor->heap = heap; - - cursor->query_heap = mem_heap_create(64); - - if (cursor->query_heap == NULL) { - mem_heap_free(heap); - - return(DB_OUT_OF_MEMORY); - } - - cursor->prebuilt = row_create_prebuilt(table, 0); - - prebuilt = cursor->prebuilt; - - prebuilt->trx = trx; - - cursor->valid_trx = TRUE; - - prebuilt->table = table; - prebuilt->select_lock_type = LOCK_NONE; - prebuilt->innodb_api = TRUE; - - prebuilt->index = index; - - ut_a(prebuilt->index != NULL); - - if (prebuilt->trx != NULL) { - ++prebuilt->trx->n_mysql_tables_in_use; - - prebuilt->index_usable = - row_merge_is_index_usable( - prebuilt->trx, prebuilt->index); - - /* Assign a read view if the transaction does - not have it yet */ - - trx_assign_read_view(prebuilt->trx); - } - - *ib_crsr = (ib_crsr_t) cursor; - } else { - err = DB_OUT_OF_MEMORY; - } - - return(err); -} - -/*****************************************************************//** -Create an internal cursor instance, and set prebuilt->index to index -with supplied index_id. -@return DB_SUCCESS or err code */ -static -ib_err_t -ib_create_cursor_with_index_id( -/*===========================*/ - ib_crsr_t* ib_crsr, /*!< out: InnoDB cursor */ - dict_table_t* table, /*!< in: table instance */ - ib_id_u64_t index_id, /*!< in: index id or 0 */ - trx_t* trx) /*!< in: transaction */ -{ - dict_index_t* index; - - if (index_id != 0) { - mutex_enter(&dict_sys->mutex); - index = dict_index_find_on_id_low(index_id); - mutex_exit(&dict_sys->mutex); - } else { - index = dict_table_get_first_index(table); - } - - return(ib_create_cursor(ib_crsr, table, index, trx)); -} - -/*****************************************************************//** -Open an InnoDB table and return a cursor handle to it. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_open_table_using_id( -/*==========================*/ - ib_id_u64_t table_id, /*!< in: table id of table to open */ - ib_trx_t ib_trx, /*!< in: Current transaction handle - can be NULL */ - ib_crsr_t* ib_crsr) /*!< out,own: InnoDB cursor */ -{ - ib_err_t err; - dict_table_t* table; - const ib_bool_t locked - = ib_trx && ib_schema_lock_is_exclusive(ib_trx); - - table = ib_open_table_by_id(table_id, locked); - - if (table == NULL) { - - return(DB_TABLE_NOT_FOUND); - } - - err = ib_create_cursor_with_index_id(ib_crsr, table, 0, - (trx_t*) ib_trx); - - return(err); -} - -/*****************************************************************//** -Open an InnoDB secondary index cursor and return a cursor handle to it. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_open_index_using_name( -/*============================*/ - ib_crsr_t ib_open_crsr, /*!< in: open/active cursor */ - const char* index_name, /*!< in: secondary index name */ - ib_crsr_t* ib_crsr, /*!< out,own: InnoDB index cursor */ - int* idx_type, /*!< out: index is cluster index */ - ib_id_u64_t* idx_id) /*!< out: index id */ -{ - dict_table_t* table; - dict_index_t* index; - index_id_t index_id = 0; - ib_err_t err = DB_TABLE_NOT_FOUND; - ib_cursor_t* cursor = (ib_cursor_t*) ib_open_crsr; - - *idx_type = 0; - *idx_id = 0; - *ib_crsr = NULL; - - /* We want to increment the ref count, so we do a redundant search. */ - table = dict_table_open_on_id(cursor->prebuilt->table->id, - FALSE, DICT_TABLE_OP_NORMAL); - ut_a(table != NULL); - - /* The first index is always the cluster index. */ - index = dict_table_get_first_index(table); - - /* Traverse the user defined indexes. */ - while (index != NULL) { - if (innobase_strcasecmp(index->name, index_name) == 0) { - index_id = index->id; - *idx_type = index->type; - *idx_id = index_id; - break; - } - index = UT_LIST_GET_NEXT(indexes, index); - } - - if (!index_id) { - dict_table_close(table, FALSE, FALSE); - return(DB_ERROR); - } - - if (index_id > 0) { - ut_ad(index->id == index_id); - err = ib_create_cursor( - ib_crsr, table, index, cursor->prebuilt->trx); - } - - if (*ib_crsr != NULL) { - const ib_cursor_t* cursor; - - cursor = *(ib_cursor_t**) ib_crsr; - - if (cursor->prebuilt->index == NULL) { - err = ib_cursor_close(*ib_crsr); - ut_a(err == DB_SUCCESS); - *ib_crsr = NULL; - } - } - - return(err); -} - -/*****************************************************************//** -Open an InnoDB table and return a cursor handle to it. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_open_table( -/*=================*/ - const char* name, /*!< in: table name */ - ib_trx_t ib_trx, /*!< in: Current transaction handle - can be NULL */ - ib_crsr_t* ib_crsr) /*!< out,own: InnoDB cursor */ -{ - ib_err_t err; - dict_table_t* table; - char* normalized_name; - - normalized_name = static_cast<char*>(ut_malloc_nokey(ut_strlen(name) - + 1)); - ib_normalize_table_name(normalized_name, name); - - if (ib_trx != NULL) { - if (!ib_schema_lock_is_exclusive(ib_trx)) { - table = ib_open_table_by_name(normalized_name); - } else { - /* NOTE: We do not acquire MySQL metadata lock */ - table = ib_lookup_table_by_name(normalized_name); - } - } else { - table = ib_open_table_by_name(normalized_name); - } - - ut_free(normalized_name); - normalized_name = NULL; - - /* It can happen that another thread has created the table but - not the cluster index or it's a broken table definition. Refuse to - open if that's the case. */ - if (table != NULL && dict_table_get_first_index(table) == NULL) { - table = NULL; - } - - if (table != NULL) { - err = ib_create_cursor_with_index_id(ib_crsr, table, 0, - (trx_t*) ib_trx); - } else { - err = DB_TABLE_NOT_FOUND; - } - - return(err); -} - -/** Check the table whether it contains virtual columns. -@param[in] crsr InnoDB Cursor -@return true if table contains virtual column else false. */ -ib_bool_t -ib_is_virtual_table( - ib_crsr_t crsr) -{ - return(crsr->prebuilt->table->n_v_cols > 0); -} - -/********************************************************************//** -Free a context struct for a table handle. */ -static -void -ib_qry_proc_free( -/*=============*/ - ib_qry_proc_t* q_proc) /*!< in, own: qproc struct */ -{ - que_graph_free_recursive(q_proc->grph.ins); - que_graph_free_recursive(q_proc->grph.upd); - que_graph_free_recursive(q_proc->grph.sel); - - memset(q_proc, 0x0, sizeof(*q_proc)); -} - -/*****************************************************************//** -Reset the cursor. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_reset( -/*============*/ - ib_crsr_t ib_crsr) /*!< in/out: InnoDB cursor */ -{ - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - row_prebuilt_t* prebuilt = cursor->prebuilt; - - if (cursor->valid_trx && prebuilt->trx != NULL - && prebuilt->trx->n_mysql_tables_in_use > 0) { - - --prebuilt->trx->n_mysql_tables_in_use; - } - - /* The fields in this data structure are allocated from - the query heap and so need to be reset too. */ - ib_qry_proc_free(&cursor->q_proc); - - mem_heap_empty(cursor->query_heap); - - return(DB_SUCCESS); -} - -/*****************************************************************//** -update the cursor with new transactions and also reset the cursor -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_new_trx( -/*==============*/ - ib_crsr_t ib_crsr, /*!< in/out: InnoDB cursor */ - ib_trx_t ib_trx) /*!< in: transaction */ -{ - ib_err_t err = DB_SUCCESS; - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - trx_t* trx = (trx_t*) ib_trx; - - row_prebuilt_t* prebuilt = cursor->prebuilt; - - row_update_prebuilt_trx(prebuilt, trx); - - cursor->valid_trx = TRUE; - - trx_assign_read_view(prebuilt->trx); - - ib_qry_proc_free(&cursor->q_proc); - - mem_heap_empty(cursor->query_heap); - - return(err); -} - -/*****************************************************************//** -Commit the transaction in a cursor -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_commit_trx( -/*=================*/ - ib_crsr_t ib_crsr, /*!< in/out: InnoDB cursor */ - ib_trx_t ib_trx) /*!< in: transaction */ -{ - ib_err_t err = DB_SUCCESS; - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; -#ifdef UNIV_DEBUG - row_prebuilt_t* prebuilt = cursor->prebuilt; - - ut_ad(prebuilt->trx == (trx_t*) ib_trx); -#endif /* UNIV_DEBUG */ - ib_trx_commit(ib_trx); - cursor->valid_trx = FALSE; - return(err); -} - -/*****************************************************************//** -Close an InnoDB table and free the cursor. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_close( -/*============*/ - ib_crsr_t ib_crsr) /*!< in,own: InnoDB cursor */ -{ - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - row_prebuilt_t* prebuilt; - trx_t* trx; - - if (!cursor) { - return(DB_SUCCESS); - } - - prebuilt = cursor->prebuilt; - trx = prebuilt->trx; - - ib_qry_proc_free(&cursor->q_proc); - - /* The transaction could have been detached from the cursor. */ - if (cursor->valid_trx && trx != NULL - && trx->n_mysql_tables_in_use > 0) { - --trx->n_mysql_tables_in_use; - } - - row_prebuilt_free(prebuilt, FALSE); - cursor->prebuilt = NULL; - - mem_heap_free(cursor->query_heap); - mem_heap_free(cursor->heap); - cursor = NULL; - - return(DB_SUCCESS); -} - -/*****************************************************************//** -Close the table, decrement n_ref_count count. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_close_table( -/*==================*/ - ib_crsr_t ib_crsr) /*!< in,own: InnoDB cursor */ -{ - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - row_prebuilt_t* prebuilt = cursor->prebuilt; - - if (prebuilt && prebuilt->table) { - dict_table_close(prebuilt->table, FALSE, FALSE); - } - - return(DB_SUCCESS); -} -/**********************************************************************//** -Run the insert query and do error handling. -@return DB_SUCCESS or error code */ -UNIV_INLINE -ib_err_t -ib_insert_row_with_lock_retry( -/*==========================*/ - que_thr_t* thr, /*!< in: insert query graph */ - ins_node_t* node, /*!< in: insert node for the query */ - trx_savept_t* savept) /*!< in: savepoint to rollback to - in case of an error */ -{ - trx_t* trx; - ib_err_t err; - ib_bool_t lock_wait; - - trx = thr_get_trx(thr); - - do { - thr->run_node = node; - thr->prev_node = node; - - row_ins_step(thr); - - err = trx->error_state; - - if (err != DB_SUCCESS) { - que_thr_stop_for_mysql(thr); - - thr->lock_state = QUE_THR_LOCK_ROW; - lock_wait = static_cast<ib_bool_t>( - ib_handle_errors(&err, trx, thr, savept)); - thr->lock_state = QUE_THR_LOCK_NOLOCK; - } else { - lock_wait = FALSE; - } - } while (lock_wait); - - return(err); -} - -/*****************************************************************//** -Write a row. -@return DB_SUCCESS or err code */ -static -ib_err_t -ib_execute_insert_query_graph( -/*==========================*/ - dict_table_t* table, /*!< in: table where to insert */ - que_fork_t* ins_graph, /*!< in: query graph */ - ins_node_t* node) /*!< in: insert node */ -{ - trx_t* trx; - que_thr_t* thr; - trx_savept_t savept; - ib_err_t err = DB_SUCCESS; - - trx = ins_graph->trx; - - savept = trx_savept_take(trx); - - thr = que_fork_get_first_thr(ins_graph); - - que_thr_move_to_run_state_for_mysql(thr, trx); - - err = ib_insert_row_with_lock_retry(thr, node, &savept); - - if (err == DB_SUCCESS) { - que_thr_stop_for_mysql_no_error(thr, trx); - - dict_table_n_rows_inc(table); - - if (table->is_system_db) { - srv_stats.n_system_rows_inserted.inc(); - } else { - srv_stats.n_rows_inserted.inc(); - } - } - - trx->op_info = ""; - - return(err); -} - -/*****************************************************************//** -Create an insert query graph node. */ -static -void -ib_insert_query_graph_create( -/*==========================*/ - ib_cursor_t* cursor) /*!< in: Cursor instance */ -{ - ib_qry_proc_t* q_proc = &cursor->q_proc; - ib_qry_node_t* node = &q_proc->node; - trx_t* trx = cursor->prebuilt->trx; - - ut_a(trx_is_started(trx)); - - if (node->ins == NULL) { - dtuple_t* row; - ib_qry_grph_t* grph = &q_proc->grph; - mem_heap_t* heap = cursor->query_heap; - dict_table_t* table = cursor->prebuilt->table; - - node->ins = ins_node_create(INS_DIRECT, table, heap); - - node->ins->select = NULL; - node->ins->values_list = NULL; - - row = dtuple_create(heap, dict_table_get_n_cols(table)); - dict_table_copy_types(row, table); - - ut_ad(!dict_table_have_virtual_index(table)); - - ins_node_set_new_row(node->ins, row); - - grph->ins = static_cast<que_fork_t*>( - que_node_get_parent( - pars_complete_graph_for_exec(node->ins, trx, - heap, NULL))); - - grph->ins->state = QUE_FORK_ACTIVE; - } -} - -/*****************************************************************//** -Insert a row to a table. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_insert_row( -/*=================*/ - ib_crsr_t ib_crsr, /*!< in/out: InnoDB cursor instance */ - const ib_tpl_t ib_tpl) /*!< in: tuple to insert */ -{ - ib_ulint_t i; - ib_qry_node_t* node; - ib_qry_proc_t* q_proc; - ulint n_fields; - dtuple_t* dst_dtuple; - ib_err_t err = DB_SUCCESS; - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - const ib_tuple_t* src_tuple = (const ib_tuple_t*) ib_tpl; - - ib_insert_query_graph_create(cursor); - - ut_ad(src_tuple->type == TPL_TYPE_ROW); - - q_proc = &cursor->q_proc; - node = &q_proc->node; - - node->ins->state = INS_NODE_ALLOC_ROW_ID; - dst_dtuple = node->ins->row; - - n_fields = dtuple_get_n_fields(src_tuple->ptr); - ut_ad(n_fields == dtuple_get_n_fields(dst_dtuple)); - - /* Do a shallow copy of the data fields and check for NULL - constraints on columns. */ - for (i = 0; i < n_fields; i++) { - ulint mtype; - dfield_t* src_field; - dfield_t* dst_field; - - src_field = dtuple_get_nth_field(src_tuple->ptr, i); - - mtype = dtype_get_mtype(dfield_get_type(src_field)); - - /* Don't touch the system columns. */ - if (mtype != DATA_SYS) { - ulint prtype; - - prtype = dtype_get_prtype(dfield_get_type(src_field)); - - if ((prtype & DATA_NOT_NULL) - && dfield_is_null(src_field)) { - - err = DB_DATA_MISMATCH; - break; - } - - dst_field = dtuple_get_nth_field(dst_dtuple, i); - ut_ad(mtype - == dtype_get_mtype(dfield_get_type(dst_field))); - - /* Do a shallow copy. */ - dfield_set_data( - dst_field, src_field->data, src_field->len); - - if (dst_field->len != IB_SQL_NULL) { - UNIV_MEM_ASSERT_RW(dst_field->data, - dst_field->len); - } - } - } - - if (err == DB_SUCCESS) { - err = ib_execute_insert_query_graph( - src_tuple->index->table, q_proc->grph.ins, node->ins); - } - - ib_wake_master_thread(); - - return(err); -} - -/*********************************************************************//** -Gets pointer to a prebuilt update vector used in updates. -@return update vector */ -UNIV_INLINE -upd_t* -ib_update_vector_create( -/*====================*/ - ib_cursor_t* cursor) /*!< in: current cursor */ -{ - trx_t* trx = cursor->prebuilt->trx; - mem_heap_t* heap = cursor->query_heap; - dict_table_t* table = cursor->prebuilt->table; - ib_qry_proc_t* q_proc = &cursor->q_proc; - ib_qry_grph_t* grph = &q_proc->grph; - ib_qry_node_t* node = &q_proc->node; - - ut_a(trx_is_started(trx)); - - if (node->upd == NULL) { - node->upd = static_cast<upd_node_t*>( - row_create_update_node_for_mysql(table, heap)); - } - - ut_ad(!dict_table_have_virtual_index(table)); - - grph->upd = static_cast<que_fork_t*>( - que_node_get_parent( - pars_complete_graph_for_exec(node->upd, trx, - heap, NULL))); - - grph->upd->state = QUE_FORK_ACTIVE; - - return(node->upd->update); -} - -/**********************************************************************//** -Note that a column has changed. */ -static -void -ib_update_col( -/*==========*/ - - ib_cursor_t* cursor, /*!< in: current cursor */ - upd_field_t* upd_field, /*!< in/out: update field */ - ulint col_no, /*!< in: column number */ - dfield_t* dfield) /*!< in: updated dfield */ -{ - ulint data_len; - dict_table_t* table = cursor->prebuilt->table; - dict_index_t* index = dict_table_get_first_index(table); - - data_len = dfield_get_len(dfield); - - if (data_len == UNIV_SQL_NULL) { - dfield_set_null(&upd_field->new_val); - } else { - dfield_copy_data(&upd_field->new_val, dfield); - } - - upd_field->exp = NULL; - - upd_field->orig_len = 0; - - upd_field->field_no = dict_col_get_clust_pos( - &table->cols[col_no], index); -} - -/**********************************************************************//** -Checks which fields have changed in a row and stores the new data -to an update vector. -@return DB_SUCCESS or err code */ -static -ib_err_t -ib_calc_diff( -/*=========*/ - ib_cursor_t* cursor, /*!< in: current cursor */ - upd_t* upd, /*!< in/out: update vector */ - const ib_tuple_t*old_tuple, /*!< in: Old tuple in table */ - const ib_tuple_t*new_tuple) /*!< in: New tuple to update */ -{ - ulint i; - ulint n_changed = 0; - ib_err_t err = DB_SUCCESS; - ulint n_fields = dtuple_get_n_fields(new_tuple->ptr); - - ut_a(old_tuple->type == TPL_TYPE_ROW); - ut_a(new_tuple->type == TPL_TYPE_ROW); - ut_a(old_tuple->index->table == new_tuple->index->table); - - for (i = 0; i < n_fields; ++i) { - ulint mtype; - ulint prtype; - upd_field_t* upd_field; - dfield_t* new_dfield; - dfield_t* old_dfield; - - new_dfield = dtuple_get_nth_field(new_tuple->ptr, i); - old_dfield = dtuple_get_nth_field(old_tuple->ptr, i); - - mtype = dtype_get_mtype(dfield_get_type(old_dfield)); - prtype = dtype_get_prtype(dfield_get_type(old_dfield)); - - /* Skip the system columns */ - if (mtype == DATA_SYS) { - continue; - - } else if ((prtype & DATA_NOT_NULL) - && dfield_is_null(new_dfield)) { - - err = DB_DATA_MISMATCH; - break; - } - - if (dfield_get_len(new_dfield) != dfield_get_len(old_dfield) - || (!dfield_is_null(old_dfield) - && memcmp(dfield_get_data(new_dfield), - dfield_get_data(old_dfield), - dfield_get_len(old_dfield)) != 0)) { - - upd_field = &upd->fields[n_changed]; - - ib_update_col(cursor, upd_field, i, new_dfield); - - ++n_changed; - } - } - - if (err == DB_SUCCESS) { - upd->info_bits = 0; - upd->n_fields = n_changed; - } - - return(err); -} - -/**********************************************************************//** -Run the update query and do error handling. -@return DB_SUCCESS or error code */ -UNIV_INLINE -ib_err_t -ib_update_row_with_lock_retry( -/*==========================*/ - que_thr_t* thr, /*!< in: Update query graph */ - upd_node_t* node, /*!< in: Update node for the query */ - trx_savept_t* savept) /*!< in: savepoint to rollback to - in case of an error */ - -{ - trx_t* trx; - ib_err_t err; - ib_bool_t lock_wait; - - trx = thr_get_trx(thr); - - do { - thr->run_node = node; - thr->prev_node = node; - - row_upd_step(thr); - - err = trx->error_state; - - if (err != DB_SUCCESS) { - que_thr_stop_for_mysql(thr); - - if (err != DB_RECORD_NOT_FOUND) { - thr->lock_state = QUE_THR_LOCK_ROW; - - lock_wait = static_cast<ib_bool_t>( - ib_handle_errors(&err, trx, thr, savept)); - - thr->lock_state = QUE_THR_LOCK_NOLOCK; - } else { - lock_wait = FALSE; - } - } else { - lock_wait = FALSE; - } - } while (lock_wait); - - return(err); -} - -/*********************************************************************//** -Does an update or delete of a row. -@return DB_SUCCESS or err code */ -UNIV_INLINE -ib_err_t -ib_execute_update_query_graph( -/*==========================*/ - ib_cursor_t* cursor, /*!< in: Cursor instance */ - btr_pcur_t* pcur) /*!< in: Btree persistent cursor */ -{ - ib_err_t err; - que_thr_t* thr; - upd_node_t* node; - trx_savept_t savept; - trx_t* trx = cursor->prebuilt->trx; - dict_table_t* table = cursor->prebuilt->table; - ib_qry_proc_t* q_proc = &cursor->q_proc; - - /* The transaction must be running. */ - ut_a(trx_is_started(trx)); - - node = q_proc->node.upd; - - ut_a(dict_index_is_clust(pcur->btr_cur.index)); - btr_pcur_copy_stored_position(node->pcur, pcur); - - ut_a(node->pcur->rel_pos == BTR_PCUR_ON); - - savept = trx_savept_take(trx); - - thr = que_fork_get_first_thr(q_proc->grph.upd); - - node->state = UPD_NODE_UPDATE_CLUSTERED; - - que_thr_move_to_run_state_for_mysql(thr, trx); - - err = ib_update_row_with_lock_retry(thr, node, &savept); - - if (err == DB_SUCCESS) { - - que_thr_stop_for_mysql_no_error(thr, trx); - - if (node->is_delete) { - - dict_table_n_rows_dec(table); - - if (table->is_system_db) { - srv_stats.n_system_rows_deleted.inc(); - } else { - srv_stats.n_rows_deleted.inc(); - } - } else { - if (table->is_system_db) { - srv_stats.n_system_rows_updated.inc(); - } else { - srv_stats.n_rows_updated.inc(); - } - } - - } else if (err == DB_RECORD_NOT_FOUND) { - trx->error_state = DB_SUCCESS; - } - - trx->op_info = ""; - - return(err); -} - -/*****************************************************************//** -Update a row in a table. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_update_row( -/*=================*/ - ib_crsr_t ib_crsr, /*!< in: InnoDB cursor instance */ - const ib_tpl_t ib_old_tpl, /*!< in: Old tuple in table */ - const ib_tpl_t ib_new_tpl) /*!< in: New tuple to update */ -{ - upd_t* upd; - ib_err_t err; - btr_pcur_t* pcur; - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - row_prebuilt_t* prebuilt = cursor->prebuilt; - const ib_tuple_t*old_tuple = (const ib_tuple_t*) ib_old_tpl; - const ib_tuple_t*new_tuple = (const ib_tuple_t*) ib_new_tpl; - - if (dict_index_is_clust(prebuilt->index)) { - pcur = cursor->prebuilt->pcur; - } else if (prebuilt->need_to_access_clustered) { - pcur = cursor->prebuilt->clust_pcur; - } else { - return(DB_ERROR); - } - - ut_a(old_tuple->type == TPL_TYPE_ROW); - ut_a(new_tuple->type == TPL_TYPE_ROW); - - upd = ib_update_vector_create(cursor); - - err = ib_calc_diff(cursor, upd, old_tuple, new_tuple); - - if (err == DB_SUCCESS) { - /* Note that this is not a delete. */ - cursor->q_proc.node.upd->is_delete = FALSE; - - err = ib_execute_update_query_graph(cursor, pcur); - } - - ib_wake_master_thread(); - - return(err); -} - -/**********************************************************************//** -Build the update query graph to delete a row from an index. -@return DB_SUCCESS or err code */ -static -ib_err_t -ib_delete_row( -/*==========*/ - ib_cursor_t* cursor, /*!< in: current cursor */ - btr_pcur_t* pcur, /*!< in: Btree persistent cursor */ - const rec_t* rec) /*!< in: record to delete */ -{ - ulint i; - upd_t* upd; - ib_err_t err; - ib_tuple_t* tuple; - ib_tpl_t ib_tpl; - ulint n_cols; - upd_field_t* upd_field; - ib_bool_t page_format; - dict_table_t* table = cursor->prebuilt->table; - dict_index_t* index = dict_table_get_first_index(table); - - n_cols = dict_index_get_n_ordering_defined_by_user(index); - ib_tpl = ib_key_tuple_new(index, n_cols); - - if (!ib_tpl) { - return(DB_OUT_OF_MEMORY); - } - - tuple = (ib_tuple_t*) ib_tpl; - - upd = ib_update_vector_create(cursor); - - page_format = static_cast<ib_bool_t>( - dict_table_is_comp(index->table)); - ib_read_tuple(rec, page_format, tuple, NULL, NULL); - - upd->n_fields = ib_tuple_get_n_cols(ib_tpl); - - for (i = 0; i < upd->n_fields; ++i) { - dfield_t* dfield; - - upd_field = &upd->fields[i]; - dfield = dtuple_get_nth_field(tuple->ptr, i); - - dfield_copy_data(&upd_field->new_val, dfield); - - upd_field->exp = NULL; - - upd_field->orig_len = 0; - - upd->info_bits = 0; - - upd_field->field_no = dict_col_get_clust_pos( - &table->cols[i], index); - } - - /* Note that this is a delete. */ - cursor->q_proc.node.upd->is_delete = TRUE; - - err = ib_execute_update_query_graph(cursor, pcur); - - ib_tuple_delete(ib_tpl); - - return(err); -} - -/*****************************************************************//** -Delete a row in a table. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_delete_row( -/*=================*/ - ib_crsr_t ib_crsr) /*!< in: InnoDB cursor instance */ -{ - ib_err_t err; - btr_pcur_t* pcur; - dict_index_t* index; - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - row_prebuilt_t* prebuilt = cursor->prebuilt; - - index = dict_table_get_first_index(prebuilt->index->table); - - /* Check whether this is a secondary index cursor */ - if (index != prebuilt->index) { - if (prebuilt->need_to_access_clustered) { - pcur = prebuilt->clust_pcur; - } else { - return(DB_ERROR); - } - } else { - pcur = prebuilt->pcur; - } - - if (ib_btr_cursor_is_positioned(pcur)) { - const rec_t* rec; - ib_bool_t page_format; - mtr_t mtr; - rec_t* copy = NULL; - byte ptr[UNIV_PAGE_SIZE_MAX]; - - page_format = static_cast<ib_bool_t>( - dict_table_is_comp(index->table)); - - mtr_start(&mtr); - - if (btr_pcur_restore_position( - BTR_SEARCH_LEAF, pcur, &mtr)) { - mem_heap_t* heap = NULL; - ulint offsets_[REC_OFFS_NORMAL_SIZE]; - ulint* offsets = offsets_; - - rec_offs_init(offsets_); - - rec = btr_pcur_get_rec(pcur); - - /* Since mtr will be commited, the rec - will not be protected. Make a copy of - the rec. */ - offsets = rec_get_offsets( - rec, index, offsets, ULINT_UNDEFINED, &heap); - ut_ad(rec_offs_size(offsets) < UNIV_PAGE_SIZE_MAX); - copy = rec_copy(ptr, rec, offsets); - } - - mtr_commit(&mtr); - - if (copy && !rec_get_deleted_flag(copy, page_format)) { - err = ib_delete_row(cursor, pcur, copy); - } else { - err = DB_RECORD_NOT_FOUND; - } - } else { - err = DB_RECORD_NOT_FOUND; - } - - ib_wake_master_thread(); - - return(err); -} - -/*****************************************************************//** -Read current row. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_read_row( -/*===============*/ - ib_crsr_t ib_crsr, /*!< in: InnoDB cursor instance */ - ib_tpl_t ib_tpl, /*!< out: read cols into this tuple */ - void** row_buf, /*!< in/out: row buffer */ - ib_ulint_t* row_len) /*!< in/out: row buffer len */ -{ - ib_err_t err; - ib_tuple_t* tuple = (ib_tuple_t*) ib_tpl; - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - - ut_a(trx_is_started(cursor->prebuilt->trx)); - - /* When searching with IB_EXACT_MATCH set, row_search_for_mysql() - will not position the persistent cursor but will copy the record - found into the row cache. It should be the only entry. */ - if (!ib_cursor_is_positioned(ib_crsr) ) { - err = DB_RECORD_NOT_FOUND; - } else { - mtr_t mtr; - btr_pcur_t* pcur; - row_prebuilt_t* prebuilt = cursor->prebuilt; - - if (prebuilt->need_to_access_clustered - && tuple->type == TPL_TYPE_ROW) { - pcur = prebuilt->clust_pcur; - } else { - pcur = prebuilt->pcur; - } - - if (pcur == NULL) { - return(DB_ERROR); - } - - mtr_start(&mtr); - - if (btr_pcur_restore_position(BTR_SEARCH_LEAF, pcur, &mtr)) { - const rec_t* rec; - ib_bool_t page_format; - - page_format = static_cast<ib_bool_t>( - dict_table_is_comp(tuple->index->table)); - rec = btr_pcur_get_rec(pcur); - - if (prebuilt->innodb_api_rec && - prebuilt->innodb_api_rec != rec) { - rec = prebuilt->innodb_api_rec; - } - - if (!rec_get_deleted_flag(rec, page_format)) { - ib_read_tuple(rec, page_format, tuple, - row_buf, (ulint*) row_len); - err = DB_SUCCESS; - } else{ - err = DB_RECORD_NOT_FOUND; - } - - } else { - err = DB_RECORD_NOT_FOUND; - } - - mtr_commit(&mtr); - } - - return(err); -} - -/*****************************************************************//** -Move cursor to the first record in the table. -@return DB_SUCCESS or err code */ -UNIV_INLINE -ib_err_t -ib_cursor_position( -/*===============*/ - ib_cursor_t* cursor, /*!< in: InnoDB cursor instance */ - ib_srch_mode_t mode) /*!< in: Search mode */ -{ - ib_err_t err; - row_prebuilt_t* prebuilt = cursor->prebuilt; - unsigned char* buf; - - buf = static_cast<unsigned char*>(ut_malloc_nokey(UNIV_PAGE_SIZE)); - - /* We want to position at one of the ends, row_search_for_mysql() - uses the search_tuple fields to work out what to do. */ - dtuple_set_n_fields(prebuilt->search_tuple, 0); - - err = static_cast<ib_err_t>(row_search_for_mysql( - buf, static_cast<page_cur_mode_t>(mode), prebuilt, 0, 0)); - - ut_free(buf); - - return(err); -} - -/*****************************************************************//** -Move cursor to the first record in the table. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_first( -/*============*/ - ib_crsr_t ib_crsr) /*!< in: InnoDB cursor instance */ -{ - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - - return(ib_cursor_position(cursor, IB_CUR_G)); -} - -/*****************************************************************//** -Move cursor to the next user record in the table. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_next( -/*===========*/ - ib_crsr_t ib_crsr) /*!< in: InnoDB cursor instance */ -{ - ib_err_t err; - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - row_prebuilt_t* prebuilt = cursor->prebuilt; - byte buf[UNIV_PAGE_SIZE_MAX]; - - /* We want to move to the next record */ - dtuple_set_n_fields(prebuilt->search_tuple, 0); - - err = static_cast<ib_err_t>(row_search_for_mysql( - buf, PAGE_CUR_G, prebuilt, 0, ROW_SEL_NEXT)); - - return(err); -} - -/*****************************************************************//** -Search for key. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_moveto( -/*=============*/ - ib_crsr_t ib_crsr, /*!< in: InnoDB cursor instance */ - ib_tpl_t ib_tpl, /*!< in: Key to search for */ - ib_srch_mode_t ib_srch_mode) /*!< in: search mode */ -{ - ulint i; - ulint n_fields; - ib_err_t err = DB_SUCCESS; - ib_tuple_t* tuple = (ib_tuple_t*) ib_tpl; - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - row_prebuilt_t* prebuilt = cursor->prebuilt; - dtuple_t* search_tuple = prebuilt->search_tuple; - unsigned char* buf; - - ut_a(tuple->type == TPL_TYPE_KEY); - - n_fields = dict_index_get_n_ordering_defined_by_user(prebuilt->index); - - if (n_fields > dtuple_get_n_fields(tuple->ptr)) { - n_fields = dtuple_get_n_fields(tuple->ptr); - } - - dtuple_set_n_fields(search_tuple, n_fields); - dtuple_set_n_fields_cmp(search_tuple, n_fields); - - /* Do a shallow copy */ - for (i = 0; i < n_fields; ++i) { - dfield_copy(dtuple_get_nth_field(search_tuple, i), - dtuple_get_nth_field(tuple->ptr, i)); - } - - ut_a(prebuilt->select_lock_type <= LOCK_NUM); - - prebuilt->innodb_api_rec = NULL; - - buf = static_cast<unsigned char*>(ut_malloc_nokey(UNIV_PAGE_SIZE)); - - err = static_cast<ib_err_t>(row_search_for_mysql( - buf, static_cast<page_cur_mode_t>(ib_srch_mode), prebuilt, - cursor->match_mode, 0)); - - ut_free(buf); - - return(err); -} - -/*****************************************************************//** -Set the cursor search mode. */ -void -ib_cursor_set_match_mode( -/*=====================*/ - ib_crsr_t ib_crsr, /*!< in: Cursor instance */ - ib_match_mode_t match_mode) /*!< in: ib_cursor_moveto match mode */ -{ - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - - cursor->match_mode = match_mode; -} - -/*****************************************************************//** -Get the dfield instance for the column in the tuple. -@return dfield instance in tuple */ -UNIV_INLINE -dfield_t* -ib_col_get_dfield( -/*==============*/ - ib_tuple_t* tuple, /*!< in: tuple instance */ - ulint col_no) /*!< in: col no. in tuple */ -{ - dfield_t* dfield; - - dfield = dtuple_get_nth_field(tuple->ptr, col_no); - - return(dfield); -} - -/*****************************************************************//** -Predicate to check whether a column type contains variable length data. -@return DB_SUCCESS or error code */ -UNIV_INLINE -ib_err_t -ib_col_is_capped( -/*==============*/ - const dtype_t* dtype) /*!< in: column type */ -{ - return(static_cast<ib_err_t>( - (dtype_get_mtype(dtype) == DATA_VARCHAR - || dtype_get_mtype(dtype) == DATA_CHAR - || dtype_get_mtype(dtype) == DATA_MYSQL - || dtype_get_mtype(dtype) == DATA_VARMYSQL - || dtype_get_mtype(dtype) == DATA_FIXBINARY - || dtype_get_mtype(dtype) == DATA_BINARY - || dtype_get_mtype(dtype) == DATA_POINT) - && dtype_get_len(dtype) > 0)); -} - -/*****************************************************************//** -Set a column of the tuple. Make a copy using the tuple's heap. -@return DB_SUCCESS or error code */ -ib_err_t -ib_col_set_value( -/*=============*/ - ib_tpl_t ib_tpl, /*!< in: tuple instance */ - ib_ulint_t col_no, /*!< in: column index in tuple */ - const void* src, /*!< in: data value */ - ib_ulint_t len, /*!< in: data value len */ - ib_bool_t need_cpy) /*!< in: if need memcpy */ -{ - const dtype_t* dtype; - dfield_t* dfield; - void* dst = NULL; - ib_tuple_t* tuple = (ib_tuple_t*) ib_tpl; - ulint col_len; - - dfield = ib_col_get_dfield(tuple, col_no); - - /* User wants to set the column to NULL. */ - if (len == IB_SQL_NULL) { - dfield_set_null(dfield); - return(DB_SUCCESS); - } - - dtype = dfield_get_type(dfield); - col_len = dtype_get_len(dtype); - - /* Not allowed to update system columns. */ - if (dtype_get_mtype(dtype) == DATA_SYS) { - return(DB_DATA_MISMATCH); - } - - dst = dfield_get_data(dfield); - - /* Since TEXT/CLOB also map to DATA_VARCHAR we need to make an - exception. Perhaps we need to set the precise type and check - for that. */ - if (ib_col_is_capped(dtype)) { - - len = ut_min(len, static_cast<ib_ulint_t>(col_len)); - - if (dst == NULL || len > dfield_get_len(dfield)) { - dst = mem_heap_alloc(tuple->heap, col_len); - ut_a(dst != NULL); - } - } else if (dst == NULL || len > dfield_get_len(dfield)) { - dst = mem_heap_alloc(tuple->heap, len); - } - - if (dst == NULL) { - return(DB_OUT_OF_MEMORY); - } - - switch (dtype_get_mtype(dtype)) { - case DATA_INT: { - - if (col_len == len) { - ibool usign; - - usign = dtype_get_prtype(dtype) & DATA_UNSIGNED; - mach_write_int_type(static_cast<byte*>(dst), - static_cast<const byte*>(src), - len, usign); - - } else { - return(DB_DATA_MISMATCH); - } - break; - } - - case DATA_FLOAT: - if (len == sizeof(float)) { - mach_float_write(static_cast<byte*>(dst), *(float*)src); - } else { - return(DB_DATA_MISMATCH); - } - break; - - case DATA_DOUBLE: - if (len == sizeof(double)) { - mach_double_write(static_cast<byte*>(dst), - *(double*)src); - } else { - return(DB_DATA_MISMATCH); - } - break; - - case DATA_SYS: - ut_error; - break; - - case DATA_CHAR: - memcpy(dst, src, len); - memset((byte*) dst + len, 0x20, col_len - len); - len = col_len; - break; - - case DATA_POINT: - memcpy(dst, src, len); - break; - - case DATA_BLOB: - case DATA_VAR_POINT: - case DATA_GEOMETRY: - case DATA_BINARY: - case DATA_DECIMAL: - case DATA_VARCHAR: - case DATA_FIXBINARY: - if (need_cpy) { - memcpy(dst, src, len); - } else { - dfield_set_data(dfield, src, len); - dst = dfield_get_data(dfield); - } - break; - - case DATA_MYSQL: - case DATA_VARMYSQL: { - ulint cset; - CHARSET_INFO* cs; - int error = 0; - ulint true_len = len; - - /* For multi byte character sets we need to - calculate the true length of the data. */ - cset = dtype_get_charset_coll( - dtype_get_prtype(dtype)); - cs = all_charsets[cset]; - if (cs) { - uint pos = (uint)(col_len / cs->mbmaxlen); - - if (len > 0 && cs->mbmaxlen > 1) { - true_len = (ulint) - my_well_formed_length( - cs, - (const char*)src, - (const char*)src + len, - pos, - &error); - - if (true_len < len) { - len = static_cast<ib_ulint_t>(true_len); - } - } - } - - /* All invalid bytes in data need be truncated. - If len == 0, means all bytes of the data is invalid. - In this case, the data will be truncated to empty.*/ - memcpy(dst, src, len); - - /* For DATA_MYSQL, need to pad the unused - space with spaces. */ - if (dtype_get_mtype(dtype) == DATA_MYSQL) { - ulint n_chars; - - if (len < col_len) { - ulint pad_len = col_len - len; - - ut_a(cs != NULL); - ut_a(!(pad_len % cs->mbminlen)); - - cs->cset->fill(cs, (char*)dst + len, - pad_len, - 0x20 /* space */); - } - - /* Why we should do below? See function - row_mysql_store_col_in_innobase_format */ - - ut_a(!(dtype_get_len(dtype) - % dtype_get_mbmaxlen(dtype))); - - n_chars = dtype_get_len(dtype) - / dtype_get_mbmaxlen(dtype); - - /* Strip space padding. */ - while (col_len > n_chars - && ((char*)dst)[col_len - 1] == 0x20) { - col_len--; - } - - len = static_cast<ib_ulint_t>(col_len); - } - break; - } - - default: - ut_error; - } - - if (dst != dfield_get_data(dfield)) { - dfield_set_data(dfield, dst, len); - } else { - dfield_set_len(dfield, len); - } - - return(DB_SUCCESS); -} - -/*****************************************************************//** -Get the size of the data available in a column of the tuple. -@return bytes avail or IB_SQL_NULL */ -ib_ulint_t -ib_col_get_len( -/*===========*/ - ib_tpl_t ib_tpl, /*!< in: tuple instance */ - ib_ulint_t i) /*!< in: column index in tuple */ -{ - const dfield_t* dfield; - ulint data_len; - ib_tuple_t* tuple = (ib_tuple_t*) ib_tpl; - - dfield = ib_col_get_dfield(tuple, i); - - data_len = dfield_get_len(dfield); - - return(static_cast<ib_ulint_t>( - data_len == UNIV_SQL_NULL ? IB_SQL_NULL : data_len)); -} - -/*****************************************************************//** -Copy a column value from the tuple. -@return bytes copied or IB_SQL_NULL */ -UNIV_INLINE -ib_ulint_t -ib_col_copy_value_low( -/*==================*/ - ib_tpl_t ib_tpl, /*!< in: tuple instance */ - ib_ulint_t i, /*!< in: column index in tuple */ - void* dst, /*!< out: copied data value */ - ib_ulint_t len) /*!< in: max data value len to copy */ -{ - const void* data; - const dfield_t* dfield; - ulint data_len; - ib_tuple_t* tuple = (ib_tuple_t*) ib_tpl; - - dfield = ib_col_get_dfield(tuple, i); - - data = dfield_get_data(dfield); - data_len = dfield_get_len(dfield); - - if (data_len != UNIV_SQL_NULL) { - - const dtype_t* dtype = dfield_get_type(dfield); - - switch (dtype_get_mtype(dfield_get_type(dfield))) { - case DATA_INT: { - ibool usign; - uintmax_t ret; - - ut_a(data_len == len); - - usign = dtype_get_prtype(dtype) & DATA_UNSIGNED; - ret = mach_read_int_type(static_cast<const byte*>(data), - data_len, usign); - - if (usign) { - if (len == 1) { - *(ib_i8_t*)dst = (ib_i8_t)ret; - } else if (len == 2) { - *(ib_i16_t*)dst = (ib_i16_t)ret; - } else if (len == 4) { - *(ib_i32_t*)dst = (ib_i32_t)ret; - } else { - *(ib_i64_t*)dst = (ib_i64_t)ret; - } - } else { - if (len == 1) { - *(ib_u8_t*)dst = (ib_i8_t)ret; - } else if (len == 2) { - *(ib_u16_t*)dst = (ib_i16_t)ret; - } else if (len == 4) { - *(ib_u32_t*)dst = (ib_i32_t)ret; - } else { - *(ib_u64_t*)dst = (ib_i64_t)ret; - } - } - - break; - } - case DATA_FLOAT: - if (len == data_len) { - float f; - - ut_a(data_len == sizeof(f)); - f = mach_float_read(static_cast<const byte*>( - data)); - memcpy(dst, &f, sizeof(f)); - } else { - data_len = 0; - } - break; - case DATA_DOUBLE: - if (len == data_len) { - double d; - - ut_a(data_len == sizeof(d)); - d = mach_double_read(static_cast<const byte*>( - data)); - memcpy(dst, &d, sizeof(d)); - } else { - data_len = 0; - } - break; - default: - data_len = ut_min(data_len, len); - memcpy(dst, data, data_len); - } - } else { - data_len = IB_SQL_NULL; - } - - return(static_cast<ib_ulint_t>(data_len)); -} - -/*****************************************************************//** -Copy a column value from the tuple. -@return bytes copied or IB_SQL_NULL */ -ib_ulint_t -ib_col_copy_value( -/*==============*/ - ib_tpl_t ib_tpl, /*!< in: tuple instance */ - ib_ulint_t i, /*!< in: column index in tuple */ - void* dst, /*!< out: copied data value */ - ib_ulint_t len) /*!< in: max data value len to copy */ -{ - return(ib_col_copy_value_low(ib_tpl, i, dst, len)); -} - -/*****************************************************************//** -Get the InnoDB column attribute from the internal column precise type. -@return precise type in api format */ -UNIV_INLINE -ib_col_attr_t -ib_col_get_attr( -/*============*/ - ulint prtype) /*!< in: column definition */ -{ - ib_col_attr_t attr = IB_COL_NONE; - - if (prtype & DATA_UNSIGNED) { - attr = static_cast<ib_col_attr_t>(attr | IB_COL_UNSIGNED); - } - - if (prtype & DATA_NOT_NULL) { - attr = static_cast<ib_col_attr_t>(attr | IB_COL_NOT_NULL); - } - - return(attr); -} - -/*****************************************************************//** -Get a column name from the tuple. -@return name of the column */ -const char* -ib_col_get_name( -/*============*/ - ib_crsr_t ib_crsr, /*!< in: InnoDB cursor instance */ - ib_ulint_t i) /*!< in: column index in tuple */ -{ - const char* name; - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - dict_table_t* table = cursor->prebuilt->table; - dict_col_t* col = dict_table_get_nth_col(table, i); - ulint col_no = dict_col_get_no(col); - - name = dict_table_get_col_name(table, col_no); - - return(name); -} - -/*****************************************************************//** -Get an index field name from the cursor. -@return name of the field */ -const char* -ib_get_idx_field_name( -/*==================*/ - ib_crsr_t ib_crsr, /*!< in: InnoDB cursor instance */ - ib_ulint_t i) /*!< in: column index in tuple */ -{ - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - dict_index_t* index = cursor->prebuilt->index; - dict_field_t* field; - - if (index) { - field = dict_index_get_nth_field(cursor->prebuilt->index, i); - - if (field) { - return(field->name); - } - } - - return(NULL); -} - -/*****************************************************************//** -Get a column type, length and attributes from the tuple. -@return len of column data */ -UNIV_INLINE -ib_ulint_t -ib_col_get_meta_low( -/*================*/ - ib_tpl_t ib_tpl, /*!< in: tuple instance */ - ib_ulint_t i, /*!< in: column index in tuple */ - ib_col_meta_t* ib_col_meta) /*!< out: column meta data */ -{ - ib_u16_t prtype; - const dfield_t* dfield; - ulint data_len; - ib_tuple_t* tuple = (ib_tuple_t*) ib_tpl; - - dfield = ib_col_get_dfield(tuple, i); - - data_len = dfield_get_len(dfield); - - /* We assume 1-1 mapping between the ENUM and internal type codes. */ - ib_col_meta->type = static_cast<ib_col_type_t>( - dtype_get_mtype(dfield_get_type(dfield))); - - ib_col_meta->type_len = static_cast<ib_u32_t>( - dtype_get_len(dfield_get_type(dfield))); - - prtype = (ib_u16_t) dtype_get_prtype(dfield_get_type(dfield)); - - ib_col_meta->attr = ib_col_get_attr(prtype); - ib_col_meta->client_type = prtype & DATA_MYSQL_TYPE_MASK; - - return(static_cast<ib_ulint_t>(data_len)); -} - -/*************************************************************//** -Read a signed int 8 bit column from an InnoDB tuple. */ -UNIV_INLINE -ib_err_t -ib_tuple_check_int( -/*===============*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t i, /*!< in: column number */ - ib_bool_t usign, /*!< in: true if unsigned */ - ulint size) /*!< in: size of integer */ -{ - ib_col_meta_t ib_col_meta; - - ib_col_get_meta_low(ib_tpl, i, &ib_col_meta); - - if (ib_col_meta.type != IB_INT) { - return(DB_DATA_MISMATCH); - } else if (ib_col_meta.type_len == IB_SQL_NULL) { - return(DB_UNDERFLOW); - } else if (ib_col_meta.type_len != size) { - return(DB_DATA_MISMATCH); - } else if ((ib_col_meta.attr & IB_COL_UNSIGNED) && !usign) { - return(DB_DATA_MISMATCH); - } - - return(DB_SUCCESS); -} - -/*************************************************************//** -Read a signed int 8 bit column from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_i8( -/*=============*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t i, /*!< in: column number */ - ib_i8_t* ival) /*!< out: integer value */ -{ - ib_err_t err; - - err = ib_tuple_check_int(ib_tpl, i, IB_FALSE, sizeof(*ival)); - - if (err == DB_SUCCESS) { - ib_col_copy_value_low(ib_tpl, i, ival, sizeof(*ival)); - } - - return(err); -} - -/*************************************************************//** -Read an unsigned int 8 bit column from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_u8( -/*=============*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t i, /*!< in: column number */ - ib_u8_t* ival) /*!< out: integer value */ -{ - ib_err_t err; - - err = ib_tuple_check_int(ib_tpl, i, IB_TRUE, sizeof(*ival)); - - if (err == DB_SUCCESS) { - ib_col_copy_value_low(ib_tpl, i, ival, sizeof(*ival)); - } - - return(err); -} - -/*************************************************************//** -Read a signed int 16 bit column from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_i16( -/*==============*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t i, /*!< in: column number */ - ib_i16_t* ival) /*!< out: integer value */ -{ - ib_err_t err; - - err = ib_tuple_check_int(ib_tpl, i, FALSE, sizeof(*ival)); - - if (err == DB_SUCCESS) { - ib_col_copy_value_low(ib_tpl, i, ival, sizeof(*ival)); - } - - return(err); -} - -/*************************************************************//** -Read an unsigned int 16 bit column from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_u16( -/*==============*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t i, /*!< in: column number */ - ib_u16_t* ival) /*!< out: integer value */ -{ - ib_err_t err; - - err = ib_tuple_check_int(ib_tpl, i, IB_TRUE, sizeof(*ival)); - - if (err == DB_SUCCESS) { - ib_col_copy_value_low(ib_tpl, i, ival, sizeof(*ival)); - } - - return(err); -} - -/*************************************************************//** -Read a signed int 32 bit column from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_i32( -/*==============*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t i, /*!< in: column number */ - ib_i32_t* ival) /*!< out: integer value */ -{ - ib_err_t err; - - err = ib_tuple_check_int(ib_tpl, i, FALSE, sizeof(*ival)); - - if (err == DB_SUCCESS) { - ib_col_copy_value_low(ib_tpl, i, ival, sizeof(*ival)); - } - - return(err); -} - -/*************************************************************//** -Read an unsigned int 32 bit column from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_u32( -/*==============*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t i, /*!< in: column number */ - ib_u32_t* ival) /*!< out: integer value */ -{ - ib_err_t err; - - err = ib_tuple_check_int(ib_tpl, i, IB_TRUE, sizeof(*ival)); - - if (err == DB_SUCCESS) { - ib_col_copy_value_low(ib_tpl, i, ival, sizeof(*ival)); - } - - return(err); -} - -/*************************************************************//** -Read a signed int 64 bit column from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_i64( -/*==============*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t i, /*!< in: column number */ - ib_i64_t* ival) /*!< out: integer value */ -{ - ib_err_t err; - - err = ib_tuple_check_int(ib_tpl, i, FALSE, sizeof(*ival)); - - if (err == DB_SUCCESS) { - ib_col_copy_value_low(ib_tpl, i, ival, sizeof(*ival)); - } - - return(err); -} - -/*************************************************************//** -Read an unsigned int 64 bit column from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_u64( -/*==============*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t i, /*!< in: column number */ - ib_u64_t* ival) /*!< out: integer value */ -{ - ib_err_t err; - - err = ib_tuple_check_int(ib_tpl, i, IB_TRUE, sizeof(*ival)); - - if (err == DB_SUCCESS) { - ib_col_copy_value_low(ib_tpl, i, ival, sizeof(*ival)); - } - - return(err); -} - -/*****************************************************************//** -Get a column value pointer from the tuple. -@return NULL or pointer to buffer */ -const void* -ib_col_get_value( -/*=============*/ - ib_tpl_t ib_tpl, /*!< in: tuple instance */ - ib_ulint_t i) /*!< in: column index in tuple */ -{ - const void* data; - const dfield_t* dfield; - ulint data_len; - ib_tuple_t* tuple = (ib_tuple_t*) ib_tpl; - - dfield = ib_col_get_dfield(tuple, i); - - data = dfield_get_data(dfield); - data_len = dfield_get_len(dfield); - - return(data_len != UNIV_SQL_NULL ? data : NULL); -} - -/*****************************************************************//** -Get a column type, length and attributes from the tuple. -@return len of column data */ -ib_ulint_t -ib_col_get_meta( -/*============*/ - ib_tpl_t ib_tpl, /*!< in: tuple instance */ - ib_ulint_t i, /*!< in: column index in tuple */ - ib_col_meta_t* ib_col_meta) /*!< out: column meta data */ -{ - return(ib_col_get_meta_low(ib_tpl, i, ib_col_meta)); -} - -/*****************************************************************//** -"Clear" or reset an InnoDB tuple. We free the heap and recreate the tuple. -@return new tuple, or NULL */ -ib_tpl_t -ib_tuple_clear( -/*============*/ - ib_tpl_t ib_tpl) /*!< in,own: tuple (will be freed) */ -{ - const dict_index_t* index; - ulint n_cols; - ib_tuple_t* tuple = (ib_tuple_t*) ib_tpl; - ib_tuple_type_t type = tuple->type; - mem_heap_t* heap = tuple->heap; - - index = tuple->index; - n_cols = dtuple_get_n_fields(tuple->ptr); - - mem_heap_empty(heap); - - if (type == TPL_TYPE_ROW) { - return(ib_row_tuple_new_low(index, n_cols, heap)); - } else { - return(ib_key_tuple_new_low(index, n_cols, heap)); - } -} - -/*****************************************************************//** -Create a new cluster key search tuple and copy the contents of the -secondary index key tuple columns that refer to the cluster index record -to the cluster key. It does a deep copy of the column data. -@return DB_SUCCESS or error code */ -ib_err_t -ib_tuple_get_cluster_key( -/*=====================*/ - ib_crsr_t ib_crsr, /*!< in: secondary index cursor */ - ib_tpl_t* ib_dst_tpl, /*!< out,own: destination tuple */ - const ib_tpl_t ib_src_tpl) /*!< in: source tuple */ -{ - ulint i; - ulint n_fields; - ib_err_t err = DB_SUCCESS; - ib_tuple_t* dst_tuple = NULL; - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - ib_tuple_t* src_tuple = (ib_tuple_t*) ib_src_tpl; - dict_index_t* clust_index; - - clust_index = dict_table_get_first_index(cursor->prebuilt->table); - - /* We need to ensure that the src tuple belongs to the same table - as the open cursor and that it's not a tuple for a cluster index. */ - if (src_tuple->type != TPL_TYPE_KEY) { - return(DB_ERROR); - } else if (src_tuple->index->table != cursor->prebuilt->table) { - return(DB_DATA_MISMATCH); - } else if (src_tuple->index == clust_index) { - return(DB_ERROR); - } - - /* Create the cluster index key search tuple. */ - *ib_dst_tpl = ib_clust_search_tuple_create(ib_crsr); - - if (!*ib_dst_tpl) { - return(DB_OUT_OF_MEMORY); - } - - dst_tuple = (ib_tuple_t*) *ib_dst_tpl; - ut_a(dst_tuple->index == clust_index); - - n_fields = dict_index_get_n_unique(dst_tuple->index); - - /* Do a deep copy of the data fields. */ - for (i = 0; i < n_fields; i++) { - ulint pos; - dfield_t* src_field; - dfield_t* dst_field; - - pos = dict_index_get_nth_field_pos( - src_tuple->index, dst_tuple->index, i); - - ut_a(pos != ULINT_UNDEFINED); - - src_field = dtuple_get_nth_field(src_tuple->ptr, pos); - dst_field = dtuple_get_nth_field(dst_tuple->ptr, i); - - if (!dfield_is_null(src_field)) { - UNIV_MEM_ASSERT_RW(src_field->data, src_field->len); - - dst_field->data = mem_heap_dup( - dst_tuple->heap, - src_field->data, - src_field->len); - - dst_field->len = src_field->len; - } else { - dfield_set_null(dst_field); - } - } - - return(err); -} - -/*****************************************************************//** -Create an InnoDB tuple used for index/table search. -@return own: Tuple for current index */ -ib_tpl_t -ib_sec_search_tuple_create( -/*=======================*/ - ib_crsr_t ib_crsr) /*!< in: Cursor instance */ -{ - ulint n_cols; - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - dict_index_t* index = cursor->prebuilt->index; - - n_cols = dict_index_get_n_unique_in_tree(index); - return(ib_key_tuple_new(index, n_cols)); -} - -/*****************************************************************//** -Create an InnoDB tuple used for index/table search. -@return own: Tuple for current index */ -ib_tpl_t -ib_sec_read_tuple_create( -/*=====================*/ - ib_crsr_t ib_crsr) /*!< in: Cursor instance */ -{ - ulint n_cols; - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - dict_index_t* index = cursor->prebuilt->index; - - n_cols = dict_index_get_n_fields(index); - return(ib_row_tuple_new(index, n_cols)); -} - -/*****************************************************************//** -Create an InnoDB tuple used for table key operations. -@return own: Tuple for current table */ -ib_tpl_t -ib_clust_search_tuple_create( -/*=========================*/ - ib_crsr_t ib_crsr) /*!< in: Cursor instance */ -{ - ulint n_cols; - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - dict_index_t* index; - - index = dict_table_get_first_index(cursor->prebuilt->table); - - n_cols = dict_index_get_n_ordering_defined_by_user(index); - return(ib_key_tuple_new(index, n_cols)); -} - -/*****************************************************************//** -Create an InnoDB tuple for table row operations. -@return own: Tuple for current table */ -ib_tpl_t -ib_clust_read_tuple_create( -/*=======================*/ - ib_crsr_t ib_crsr) /*!< in: Cursor instance */ -{ - ulint n_cols; - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - dict_index_t* index; - - index = dict_table_get_first_index(cursor->prebuilt->table); - - n_cols = dict_table_get_n_cols(cursor->prebuilt->table); - return(ib_row_tuple_new(index, n_cols)); -} - -/*****************************************************************//** -Return the number of user columns in the tuple definition. -@return number of user columns */ -ib_ulint_t -ib_tuple_get_n_user_cols( -/*=====================*/ - const ib_tpl_t ib_tpl) /*!< in: Tuple for current table */ -{ - const ib_tuple_t* tuple = (const ib_tuple_t*) ib_tpl; - - if (tuple->type == TPL_TYPE_ROW) { - return(static_cast<ib_ulint_t>( - dict_table_get_n_user_cols(tuple->index->table))); - } - - return(static_cast<ib_ulint_t>( - dict_index_get_n_ordering_defined_by_user(tuple->index))); -} - -/*****************************************************************//** -Return the number of columns in the tuple definition. -@return number of columns */ -ib_ulint_t -ib_tuple_get_n_cols( -/*================*/ - const ib_tpl_t ib_tpl) /*!< in: Tuple for table/index */ -{ - const ib_tuple_t* tuple = (const ib_tuple_t*) ib_tpl; - - return(static_cast<ib_ulint_t>(dtuple_get_n_fields(tuple->ptr))); -} - -/*****************************************************************//** -Destroy an InnoDB tuple. */ -void -ib_tuple_delete( -/*============*/ - ib_tpl_t ib_tpl) /*!< in,own: Tuple instance to delete */ -{ - ib_tuple_t* tuple = (ib_tuple_t*) ib_tpl; - - if (!ib_tpl) { - return; - } - - mem_heap_free(tuple->heap); -} - -/*****************************************************************//** -Get a table id. This function will acquire the dictionary mutex. -@return DB_SUCCESS if found */ -ib_err_t -ib_table_get_id( -/*============*/ - const char* table_name, /*!< in: table to find */ - ib_id_u64_t* table_id) /*!< out: table id if found */ -{ - ib_err_t err; - - dict_mutex_enter_for_mysql(); - - err = ib_table_get_id_low(table_name, table_id); - - dict_mutex_exit_for_mysql(); - - return(err); -} - -/*****************************************************************//** -Check if cursor is positioned. -@return IB_TRUE if positioned */ -ib_bool_t -ib_cursor_is_positioned( -/*====================*/ - const ib_crsr_t ib_crsr) /*!< in: InnoDB cursor instance */ -{ - const ib_cursor_t* cursor = (const ib_cursor_t*) ib_crsr; - row_prebuilt_t* prebuilt = cursor->prebuilt; - - return(ib_btr_cursor_is_positioned(prebuilt->pcur)); -} - - -/*****************************************************************//** -Checks if the data dictionary is latched in exclusive mode. -@return TRUE if exclusive latch */ -ib_bool_t -ib_schema_lock_is_exclusive( -/*========================*/ - const ib_trx_t ib_trx) /*!< in: transaction */ -{ - const trx_t* trx = (const trx_t*) ib_trx; - - return(trx->dict_operation_lock_mode == RW_X_LATCH); -} - -/*****************************************************************//** -Checks if the data dictionary is latched in shared mode. -@return TRUE if shared latch */ -ib_bool_t -ib_schema_lock_is_shared( -/*=====================*/ - const ib_trx_t ib_trx) /*!< in: transaction */ -{ - const trx_t* trx = (const trx_t*) ib_trx; - - return(trx->dict_operation_lock_mode == RW_S_LATCH); -} - -/*****************************************************************//** -Set the Lock an InnoDB cursor/table. -@return DB_SUCCESS or error code */ -ib_err_t -ib_cursor_lock( -/*===========*/ - ib_crsr_t ib_crsr, /*!< in/out: InnoDB cursor */ - ib_lck_mode_t ib_lck_mode) /*!< in: InnoDB lock mode */ -{ - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - row_prebuilt_t* prebuilt = cursor->prebuilt; - trx_t* trx = prebuilt->trx; - dict_table_t* table = prebuilt->table; - - return(ib_trx_lock_table_with_retry( - trx, table, (enum lock_mode) ib_lck_mode)); -} - -/*****************************************************************//** -Set the Lock an InnoDB table using the table id. -@return DB_SUCCESS or error code */ -ib_err_t -ib_table_lock( -/*==========*/ - ib_trx_t ib_trx, /*!< in/out: transaction */ - ib_id_u64_t table_id, /*!< in: table id */ - ib_lck_mode_t ib_lck_mode) /*!< in: InnoDB lock mode */ -{ - ib_err_t err; - que_thr_t* thr; - mem_heap_t* heap; - dict_table_t* table; - ib_qry_proc_t q_proc; - trx_t* trx = (trx_t*) ib_trx; - - ut_ad(trx_is_started(trx)); - - table = ib_open_table_by_id(table_id, FALSE); - - if (table == NULL) { - return(DB_TABLE_NOT_FOUND); - } - - ut_a(ib_lck_mode <= static_cast<ib_lck_mode_t>(LOCK_NUM)); - ut_ad(!dict_table_have_virtual_index(table)); - - heap = mem_heap_create(128); - - q_proc.node.sel = sel_node_create(heap); - - thr = pars_complete_graph_for_exec(q_proc.node.sel, trx, heap, NULL); - - q_proc.grph.sel = static_cast<que_fork_t*>(que_node_get_parent(thr)); - q_proc.grph.sel->state = QUE_FORK_ACTIVE; - - trx->op_info = "setting table lock"; - - ut_a(ib_lck_mode == IB_LOCK_IS || ib_lck_mode == IB_LOCK_IX); - err = static_cast<ib_err_t>( - lock_table(0, table, (enum lock_mode) ib_lck_mode, thr)); - - trx->error_state = err; - - mem_heap_free(heap); - - return(err); -} - -/*****************************************************************//** -Unlock an InnoDB table. -@return DB_SUCCESS or error code */ -ib_err_t -ib_cursor_unlock( -/*=============*/ - ib_crsr_t ib_crsr) /*!< in/out: InnoDB cursor */ -{ - ib_err_t err = DB_SUCCESS; - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - row_prebuilt_t* prebuilt = cursor->prebuilt; - - if (prebuilt->trx->mysql_n_tables_locked > 0) { - --prebuilt->trx->mysql_n_tables_locked; - } else { - err = DB_ERROR; - } - - return(err); -} - -/*****************************************************************//** -Set the Lock mode of the cursor. -@return DB_SUCCESS or error code */ -ib_err_t -ib_cursor_set_lock_mode( -/*====================*/ - ib_crsr_t ib_crsr, /*!< in/out: InnoDB cursor */ - ib_lck_mode_t ib_lck_mode) /*!< in: InnoDB lock mode */ -{ - ib_err_t err = DB_SUCCESS; - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - row_prebuilt_t* prebuilt = cursor->prebuilt; - - ut_a(ib_lck_mode <= static_cast<ib_lck_mode_t>(LOCK_NUM)); - - if (ib_lck_mode == IB_LOCK_X) { - err = ib_cursor_lock(ib_crsr, IB_LOCK_IX); - } else if (ib_lck_mode == IB_LOCK_S) { - err = ib_cursor_lock(ib_crsr, IB_LOCK_IS); - } - - if (err == DB_SUCCESS) { - prebuilt->select_lock_type = (lock_mode) ib_lck_mode; - ut_a(trx_is_started(prebuilt->trx)); - } - - return(err); -} - -/*****************************************************************//** -Set need to access clustered index record. */ -void -ib_cursor_set_cluster_access( -/*=========================*/ - ib_crsr_t ib_crsr) /*!< in/out: InnoDB cursor */ -{ - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - row_prebuilt_t* prebuilt = cursor->prebuilt; - - prebuilt->need_to_access_clustered = TRUE; -} - -/*****************************************************************//** -Inform the cursor that it's the start of an SQL statement. */ -void -ib_cursor_stmt_begin( -/*=================*/ - ib_crsr_t ib_crsr) /*!< in: cursor */ -{ - ib_cursor_t* cursor = (ib_cursor_t*) ib_crsr; - - cursor->prebuilt->sql_stat_start = TRUE; -} - -/*****************************************************************//** -Write a double value to a column. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_write_double( -/*==================*/ - ib_tpl_t ib_tpl, /*!< in/out: tuple to write to */ - int col_no, /*!< in: column number */ - double val) /*!< in: value to write */ -{ - const dfield_t* dfield; - ib_tuple_t* tuple = (ib_tuple_t*) ib_tpl; - - dfield = ib_col_get_dfield(tuple, col_no); - - if (dtype_get_mtype(dfield_get_type(dfield)) == DATA_DOUBLE) { - return(ib_col_set_value(ib_tpl, col_no, - &val, sizeof(val), true)); - } else { - return(DB_DATA_MISMATCH); - } -} - -/*************************************************************//** -Read a double column value from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_double( -/*=================*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t col_no, /*!< in: column number */ - double* dval) /*!< out: double value */ -{ - ib_err_t err; - const dfield_t* dfield; - ib_tuple_t* tuple = (ib_tuple_t*) ib_tpl; - - dfield = ib_col_get_dfield(tuple, col_no); - - if (dtype_get_mtype(dfield_get_type(dfield)) == DATA_DOUBLE) { - ib_col_copy_value_low(ib_tpl, col_no, dval, sizeof(*dval)); - err = DB_SUCCESS; - } else { - err = DB_DATA_MISMATCH; - } - - return(err); -} - -/*****************************************************************//** -Write a float value to a column. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_write_float( -/*=================*/ - ib_tpl_t ib_tpl, /*!< in/out: tuple to write to */ - int col_no, /*!< in: column number */ - float val) /*!< in: value to write */ -{ - const dfield_t* dfield; - ib_tuple_t* tuple = (ib_tuple_t*) ib_tpl; - - dfield = ib_col_get_dfield(tuple, col_no); - - if (dtype_get_mtype(dfield_get_type(dfield)) == DATA_FLOAT) { - return(ib_col_set_value(ib_tpl, col_no, - &val, sizeof(val), true)); - } else { - return(DB_DATA_MISMATCH); - } -} - -/*************************************************************//** -Read a float value from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_float( -/*================*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t col_no, /*!< in: column number */ - float* fval) /*!< out: float value */ -{ - ib_err_t err; - const dfield_t* dfield; - ib_tuple_t* tuple = (ib_tuple_t*) ib_tpl; - - dfield = ib_col_get_dfield(tuple, col_no); - - if (dtype_get_mtype(dfield_get_type(dfield)) == DATA_FLOAT) { - ib_col_copy_value_low(ib_tpl, col_no, fval, sizeof(*fval)); - err = DB_SUCCESS; - } else { - err = DB_DATA_MISMATCH; - } - - return(err); -} - -/*****************************************************************//** -Truncate a table. The cursor handle will be closed and set to NULL -on success. -@return DB_SUCCESS or error code */ -ib_err_t -ib_cursor_truncate( -/*===============*/ - ib_crsr_t* ib_crsr, /*!< in/out: cursor for table - to truncate */ - ib_id_u64_t* table_id) /*!< out: new table id */ -{ - ib_err_t err; - ib_cursor_t* cursor = *(ib_cursor_t**) ib_crsr; - row_prebuilt_t* prebuilt = cursor->prebuilt; - - *table_id = 0; - - err = ib_cursor_lock(*ib_crsr, IB_LOCK_X); - - if (err == DB_SUCCESS) { - trx_t* trx; - dict_table_t* table = prebuilt->table; - - /* We are going to free the cursor and the prebuilt. Store - the transaction handle locally. */ - trx = prebuilt->trx; - err = ib_cursor_close(*ib_crsr); - ut_a(err == DB_SUCCESS); - - *ib_crsr = NULL; - - /* A temp go around for assertion in trx_start_for_ddl_low - we already start the trx */ - if (trx->state == TRX_STATE_ACTIVE) { -#ifdef UNIV_DEBUG - trx->start_file = 0; -#endif /* UNIV_DEBUG */ - trx->dict_operation = TRX_DICT_OP_TABLE; - } - - /* This function currently commits the transaction - on success. */ - err = static_cast<ib_err_t>( - row_truncate_table_for_mysql(table, trx)); - - if (err == DB_SUCCESS) { - *table_id = (table->id); - } - } - - return(err); -} - -/*****************************************************************//** -Truncate a table. -@return DB_SUCCESS or error code */ -ib_err_t -ib_table_truncate( -/*==============*/ - const char* table_name, /*!< in: table name */ - ib_id_u64_t* table_id) /*!< out: new table id */ -{ - ib_err_t err; - dict_table_t* table; - ib_err_t trunc_err; - ib_trx_t ib_trx = NULL; - ib_crsr_t ib_crsr = NULL; - ib_ulint_t memcached_sync = 0; - - ib_trx = ib_trx_begin(IB_TRX_SERIALIZABLE, true, false); - - dict_mutex_enter_for_mysql(); - - table = dict_table_open_on_name(table_name, TRUE, FALSE, - DICT_ERR_IGNORE_NONE); - - if (table != NULL && dict_table_get_first_index(table)) { - err = ib_create_cursor_with_index_id(&ib_crsr, table, 0, - (trx_t*) ib_trx); - } else { - err = DB_TABLE_NOT_FOUND; - } - - /* Remember the memcached_sync_count and set it to 0, so the - truncate can be executed. */ - if (table != NULL && err == DB_SUCCESS) { - memcached_sync = static_cast<ib_ulint_t>( - table->memcached_sync_count); - table->memcached_sync_count = 0; - } - - dict_mutex_exit_for_mysql(); - - if (err == DB_SUCCESS) { - trunc_err = ib_cursor_truncate(&ib_crsr, table_id); - ut_a(err == DB_SUCCESS); - } else { - trunc_err = err; - } - - if (ib_crsr != NULL) { - err = ib_cursor_close(ib_crsr); - ut_a(err == DB_SUCCESS); - } - - if (trunc_err == DB_SUCCESS) { - ut_a(!trx_is_started(static_cast<trx_t*>(ib_trx))); - } else { - err = ib_trx_rollback(ib_trx); - ut_a(err == DB_SUCCESS); - } - - err = ib_trx_release(ib_trx); - ut_a(err == DB_SUCCESS); - - /* Set the memcached_sync_count back. */ - if (table != NULL && memcached_sync != 0) { - dict_mutex_enter_for_mysql(); - - table->memcached_sync_count = memcached_sync; - - dict_mutex_exit_for_mysql(); - } - - return(trunc_err); -} - -/*****************************************************************//** -Return isolation configuration set by "innodb_api_trx_level" -@return trx isolation level*/ -ib_trx_level_t -ib_cfg_trx_level() -/*==============*/ -{ - return(static_cast<ib_trx_level_t>(ib_trx_level_setting)); -} - -/*****************************************************************//** -Return configure value for background commit interval (in seconds) -@return background commit interval (in seconds) */ -ib_ulint_t -ib_cfg_bk_commit_interval() -/*=======================*/ -{ - return(static_cast<ib_ulint_t>(ib_bk_commit_interval)); -} - -/*****************************************************************//** -Get generic configure status -@return configure status*/ -int -ib_cfg_get_cfg() -/*============*/ -{ - int cfg_status; - - cfg_status = (ib_binlog_enabled) ? IB_CFG_BINLOG_ENABLED : 0; - - if (ib_mdl_enabled) { - cfg_status |= IB_CFG_MDL_ENABLED; - } - - if (ib_disable_row_lock) { - cfg_status |= IB_CFG_DISABLE_ROWLOCK; - } - - return(cfg_status); -} - -/*****************************************************************//** -Wrapper of ut_strerr() which converts an InnoDB error number to a -human readable text message. -@return string, describing the error */ -const char* -ib_ut_strerr( -/*=========*/ - ib_err_t num) /*!< in: error number */ -{ - return(ut_strerr(num)); -} - -/*****************************************************************//** -Increase/decrease the memcached sync count of table to sync memcached -DML with SQL DDLs. -@return DB_SUCCESS or error number */ -ib_err_t -ib_cursor_set_memcached_sync( -/*=========================*/ - ib_crsr_t ib_crsr, /*!< in: cursor */ - ib_bool_t flag) /*!< in: true for increase */ -{ - const ib_cursor_t* cursor = (const ib_cursor_t*) ib_crsr; - row_prebuilt_t* prebuilt = cursor->prebuilt; - dict_table_t* table = prebuilt->table; - ib_err_t err = DB_SUCCESS; - - if (table != NULL) { - /* If memcached_sync_count is -1, means table is - doing DDL, we just return error. */ - if (table->memcached_sync_count == DICT_TABLE_IN_DDL) { - return(DB_ERROR); - } - - if (flag) { - my_atomic_addlint(&table->memcached_sync_count, 1); - } else { - my_atomic_addlint(&table->memcached_sync_count, -1); - ut_a(table->memcached_sync_count >= 0); - } - } else { - err = DB_TABLE_NOT_FOUND; - } - - return(err); -} diff --git a/storage/innobase/api/api0misc.cc b/storage/innobase/api/api0misc.cc deleted file mode 100644 index 3864e4b9a7f..00000000000 --- a/storage/innobase/api/api0misc.cc +++ /dev/null @@ -1,136 +0,0 @@ -/***************************************************************************** - -Copyright (c) 2008, 2015, Oracle and/or its affiliates. All Rights Reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/**************************************************//** -@file api/api0misc.cc -InnoDB Native API - -2008-08-01 Created by Sunny Bains -3/20/2011 Jimmy Yang extracted from Embedded InnoDB -*******************************************************/ - -#include "ha_prototypes.h" - -#include "api0misc.h" -#include "trx0roll.h" -#include "srv0srv.h" -#include "dict0mem.h" -#include "dict0dict.h" -#include "pars0pars.h" -#include "row0sel.h" -#include "lock0lock.h" - -/*********************************************************************//** -Sets a lock on a table. -@return error code or DB_SUCCESS */ -dberr_t -ib_trx_lock_table_with_retry( -/*=========================*/ - trx_t* trx, /*!< in/out: transaction */ - dict_table_t* table, /*!< in: table to lock */ - enum lock_mode mode) /*!< in: LOCK_X or LOCK_S */ -{ - trx->op_info = "setting table lock"; - - return(lock_table_for_trx(table, trx, mode)); -} -/****************************************************************//** -Handles user errors and lock waits detected by the database engine. -@return TRUE if it was a lock wait and we should continue running -the query thread */ -ibool -ib_handle_errors( -/*=============*/ - dberr_t* new_err,/*!< out: possible new error encountered in - lock wait, or if no new error, the value - of trx->error_state at the entry of this - function */ - trx_t* trx, /*!< in: transaction */ - que_thr_t* thr, /*!< in: query thread */ - trx_savept_t* savept) /*!< in: savepoint or NULL */ -{ - dberr_t err; -handle_new_error: - err = trx->error_state; - - ut_a(err != DB_SUCCESS); - - trx->error_state = DB_SUCCESS; - - switch (err) { - case DB_LOCK_WAIT_TIMEOUT: - trx_rollback_for_mysql(trx); - break; - /* fall through */ - case DB_DUPLICATE_KEY: - case DB_FOREIGN_DUPLICATE_KEY: - case DB_TOO_BIG_RECORD: - case DB_ROW_IS_REFERENCED: - case DB_NO_REFERENCED_ROW: - case DB_CANNOT_ADD_CONSTRAINT: - case DB_TOO_MANY_CONCURRENT_TRXS: - case DB_OUT_OF_FILE_SPACE: - if (savept) { - /* Roll back the latest, possibly incomplete - insertion or update */ - - trx_rollback_to_savepoint(trx, savept); - } - break; - case DB_LOCK_WAIT: - lock_wait_suspend_thread(thr); - - if (trx->error_state != DB_SUCCESS) { - que_thr_stop_for_mysql(thr); - - goto handle_new_error; - } - - *new_err = err; - - return(TRUE); /* Operation needs to be retried. */ - - case DB_DEADLOCK: - case DB_LOCK_TABLE_FULL: - /* Roll back the whole transaction; this resolution was added - to version 3.23.43 */ - - trx_rollback_for_mysql(trx); - break; - - case DB_MUST_GET_MORE_FILE_SPACE: - - ut_error; - - case DB_CORRUPTION: - case DB_FOREIGN_EXCEED_MAX_CASCADE: - break; - default: - ut_error; - } - - if (trx->error_state != DB_SUCCESS) { - *new_err = trx->error_state; - } else { - *new_err = err; - } - - trx->error_state = DB_SUCCESS; - - return(FALSE); -} diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index 255788229e4..4bebb9ee7dd 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -37,7 +37,6 @@ Created 6/2/1994 Heikki Tuuri #include "page0zip.h" #include "gis0rtree.h" -#ifndef UNIV_HOTBACKUP #include "btr0cur.h" #include "btr0sea.h" #include "btr0pcur.h" @@ -50,6 +49,7 @@ Created 6/2/1994 Heikki Tuuri #include "gis0geo.h" #include "ut0new.h" #include "dict0boot.h" +#include "row0sel.h" /* row_search_max_autoinc() */ /**************************************************************//** Checks if the page in the cursor can be merged with given page. @@ -63,8 +63,6 @@ btr_can_merge_with_page( buf_block_t** merge_block, /*!< out: the merge block */ mtr_t* mtr); /*!< in: mini-transaction */ -#endif /* UNIV_HOTBACKUP */ - /**************************************************************//** Report that an index page is corrupted. */ void @@ -79,7 +77,6 @@ btr_corruption_report( << " of table " << index->table->name; } -#ifndef UNIV_HOTBACKUP /* Latching strategy of the InnoDB B-tree -------------------------------------- @@ -173,16 +170,17 @@ btr_root_block_get( buf_block_t* block = btr_block_get(page_id, page_size, mode, index, mtr); - if (!block) { - index->table->is_encrypted = TRUE; - index->table->corrupted = FALSE; - - ib_push_warning(index->table->thd, DB_DECRYPTION_FAILED, - "Table %s in tablespace %lu is encrypted but encryption service or" - " used key_id is not available. " - " Can't continue reading table.", - index->table->name, space); + if (index && index->table) { + index->table->is_encrypted = TRUE; + index->table->corrupted = FALSE; + + ib_push_warning(index->table->thd, DB_DECRYPTION_FAILED, + "Table %s in tablespace %lu is encrypted but encryption service or" + " used key_id is not available. " + " Can't continue reading table.", + index->table->name, space); + } return NULL; } @@ -242,8 +240,7 @@ btr_height_get( || mtr_memo_contains_flagged(mtr, dict_index_get_lock(index), MTR_MEMO_S_LOCK | MTR_MEMO_X_LOCK - | MTR_MEMO_SX_LOCK) - || dict_table_is_intrinsic(index->table)); + | MTR_MEMO_SX_LOCK)); /* S latches the page */ root_block = btr_root_block_get(index, RW_S_LATCH, mtr); @@ -338,12 +335,9 @@ btr_root_adjust_on_import( } else { /* Check that the table flags and the tablespace flags match. */ - ulint flags = dict_tf_to_fsp_flags( - table->flags, - false, - dict_table_is_encrypted(table)); + ulint flags = dict_tf_to_fsp_flags(table->flags); ulint fsp_flags = fil_space_get_flags(table->space); - err = fsp_flags_are_equal(flags, fsp_flags) + err = flags == fsp_flags ? DB_SUCCESS : DB_CORRUPTION; } } else { @@ -560,8 +554,7 @@ btr_get_size( ut_ad(srv_read_only_mode || mtr_memo_contains(mtr, dict_index_get_lock(index), - MTR_MEMO_S_LOCK) - || dict_table_is_intrinsic(index->table)); + MTR_MEMO_S_LOCK)); if (index->page == FIL_NULL || dict_index_is_online_ddl(index) @@ -917,8 +910,7 @@ btr_page_get_father_node_ptr_func( ut_ad(srv_read_only_mode || mtr_memo_contains_flagged(mtr, dict_index_get_lock(index), MTR_MEMO_X_LOCK - | MTR_MEMO_SX_LOCK) - || dict_table_is_intrinsic(index->table)); + | MTR_MEMO_SX_LOCK)); ut_ad(dict_index_get_page(index) != page_no); @@ -929,17 +921,11 @@ btr_page_get_father_node_ptr_func( tuple = dict_index_build_node_ptr(index, user_rec, 0, heap, level); dberr_t err = DB_SUCCESS; - - if (dict_table_is_intrinsic(index->table)) { - err = btr_cur_search_to_nth_level_with_no_latch( - index, level + 1, tuple, PAGE_CUR_LE, cursor, - file, line, mtr); - } else { - err = btr_cur_search_to_nth_level( - index, level + 1, tuple, - PAGE_CUR_LE, latch_mode, cursor, 0, - file, line, mtr); - } + + err = btr_cur_search_to_nth_level( + index, level + 1, tuple, + PAGE_CUR_LE, latch_mode, cursor, 0, + file, line, mtr); if (err != DB_SUCCESS) { ib::warn() << " Error code: " << err @@ -1106,7 +1092,7 @@ btr_free_root_check( index_id_t index_id, mtr_t* mtr) { - ut_ad(page_id.space() != srv_tmp_space.space_id()); + ut_ad(page_id.space() != SRV_TMP_SPACE_ID); ut_ad(index_id != BTR_FREED_INDEX_ID); buf_block_t* block = buf_page_get( @@ -1328,6 +1314,11 @@ leaf_loop: page_t* root = block->frame; + if (!root) { + mtr_commit(&mtr); + return; + } + #ifdef UNIV_BTR_DEBUG ut_a(btr_root_fseg_validate(FIL_PAGE_DATA + PAGE_BTR_SEG_LEAF + root, block->page.id.space())); @@ -1408,13 +1399,125 @@ btr_free( buf_block_t* block = buf_page_get( page_id, page_size, RW_X_LATCH, &mtr); - ut_ad(page_is_root(block->frame)); + if (block) { + ut_ad(page_is_root(block->frame)); - btr_free_but_not_root(block, MTR_LOG_NO_REDO); - btr_free_root(block, &mtr); + btr_free_but_not_root(block, MTR_LOG_NO_REDO); + btr_free_root(block, &mtr); + } mtr.commit(); } -#endif /* !UNIV_HOTBACKUP */ + +/** Read the last used AUTO_INCREMENT value from PAGE_ROOT_AUTO_INC. +@param[in,out] index clustered index +@return the last used AUTO_INCREMENT value +@retval 0 on error or if no AUTO_INCREMENT value was used yet */ +ib_uint64_t +btr_read_autoinc(dict_index_t* index) +{ + ut_ad(dict_index_is_clust(index)); + ut_ad(index->table->persistent_autoinc); + ut_ad(!dict_table_is_temporary(index->table)); + + if (fil_space_t* space = fil_space_acquire(index->space)) { + mtr_t mtr; + mtr.start(); + ib_uint64_t autoinc; + if (buf_block_t* block = buf_page_get( + page_id_t(index->space, index->page), + page_size_t(space->flags), + RW_S_LATCH, &mtr)) { + autoinc = page_get_autoinc(block->frame); + } else { + autoinc = 0; + } + mtr.commit(); + fil_space_release(space); + return(autoinc); + } + + return(0); +} + +/** Read the last used AUTO_INCREMENT value from PAGE_ROOT_AUTO_INC, +or fall back to MAX(auto_increment_column). +@param[in] table table containing an AUTO_INCREMENT column +@param[in] col_no index of the AUTO_INCREMENT column +@return the AUTO_INCREMENT value +@retval 0 on error or if no AUTO_INCREMENT value was used yet */ +ib_uint64_t +btr_read_autoinc_with_fallback(const dict_table_t* table, unsigned col_no) +{ + ut_ad(table->persistent_autoinc); + ut_ad(!dict_table_is_temporary(table)); + + dict_index_t* index = dict_table_get_first_index(table); + + if (index == NULL) { + } else if (fil_space_t* space = fil_space_acquire(index->space)) { + mtr_t mtr; + mtr.start(); + buf_block_t* block = buf_page_get( + page_id_t(index->space, index->page), + page_size_t(space->flags), + RW_S_LATCH, &mtr); + + ib_uint64_t autoinc = block + ? page_get_autoinc(block->frame) : 0; + const bool retry = block && autoinc == 0 + && !page_is_empty(block->frame); + mtr.commit(); + fil_space_release(space); + + if (retry) { + /* This should be an old data file where + PAGE_ROOT_AUTO_INC was initialized to 0. + Fall back to reading MAX(autoinc_col). + There should be an index on it. */ + const dict_col_t* autoinc_col + = dict_table_get_nth_col(table, col_no); + while (index != NULL + && index->fields[0].col != autoinc_col) { + index = dict_table_get_next_index(index); + } + + if (index != NULL && index->space == space->id) { + autoinc = row_search_max_autoinc(index); + } + } + + return(autoinc); + } + + return(0); +} + +/** Write the next available AUTO_INCREMENT value to PAGE_ROOT_AUTO_INC. +@param[in,out] index clustered index +@param[in] autoinc the AUTO_INCREMENT value +@param[in] reset whether to reset the AUTO_INCREMENT + to a possibly smaller value than currently + exists in the page */ +void +btr_write_autoinc(dict_index_t* index, ib_uint64_t autoinc, bool reset) +{ + ut_ad(dict_index_is_clust(index)); + ut_ad(index->table->persistent_autoinc); + ut_ad(!dict_table_is_temporary(index->table)); + + if (fil_space_t* space = fil_space_acquire(index->space)) { + mtr_t mtr; + mtr.start(); + mtr.set_named_space(space); + page_set_autoinc(buf_page_get( + page_id_t(index->space, index->page), + page_size_t(space->flags), + RW_SX_LATCH, &mtr), + index, autoinc, &mtr, reset); + mtr.commit(); + fil_space_release(space); + } +} /*************************************************************//** Reorganizes an index page. @@ -1442,9 +1545,7 @@ btr_page_reorganize_low( mtr_t* mtr) /*!< in/out: mini-transaction */ { buf_block_t* block = page_cur_get_block(cursor); -#ifndef UNIV_HOTBACKUP buf_pool_t* buf_pool = buf_pool_from_bpage(&block->page); -#endif /* !UNIV_HOTBACKUP */ page_t* page = buf_block_get_frame(block); page_zip_des_t* page_zip = buf_block_get_page_zip(block); buf_block_t* temp_block; @@ -1469,12 +1570,7 @@ btr_page_reorganize_low( /* Turn logging off */ mtr_log_t log_mode = mtr_set_log_mode(mtr, MTR_LOG_NONE); -#ifndef UNIV_HOTBACKUP temp_block = buf_block_alloc(buf_pool); -#else /* !UNIV_HOTBACKUP */ - ut_ad(block == back_block1); - temp_block = back_block2; -#endif /* !UNIV_HOTBACKUP */ temp_page = temp_block->frame; MONITOR_INC(MONITOR_INDEX_REORG_ATTEMPTS); @@ -1487,11 +1583,9 @@ btr_page_reorganize_low( /* Copy the old page to temporary space */ buf_frame_copy(temp_page, page); -#ifndef UNIV_HOTBACKUP if (!recovery) { btr_search_drop_page_hash_index(block); } -#endif /* !UNIV_HOTBACKUP */ /* Save the cursor position. */ pos = page_rec_get_n_recs_before(page_cur_get_rec(cursor)); @@ -1508,21 +1602,28 @@ btr_page_reorganize_low( page_get_infimum_rec(temp_page), index, mtr); - /* Multiple transactions cannot simultaneously operate on the - same temp-table in parallel. - max_trx_id is ignored for temp tables because it not required - for MVCC. */ - if (dict_index_is_sec_or_ibuf(index) - && page_is_leaf(page) - && !dict_table_is_temporary(index->table)) { - /* Copy max trx id to recreated page */ - trx_id_t max_trx_id = page_get_max_trx_id(temp_page); - page_set_max_trx_id(block, NULL, max_trx_id, mtr); - /* In crash recovery, dict_index_is_sec_or_ibuf() always - holds, even for clustered indexes. max_trx_id is - unused in clustered index pages. */ - ut_ad(max_trx_id != 0 || recovery); - } + /* Copy the PAGE_MAX_TRX_ID or PAGE_ROOT_AUTO_INC. */ + memcpy(page + (PAGE_HEADER + PAGE_MAX_TRX_ID), + temp_page + (PAGE_HEADER + PAGE_MAX_TRX_ID), 8); + /* PAGE_MAX_TRX_ID is unused in clustered index pages, + non-leaf pages, and in temporary tables. It was always + zero-initialized in page_create() in all InnoDB versions. + PAGE_MAX_TRX_ID must be nonzero on dict_index_is_sec_or_ibuf() + leaf pages. + + During redo log apply, dict_index_is_sec_or_ibuf() always + holds, even for clustered indexes. */ + ut_ad(recovery || dict_table_is_temporary(index->table) + || !page_is_leaf(temp_page) + || !dict_index_is_sec_or_ibuf(index) + || page_get_max_trx_id(page) != 0); + /* PAGE_MAX_TRX_ID must be zero on non-leaf pages other than + clustered index root pages. */ + ut_ad(recovery + || page_get_max_trx_id(page) == 0 + || (dict_index_is_sec_or_ibuf(index) + ? page_is_leaf(temp_page) + : page_is_root(temp_page))); /* If innodb_log_compressed_pages is ON, page reorganize should log the compressed page image.*/ @@ -1559,13 +1660,10 @@ btr_page_reorganize_low( goto func_exit; } -#ifndef UNIV_HOTBACKUP - /* No locks are acquried for intrinsic tables. */ if (!recovery && !dict_table_is_locking_disabled(index->table)) { /* Update the record lock bitmaps */ lock_move_reorganize_page(block, temp_block); } -#endif /* !UNIV_HOTBACKUP */ data_size2 = page_get_data_size(page); max_ins_size2 = page_get_max_insert_size_after_reorganize(page, 1); @@ -1594,14 +1692,11 @@ func_exit: #ifdef UNIV_ZIP_DEBUG ut_a(!page_zip || page_zip_validate(page_zip, page, index)); #endif /* UNIV_ZIP_DEBUG */ -#ifndef UNIV_HOTBACKUP buf_block_free(temp_block); -#endif /* !UNIV_HOTBACKUP */ /* Restore logging mode */ mtr_set_log_mode(mtr, log_mode); -#ifndef UNIV_HOTBACKUP if (success) { mlog_id_t type; byte* log_ptr; @@ -1630,7 +1725,6 @@ func_exit: MONITOR_INC(MONITOR_INDEX_REORG_SUCCESSFUL); } -#endif /* !UNIV_HOTBACKUP */ return(success); } @@ -1666,7 +1760,6 @@ btr_page_reorganize_block( return(btr_page_reorganize_low(recovery, z_level, &cur, index, mtr)); } -#ifndef UNIV_HOTBACKUP /*************************************************************//** Reorganizes an index page. @@ -1688,7 +1781,6 @@ btr_page_reorganize( return(btr_page_reorganize_low(false, page_zip_level, cursor, index, mtr)); } -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses a redo log record of reorganizing a page. @@ -1732,7 +1824,6 @@ btr_parse_page_reorganize( return(ptr); } -#ifndef UNIV_HOTBACKUP /*************************************************************//** Empties an index page. @see btr_page_create(). */ static @@ -1758,12 +1849,23 @@ btr_page_empty( /* Recreate the page: note that global data on page (possible segment headers, next page-field, etc.) is preserved intact */ + /* Preserve PAGE_ROOT_AUTO_INC when creating a clustered index + root page. */ + const ib_uint64_t autoinc + = dict_index_is_clust(index) && page_is_root(page) + ? page_get_autoinc(page) + : 0; + if (page_zip) { - page_create_zip(block, index, level, 0, NULL, mtr); + page_create_zip(block, index, level, autoinc, NULL, mtr); } else { page_create(block, mtr, dict_table_is_comp(index->table), dict_index_is_spatial(index)); btr_page_set_level(page, NULL, level, mtr); + if (autoinc) { + mlog_write_ull(PAGE_HEADER + PAGE_MAX_TRX_ID + page, + autoinc, mtr); + } } } @@ -1824,8 +1926,7 @@ btr_root_raise_and_insert( #endif /* UNIV_BTR_DEBUG */ ut_ad(mtr_memo_contains_flagged(mtr, dict_index_get_lock(index), MTR_MEMO_X_LOCK - | MTR_MEMO_SX_LOCK) - || dict_table_is_intrinsic(index->table)); + | MTR_MEMO_SX_LOCK)); ut_ad(mtr_is_block_fix( mtr, root_block, MTR_MEMO_PAGE_X_FIX, index->table)); @@ -2311,17 +2412,10 @@ btr_insert_on_non_leaf_level_func( ut_ad(level > 0); if (!dict_index_is_spatial(index)) { - dberr_t err = DB_SUCCESS; - if (dict_table_is_intrinsic(index->table)) { - err = btr_cur_search_to_nth_level_with_no_latch( - index, level, tuple, PAGE_CUR_LE, &cursor, - __FILE__, __LINE__, mtr); - } else { - err = btr_cur_search_to_nth_level( - index, level, tuple, PAGE_CUR_LE, - BTR_CONT_MODIFY_TREE, - &cursor, 0, file, line, mtr); - } + dberr_t err = btr_cur_search_to_nth_level( + index, level, tuple, PAGE_CUR_LE, + BTR_CONT_MODIFY_TREE, + &cursor, 0, file, line, mtr); if (err != DB_SUCCESS) { ib::warn() << " Error code: " << err @@ -2593,10 +2687,9 @@ btr_insert_into_right_sibling( page_t* page = buf_block_get_frame(block); ulint next_page_no = btr_page_get_next(page, mtr); - ut_ad(dict_table_is_intrinsic(cursor->index->table) - || mtr_memo_contains_flagged( - mtr, dict_index_get_lock(cursor->index), - MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK)); + ut_ad(mtr_memo_contains_flagged( + mtr, dict_index_get_lock(cursor->index), + MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK)); ut_ad(mtr_is_block_fix( mtr, block, MTR_MEMO_PAGE_X_FIX, cursor->index->table)); ut_ad(heap); @@ -2769,14 +2862,12 @@ func_start: ut_ad(mtr_memo_contains_flagged(mtr, dict_index_get_lock(cursor->index), - MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK) - || dict_table_is_intrinsic(cursor->index->table)); + MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK)); ut_ad(!dict_index_is_online_ddl(cursor->index) || (flags & BTR_CREATE_FLAG) || dict_index_is_clust(cursor->index)); ut_ad(rw_lock_own_flagged(dict_index_get_lock(cursor->index), - RW_LOCK_FLAG_X | RW_LOCK_FLAG_SX) - || dict_table_is_intrinsic(cursor->index->table)); + RW_LOCK_FLAG_X | RW_LOCK_FLAG_SX)); block = btr_cur_get_block(cursor); page = buf_block_get_frame(block); @@ -2930,7 +3021,6 @@ insert_empty: } if (!srv_read_only_mode - && !dict_table_is_intrinsic(cursor->index->table) && insert_will_fit && page_is_leaf(page) && !dict_index_is_online_ddl(cursor->index)) { @@ -3228,9 +3318,6 @@ btr_set_min_rec_mark_log( /* Write rec offset as a 2-byte ulint */ mlog_catenate_ulint(mtr, page_offset(rec), MLOG_2BYTES); } -#else /* !UNIV_HOTBACKUP */ -# define btr_set_min_rec_mark_log(rec,comp,mtr) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ /****************************************************************//** Parses the redo log record for setting an index record as the predefined @@ -3288,7 +3375,6 @@ btr_set_min_rec_mark( } } -#ifndef UNIV_HOTBACKUP /*************************************************************//** Deletes on the upper level the node pointer to a page. */ void @@ -3559,8 +3645,7 @@ btr_compress( } else { ut_ad(mtr_memo_contains_flagged(mtr, dict_index_get_lock(index), MTR_MEMO_X_LOCK - | MTR_MEMO_SX_LOCK) - || dict_table_is_intrinsic(index->table)); + | MTR_MEMO_SX_LOCK)); } #endif /* UNIV_DEBUG */ @@ -4152,8 +4237,7 @@ btr_discard_page( ut_ad(dict_index_get_page(index) != block->page.id.page_no()); ut_ad(mtr_memo_contains_flagged(mtr, dict_index_get_lock(index), - MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK) - || dict_table_is_intrinsic(index->table)); + MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK)); ut_ad(mtr_is_block_fix(mtr, block, MTR_MEMO_PAGE_X_FIX, index->table)); @@ -5382,7 +5466,6 @@ btr_can_merge_with_page( goto error; } - max_ins_size = page_get_max_insert_size(mpage, n_recs); if (data_size > max_ins_size) { @@ -5416,5 +5499,3 @@ error: *merge_block = NULL; DBUG_RETURN(false); } - -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 7daec068f78..bdf3bd2eca7 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -3,7 +3,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2015, 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -50,7 +50,6 @@ Created 10/16/1994 Heikki Tuuri #endif #include "row0upd.h" -#ifndef UNIV_HOTBACKUP #include "mtr0log.h" #include "page0page.h" #include "page0zip.h" @@ -151,9 +150,7 @@ can be released by page reorganize, then it is reorganized */ + (sample) - 1 + (ext_size) + (not_empty)) / ((sample) + (ext_size))) /* @} */ -#endif /* !UNIV_HOTBACKUP */ -#ifndef UNIV_HOTBACKUP /*******************************************************************//** Marks all extern fields in a record as owned by the record. This function should be called if the delete mark of a record is removed: a not delete @@ -212,9 +209,7 @@ btr_rec_free_externally_stored_fields( mtr_t* mtr); /*!< in: mini-transaction handle which contains an X-latch to record page and to the index tree */ -#endif /* !UNIV_HOTBACKUP */ -#ifndef UNIV_HOTBACKUP /*==================== B-TREE SEARCH =========================*/ #if MTR_MEMO_PAGE_S_FIX != RW_S_LATCH @@ -280,10 +275,10 @@ btr_cur_latch_leaves( case BTR_MODIFY_TREE: /* It is exclusive for other operations which calls btr_page_set_prev() */ - ut_ad(mtr_memo_contains_flagged(mtr, - dict_index_get_lock(cursor->index), - MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK) - || dict_table_is_intrinsic(cursor->index->table)); + ut_ad(mtr_memo_contains_flagged( + mtr, + dict_index_get_lock(cursor->index), + MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK)); /* x-latch also siblings from left to right */ left_page_no = btr_page_get_prev(page, mtr); mode = latch_mode; @@ -583,9 +578,7 @@ btr_cur_will_modify_tree( { ut_ad(!page_is_leaf(page)); ut_ad(mtr_memo_contains_flagged(mtr, dict_index_get_lock(index), - MTR_MEMO_X_LOCK - | MTR_MEMO_SX_LOCK) - || dict_table_is_intrinsic(index->table)); + MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK)); /* Pessimistic delete of the first record causes delete & insert of node_ptr at upper level. And a subsequent page shrink is @@ -753,7 +746,9 @@ btr_cur_search_to_nth_level( RW_S_LATCH, or 0 */ const char* file, /*!< in: file name */ ulint line, /*!< in: line where called */ - mtr_t* mtr) /*!< in: mtr */ + mtr_t* mtr, /*!< in: mtr */ + ib_uint64_t autoinc)/*!< in: PAGE_ROOT_AUTO_INC to be written + (0 if none) */ { page_t* page = NULL; /* remove warning */ buf_block_t* block; @@ -890,6 +885,12 @@ btr_cur_search_to_nth_level( || latch_mode == BTR_SEARCH_TREE || latch_mode == BTR_MODIFY_LEAF); + ut_ad(autoinc == 0 || dict_index_is_clust(index)); + ut_ad(autoinc == 0 + || latch_mode == BTR_MODIFY_TREE + || latch_mode == BTR_MODIFY_LEAF); + ut_ad(autoinc == 0 || level == 0); + cursor->flag = BTR_CUR_BINARY; cursor->index = index; @@ -909,13 +910,12 @@ btr_cur_search_to_nth_level( # ifdef UNIV_SEARCH_PERF_STAT info->n_searches++; # endif - /* Use of AHI is disabled for intrinsic table as these tables re-use - the index-id and AHI validation is based on index-id. */ - if (rw_lock_get_writer(btr_get_search_latch(index)) - == RW_LOCK_NOT_LOCKED + if (autoinc == 0 && latch_mode <= BTR_MODIFY_LEAF && info->last_hash_succ +# ifdef MYSQL_INDEX_DISABLE_AHI && !index->disable_ahi +# endif && !estimate # ifdef PAGE_CUR_LE_OR_EXTENDS && mode != PAGE_CUR_LE_OR_EXTENDS @@ -924,8 +924,10 @@ btr_cur_search_to_nth_level( /* If !has_search_latch, we do a dirty read of btr_search_enabled below, and btr_search_guess_on_hash() will have to check it again. */ - && UNIV_LIKELY(btr_search_enabled) + && btr_search_enabled && !modify_external + && rw_lock_get_writer(btr_get_search_latch(index)) + == RW_LOCK_NOT_LOCKED && btr_search_guess_on_hash(index, info, tuple, mode, latch_mode, cursor, has_search_latch, mtr)) { @@ -1073,16 +1075,16 @@ search_loop: if (height != 0) { /* We are about to fetch the root or a non-leaf page. */ - if ((latch_mode != BTR_MODIFY_TREE - || height == level) + if ((latch_mode != BTR_MODIFY_TREE || height == level) && !retrying_for_search_prev) { /* If doesn't have SX or X latch of index, each pages should be latched before reading. */ - if (modify_external - && height == ULINT_UNDEFINED - && upper_rw_latch == RW_S_LATCH) { + if (height == ULINT_UNDEFINED + && upper_rw_latch == RW_S_LATCH + && (modify_external || autoinc)) { /* needs sx-latch of root page - for fseg operation */ + for fseg operation or for writing + PAGE_ROOT_AUTO_INC */ rw_latch = RW_SX_LATCH; } else { rw_latch = upper_rw_latch; @@ -1273,11 +1275,13 @@ retry_page_get: && page_is_leaf(page) && rw_latch != RW_NO_LATCH && rw_latch != root_leaf_rw_latch) { - /* We should retry to get the page, because the root page - is latched with different level as a leaf page. */ + /* The root page is also a leaf page (root_leaf). + We should reacquire the page, because the root page + is latched differently from leaf pages. */ ut_ad(root_leaf_rw_latch != RW_NO_LATCH); ut_ad(rw_latch == RW_S_LATCH || rw_latch == RW_SX_LATCH); - ut_ad(rw_latch == RW_S_LATCH || modify_external); + ut_ad(rw_latch == RW_S_LATCH || modify_external || autoinc); + ut_ad(!autoinc || root_leaf_rw_latch == RW_X_LATCH); ut_ad(n_blocks == 0); mtr_release_block_at_savepoint( @@ -1367,6 +1371,7 @@ retry_page_get: /* release upper blocks */ if (retrying_for_search_prev) { + ut_ad(!autoinc); for (; prev_n_releases < prev_n_blocks; prev_n_releases++) { @@ -1380,8 +1385,9 @@ retry_page_get: } for (; n_releases < n_blocks; n_releases++) { - if (n_releases == 0 && modify_external) { - /* keep latch of root page */ + if (n_releases == 0 + && (modify_external || autoinc)) { + /* keep the root page latch */ ut_ad(mtr_memo_contains_flagged( mtr, tree_blocks[n_releases], MTR_MEMO_PAGE_SX_FIX @@ -1905,6 +1911,8 @@ need_opposite_intention: } if (level != 0) { + ut_ad(!autoinc); + if (upper_rw_latch == RW_NO_LATCH) { /* latch the page */ buf_block_t* child_block; @@ -1953,12 +1961,21 @@ need_opposite_intention: cursor->up_match = up_match; cursor->up_bytes = up_bytes; + if (autoinc) { + page_set_autoinc(tree_blocks[0], + index, autoinc, mtr, false); + } + #ifdef BTR_CUR_ADAPT /* We do a dirty read of btr_search_enabled here. We will properly check btr_search_enabled again in btr_search_build_page_hash_index() before building a page hash index, while holding search latch. */ - if (btr_search_enabled && !index->disable_ahi) { + if (btr_search_enabled +# ifdef MYSQL_INDEX_DISABLE_AHI + && !index->disable_ahi +# endif + ) { btr_search_info_update(index, cursor); } #endif @@ -2009,179 +2026,6 @@ func_exit: DBUG_RETURN(err); } -/** Searches an index tree and positions a tree cursor on a given level. -This function will avoid latching the traversal path and so should be -used only for cases where-in latching is not needed. - -@param[in,out] index index -@param[in] level the tree level of search -@param[in] tuple data tuple; Note: n_fields_cmp in compared - to the node ptr page node field -@param[in] mode PAGE_CUR_L, .... - Insert should always be made using PAGE_CUR_LE - to search the position. -@param[in,out] cursor tree cursor; points to record of interest. -@param[in] file file name -@param[in[ line line where called from -@param[in,out] mtr mtr -@param[in] mark_dirty - if true then mark the block as dirty */ -dberr_t -btr_cur_search_to_nth_level_with_no_latch( - dict_index_t* index, - ulint level, - const dtuple_t* tuple, - page_cur_mode_t mode, - btr_cur_t* cursor, - const char* file, - ulint line, - mtr_t* mtr, - bool mark_dirty) -{ - page_t* page = NULL; /* remove warning */ - buf_block_t* block; - ulint height; - ulint up_match; - ulint low_match; - ulint rw_latch; - page_cur_mode_t page_mode; - ulint buf_mode; - page_cur_t* page_cursor; - ulint root_height = 0; /* remove warning */ - ulint n_blocks = 0; - dberr_t err = DB_SUCCESS; - mem_heap_t* heap = NULL; - ulint offsets_[REC_OFFS_NORMAL_SIZE]; - ulint* offsets = offsets_; - rec_offs_init(offsets_); - - DBUG_ENTER("btr_cur_search_to_nth_level_with_no_latch"); - - ut_ad(dict_table_is_intrinsic(index->table)); - ut_ad(level == 0 || mode == PAGE_CUR_LE); - ut_ad(dict_index_check_search_tuple(index, tuple)); - ut_ad(dtuple_check_typed(tuple)); - ut_ad(index->page != FIL_NULL); - - UNIV_MEM_INVALID(&cursor->up_match, sizeof cursor->up_match); - UNIV_MEM_INVALID(&cursor->low_match, sizeof cursor->low_match); -#ifdef UNIV_DEBUG - cursor->up_match = ULINT_UNDEFINED; - cursor->low_match = ULINT_UNDEFINED; -#endif /* UNIV_DEBUG */ - - cursor->flag = BTR_CUR_BINARY; - cursor->index = index; - - page_cursor = btr_cur_get_page_cur(cursor); - - const ulint space = dict_index_get_space(index); - const page_size_t page_size(dict_table_page_size(index->table)); - /* Start with the root page. */ - page_id_t page_id(space, dict_index_get_page(index)); - - up_match = 0; - low_match = 0; - - height = ULINT_UNDEFINED; - - /* We use these modified search modes on non-leaf levels of the - B-tree. These let us end up in the right B-tree leaf. In that leaf - we use the original search mode. */ - - switch (mode) { - case PAGE_CUR_GE: - page_mode = PAGE_CUR_L; - break; - case PAGE_CUR_G: - page_mode = PAGE_CUR_LE; - break; - default: - page_mode = mode; - break; - } - - /* Loop and search until we arrive at the desired level */ - bool at_desired_level = false; - while (!at_desired_level) { - buf_mode = BUF_GET; - rw_latch = RW_NO_LATCH; - - ut_ad(n_blocks < BTR_MAX_LEVELS); - - block = buf_page_get_gen(page_id, page_size, rw_latch, NULL, - buf_mode, file, line, mtr, &err, mark_dirty); - - if (err != DB_SUCCESS) { - if (err == DB_DECRYPTION_FAILED) { - ib_push_warning((void *)NULL, - DB_DECRYPTION_FAILED, - "Table %s is encrypted but encryption service or" - " used key_id is not available. " - " Can't continue reading table.", - index->table->name); - index->table->is_encrypted = true; - } - - DBUG_RETURN(err); - } - - page = buf_block_get_frame(block); - - if (height == ULINT_UNDEFINED) { - /* We are in the root node */ - - height = btr_page_get_level(page, mtr); - root_height = height; - cursor->tree_height = root_height + 1; - } - - if (height == 0) { - /* On leaf level. Switch back to original search mode.*/ - page_mode = mode; - } - - page_cur_search_with_match( - block, index, tuple, page_mode, &up_match, - &low_match, page_cursor, NULL); - - ut_ad(height == btr_page_get_level( - page_cur_get_page(page_cursor), mtr)); - - if (level != height) { - - const rec_t* node_ptr; - ut_ad(height > 0); - - height--; - - node_ptr = page_cur_get_rec(page_cursor); - - offsets = rec_get_offsets( - node_ptr, index, offsets, - ULINT_UNDEFINED, &heap); - - /* Go to the child node */ - page_id.reset(space, btr_node_ptr_get_child_page_no( - node_ptr, offsets)); - - n_blocks++; - } else { - /* If this is the desired level, leave the loop */ - at_desired_level = true; - } - } - - cursor->low_match = low_match; - cursor->up_match = up_match; - - if (heap != NULL) { - mem_heap_free(heap); - } - - DBUG_RETURN(err); -} - /*****************************************************************//** Opens a cursor at either end of an index. */ dberr_t @@ -2556,125 +2400,6 @@ btr_cur_open_at_index_side_func( return err; } -/** Opens a cursor at either end of an index. -Avoid taking latches on buffer, just pin (by incrementing fix_count) -to keep them in buffer pool. This mode is used by intrinsic table -as they are not shared and so there is no need of latching. -@param[in] from_left true if open to low end, false if open - to high end. -@param[in] index index -@param[in,out] cursor cursor -@param[in] file file name -@param[in] line line where called -@param[in,out] mtr mini transaction -*/ -dberr_t -btr_cur_open_at_index_side_with_no_latch_func( - bool from_left, - dict_index_t* index, - btr_cur_t* cursor, - ulint level, - const char* file, - ulint line, - mtr_t* mtr) -{ - page_cur_t* page_cursor; - ulint height; - rec_t* node_ptr; - ulint n_blocks = 0; - mem_heap_t* heap = NULL; - ulint offsets_[REC_OFFS_NORMAL_SIZE]; - ulint* offsets = offsets_; - dberr_t err = DB_SUCCESS; - rec_offs_init(offsets_); - - ut_ad(level != ULINT_UNDEFINED); - - page_cursor = btr_cur_get_page_cur(cursor); - cursor->index = index; - page_id_t page_id(dict_index_get_space(index), - dict_index_get_page(index)); - const page_size_t& page_size = dict_table_page_size(index->table); - - height = ULINT_UNDEFINED; - - for (;;) { - buf_block_t* block; - page_t* page; - ulint rw_latch = RW_NO_LATCH; - - ut_ad(n_blocks < BTR_MAX_LEVELS); - - block = buf_page_get_gen(page_id, page_size, rw_latch, NULL, - BUF_GET, file, line, mtr, &err); - - if (err != DB_SUCCESS) { - if (err == DB_DECRYPTION_FAILED) { - ib_push_warning((void *)NULL, - DB_DECRYPTION_FAILED, - "Table %s is encrypted but encryption service or" - " used key_id is not available. " - " Can't continue reading table.", - index->table->name); - index->table->is_encrypted = true; - } - - return (err); - } - - page = buf_block_get_frame(block); - - ut_ad(fil_page_index_page_check(page)); - ut_ad(index->id == btr_page_get_index_id(page)); - - if (height == ULINT_UNDEFINED) { - /* We are in the root node */ - - height = btr_page_get_level(page, mtr); - ut_a(height >= level); - } else { - /* TODO: flag the index corrupted if this fails */ - ut_ad(height == btr_page_get_level(page, mtr)); - } - - if (from_left) { - page_cur_set_before_first(block, page_cursor); - } else { - page_cur_set_after_last(block, page_cursor); - } - - if (height == level) { - break; - } - - ut_ad(height > 0); - - if (from_left) { - page_cur_move_to_next(page_cursor); - } else { - page_cur_move_to_prev(page_cursor); - } - - height--; - - node_ptr = page_cur_get_rec(page_cursor); - offsets = rec_get_offsets(node_ptr, cursor->index, offsets, - ULINT_UNDEFINED, &heap); - - /* Go to the child node */ - page_id.set_page_no( - btr_node_ptr_get_child_page_no(node_ptr, offsets)); - - n_blocks++; - } - - if (heap != NULL) { - mem_heap_free(heap); - } - - return(err); -} - /**********************************************************************//** Positions a cursor at a randomly chosen position within a B-tree. @return true if the index is available and we have put the cursor, false @@ -3097,11 +2822,8 @@ btr_cur_ins_lock_and_undo( return(err); } - /* Now we can fill in the roll ptr field in entry - (except if table is intrinsic) */ - - if (!(flags & BTR_KEEP_SYS_FLAG) - && !dict_table_is_intrinsic(index->table)) { + /* Now we can fill in the roll ptr field in entry */ + if (!(flags & BTR_KEEP_SYS_FLAG)) { row_upd_index_entry_sys_field(entry, index, DATA_ROLL_PTR, roll_ptr); @@ -3191,8 +2913,6 @@ btr_cur_optimistic_insert( page = buf_block_get_frame(block); index = cursor->index; - /* Block are not latched for insert if table is intrinsic - and index is auto-generated clustered index. */ ut_ad(mtr_is_block_fix(mtr, block, MTR_MEMO_PAGE_X_FIX, index->table)); ut_ad(!dict_index_is_online_ddl(index) || dict_index_is_clust(index) @@ -3292,18 +3012,10 @@ fail_err: page_cursor = btr_cur_get_page_cur(cursor); -#ifdef UNIV_DEBUG - { - rec_printer p(entry); - DBUG_PRINT("ib_cur", ("insert %s (" IB_ID_FMT ") by " IB_ID_FMT " %s", - index->name(), index->id, - thr != NULL - ? trx_get_id_for_print(thr_get_trx(thr)) - : 0, - p.str().c_str())); - } -#endif - + DBUG_LOG("ib_cur", + "insert " << index->name << " (" << index->id << ") by " + << ib::hex(thr ? trx_get_id_for_print(thr_get_trx(thr)) : 0) + << ' ' << rec_printer(entry).str()); DBUG_EXECUTE_IF("do_page_reorganize", btr_page_reorganize(page_cursor, index, mtr);); @@ -3311,49 +3023,34 @@ fail_err: { const rec_t* page_cursor_rec = page_cur_get_rec(page_cursor); - if (dict_table_is_intrinsic(index->table)) { - - index->rec_cache.rec_size = rec_size; - - *rec = page_cur_tuple_direct_insert( - page_cursor, entry, index, n_ext, mtr); - } else { - /* Check locks and write to the undo log, - if specified */ - err = btr_cur_ins_lock_and_undo(flags, cursor, entry, - thr, mtr, &inherit); - if (err != DB_SUCCESS) { - goto fail_err; - } - - *rec = page_cur_tuple_insert( - page_cursor, entry, index, offsets, heap, - n_ext, mtr); + /* Check locks and write to the undo log, + if specified */ + err = btr_cur_ins_lock_and_undo(flags, cursor, entry, + thr, mtr, &inherit); + if (err != DB_SUCCESS) { + goto fail_err; } + *rec = page_cur_tuple_insert( + page_cursor, entry, index, offsets, heap, + n_ext, mtr); + reorg = page_cursor_rec != page_cur_get_rec(page_cursor); } if (*rec) { } else if (page_size.is_compressed()) { + ut_ad(!dict_table_is_temporary(index->table)); /* Reset the IBUF_BITMAP_FREE bits, because page_cur_tuple_insert() will have attempted page reorganize before failing. */ if (leaf - && !dict_index_is_clust(index) - && !dict_table_is_temporary(index->table)) { + && !dict_index_is_clust(index)) { ibuf_reset_free_bits(block); } goto fail; } else { - - /* For intrinsic table we take a consistent path - to re-organize using pessimistic path. */ - if (dict_table_is_intrinsic(index->table)) { - goto fail; - } - ut_ad(!reorg); /* If the record did not fit, reorganize */ @@ -3378,12 +3075,13 @@ fail_err: } #ifdef BTR_CUR_HASH_ADAPT - if (!index->disable_ahi) { - if (!reorg && leaf && (cursor->flag == BTR_CUR_HASH)) { - btr_search_update_hash_node_on_insert(cursor); - } else { - btr_search_update_hash_on_insert(cursor); - } +# ifdef MYSQL_INDEX_DISABLE_AHI + if (index->disable_ahi); else +# endif + if (!reorg && leaf && (cursor->flag == BTR_CUR_HASH)) { + btr_search_update_hash_node_on_insert(cursor); + } else { + btr_search_update_hash_on_insert(cursor); } #endif /* BTR_CUR_HASH_ADAPT */ @@ -3467,9 +3165,8 @@ btr_cur_pessimistic_insert( *big_rec = NULL; ut_ad(mtr_memo_contains_flagged( - mtr, dict_index_get_lock(btr_cur_get_index(cursor)), - MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK) - || dict_table_is_intrinsic(cursor->index->table)); + mtr, dict_index_get_lock(btr_cur_get_index(cursor)), + MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK)); ut_ad(mtr_is_block_fix( mtr, btr_cur_get_block(cursor), MTR_MEMO_PAGE_X_FIX, cursor->index->table)); @@ -3489,8 +3186,7 @@ btr_cur_pessimistic_insert( return(err); } - if (!(flags & BTR_NO_UNDO_LOG_FLAG) - || dict_table_is_intrinsic(index->table)) { + if (!(flags & BTR_NO_UNDO_LOG_FLAG)) { /* First reserve enough free space for the file segments of the index tree, so that the insert will not fail because of lack of space */ @@ -3548,7 +3244,6 @@ btr_cur_pessimistic_insert( ut_ad(page_rec_get_next(btr_cur_get_rec(cursor)) == *rec || dict_index_is_spatial(index)); - if (!(flags & BTR_NO_LOCKING_FLAG)) { ut_ad(!dict_table_is_temporary(index->table)); if (dict_index_is_spatial(index)) { @@ -3577,9 +3272,10 @@ btr_cur_pessimistic_insert( } #ifdef BTR_CUR_ADAPT - if (!index->disable_ahi) { - btr_search_update_hash_on_insert(cursor); - } +# ifdef MYSQL_INDEX_DISABLE_AHI + if (index->disable_ahi); else +# endif + btr_search_update_hash_on_insert(cursor); #endif if (inherit && !(flags & BTR_NO_LOCKING_FLAG)) { @@ -3715,7 +3411,6 @@ btr_cur_update_in_place_log( row_upd_index_write_log(update, log_ptr, mtr); } -#endif /* UNIV_HOTBACKUP */ /***********************************************************//** Parses a redo log record of updating a record in-place. @@ -3794,7 +3489,6 @@ func_exit: return(ptr); } -#ifndef UNIV_HOTBACKUP /*************************************************************//** See if there is enough place in the page modification log to log an update-in-place. @@ -3921,9 +3615,7 @@ btr_cur_update_in_place( index = cursor->index; ut_ad(rec_offs_validate(rec, index, offsets)); ut_ad(!!page_rec_is_comp(rec) == dict_table_is_comp(index->table)); - ut_ad(trx_id > 0 - || (flags & BTR_KEEP_SYS_FLAG) - || dict_table_is_intrinsic(index->table)); + ut_ad(trx_id > 0 || (flags & BTR_KEEP_SYS_FLAG)); /* The insert buffer tree should never be updated in place. */ ut_ad(!dict_index_is_ibuf(index)); ut_ad(dict_index_is_online_ddl(index) == !!(flags & BTR_CREATE_FLAG) @@ -3935,20 +3627,18 @@ btr_cur_update_in_place( ut_ad(fil_page_index_page_check(btr_cur_get_page(cursor))); ut_ad(btr_page_get_index_id(btr_cur_get_page(cursor)) == index->id); -#ifdef UNIV_DEBUG - { - rec_printer p(rec, offsets); - DBUG_PRINT("ib_cur", ("update-in-place %s (" IB_ID_FMT ") by "IB_ID_FMT ": %s", - index->name(), index->id, trx_id, - p.str().c_str())); - } -#endif + DBUG_LOG("ib_cur", + "update-in-place " << index->name << " (" << index->id + << ") by " << ib::hex(trx_id) << ": " + << rec_printer(rec, offsets).str()); block = btr_cur_get_block(cursor); page_zip = buf_block_get_page_zip(block); /* Check that enough space is available on the compressed page. */ if (page_zip) { + ut_ad(!dict_table_is_temporary(index->table)); + if (!btr_cur_update_alloc_zip( page_zip, btr_cur_get_page_cur(cursor), index, offsets, rec_offs_size(offsets), @@ -3970,8 +3660,7 @@ btr_cur_update_in_place( goto func_exit; } - if (!(flags & BTR_KEEP_SYS_FLAG) - && !dict_table_is_intrinsic(index->table)) { + if (!(flags & BTR_KEEP_SYS_FLAG)) { row_upd_rec_sys_fields(rec, NULL, index, offsets, thr_get_trx(thr), roll_ptr); } @@ -4026,9 +3715,9 @@ func_exit: if (page_zip && !(flags & BTR_KEEP_IBUF_BITMAP) && !dict_index_is_clust(index) - && !dict_table_is_temporary(index->table) && page_is_leaf(buf_block_get_frame(block))) { /* Update the free bits in the insert buffer. */ + ut_ad(!dict_table_is_temporary(index->table)); ibuf_update_free_bits_zip(block, mtr); } @@ -4087,9 +3776,7 @@ btr_cur_optimistic_update( page = buf_block_get_frame(block); rec = btr_cur_get_rec(cursor); index = cursor->index; - ut_ad(trx_id > 0 - || (flags & BTR_KEEP_SYS_FLAG) - || dict_table_is_intrinsic(index->table)); + ut_ad(trx_id > 0 || (flags & BTR_KEEP_SYS_FLAG)); ut_ad(!!page_rec_is_comp(rec) == dict_table_is_comp(index->table)); ut_ad(mtr_is_block_fix(mtr, block, MTR_MEMO_PAGE_X_FIX, index->table)); /* This is intended only for leaf page updates */ @@ -4143,14 +3830,10 @@ any_extern: } } -#ifdef UNIV_DEBUG - { - rec_printer p(rec, *offsets); - DBUG_PRINT("ib_cur", ("update %s (" IB_ID_FMT ") by " IB_ID_FMT ": %s", - index->name(), index->id, trx_id, - p.str().c_str())); - } -#endif + DBUG_LOG("ib_cur", + "update " << index->name << " (" << index->id << ") by " + << ib::hex(trx_id) << ": " + << rec_printer(rec, *offsets).str()); page_cursor = btr_cur_get_page_cur(cursor); @@ -4179,6 +3862,8 @@ any_extern: #endif /* UNIV_ZIP_DEBUG */ if (page_zip) { + ut_ad(!dict_table_is_temporary(index->table)); + if (page_zip_rec_needs_ext(new_rec_size, page_is_comp(page), dict_index_get_n_fields(index), dict_table_page_size(index->table))) { @@ -4275,8 +3960,7 @@ any_extern: page_cur_move_to_prev(page_cursor); - if (!(flags & BTR_KEEP_SYS_FLAG) - && !dict_table_is_intrinsic(index->table)) { + if (!(flags & BTR_KEEP_SYS_FLAG)) { row_upd_index_entry_sys_field(new_entry, index, DATA_ROLL_PTR, roll_ptr); row_upd_index_entry_sys_field(new_entry, index, DATA_TRX_ID, @@ -4298,12 +3982,12 @@ any_extern: func_exit: if (!(flags & BTR_KEEP_IBUF_BITMAP) - && !dict_index_is_clust(index) - && !dict_table_is_temporary(index->table)) { + && !dict_index_is_clust(index)) { /* Update the free bits in the insert buffer. */ if (page_zip) { + ut_ad(!dict_table_is_temporary(index->table)); ibuf_update_free_bits_zip(block, mtr); - } else { + } else if (!dict_table_is_temporary(index->table)) { ibuf_update_free_bits_low(block, max_ins_size, mtr); } } @@ -4422,17 +4106,16 @@ btr_cur_pessimistic_update( ut_ad(mtr_memo_contains_flagged(mtr, dict_index_get_lock(index), MTR_MEMO_X_LOCK | - MTR_MEMO_SX_LOCK) - || dict_table_is_intrinsic(index->table)); + MTR_MEMO_SX_LOCK)); ut_ad(mtr_is_block_fix(mtr, block, MTR_MEMO_PAGE_X_FIX, index->table)); #ifdef UNIV_ZIP_DEBUG ut_a(!page_zip || page_zip_validate(page_zip, page, index)); #endif /* UNIV_ZIP_DEBUG */ + ut_ad(!page_zip || !dict_table_is_temporary(index->table)); /* The insert buffer tree should never be updated in place. */ ut_ad(!dict_index_is_ibuf(index)); ut_ad(trx_id > 0 - || (flags & BTR_KEEP_SYS_FLAG) - || dict_table_is_intrinsic(index->table)); + || (flags & BTR_KEEP_SYS_FLAG)); ut_ad(dict_index_is_online_ddl(index) == !!(flags & BTR_CREATE_FLAG) || dict_index_is_clust(index)); ut_ad(thr_get_trx(thr)->id == trx_id @@ -4459,8 +4142,8 @@ btr_cur_pessimistic_update( if (page_zip && optim_err != DB_ZIP_OVERFLOW && !dict_index_is_clust(index) - && !dict_table_is_temporary(index->table) && page_is_leaf(page)) { + ut_ad(!dict_table_is_temporary(index->table)); ibuf_update_free_bits_zip(block, mtr); } @@ -4495,11 +4178,8 @@ btr_cur_pessimistic_update( ut_ad(rec_offs_validate(rec, index, *offsets)); n_ext += btr_push_update_extern_fields(new_entry, update, entry_heap); - /* UNDO logging is also turned-off during normal operation on intrinsic - table so condition needs to ensure that table is not intrinsic. */ if ((flags & BTR_NO_UNDO_LOG_FLAG) - && rec_offs_any_extern(*offsets) - && !dict_table_is_intrinsic(index->table)) { + && rec_offs_any_extern(*offsets)) { /* We are in a transaction rollback undoing a row update: we must free possible externally stored fields which got new values in the update, if they are not @@ -4512,7 +4192,6 @@ btr_cur_pessimistic_update( ut_ad(thr_get_trx(thr)->in_rollback); DBUG_EXECUTE_IF("ib_blob_update_rollback", DBUG_SUICIDE();); - RECOVERY_CRASH(99); btr_rec_free_updated_extern_fields( index, rec, page_zip, *offsets, update, true, mtr); @@ -4575,8 +4254,7 @@ btr_cur_pessimistic_update( } } - if (!(flags & BTR_KEEP_SYS_FLAG) - && !dict_table_is_intrinsic(index->table)) { + if (!(flags & BTR_KEEP_SYS_FLAG)) { row_upd_index_entry_sys_field(new_entry, index, DATA_ROLL_PTR, roll_ptr); row_upd_index_entry_sys_field(new_entry, index, DATA_TRX_ID, @@ -4637,14 +4315,14 @@ btr_cur_pessimistic_update( page_cursor->rec, index, *offsets); } } else if (!dict_index_is_clust(index) - && !dict_table_is_temporary(index->table) && page_is_leaf(page)) { /* Update the free bits in the insert buffer. This is the same block which was skipped by BTR_KEEP_IBUF_BITMAP. */ if (page_zip) { + ut_ad(!dict_table_is_temporary(index->table)); ibuf_update_free_bits_zip(block, mtr); - } else { + } else if (!dict_table_is_temporary(index->table)) { ibuf_update_free_bits_low(block, max_ins_size, mtr); } @@ -4682,7 +4360,7 @@ btr_cur_pessimistic_update( } } - if (big_rec_vec != NULL && !dict_table_is_intrinsic(index->table)) { + if (big_rec_vec != NULL) { ut_ad(page_is_leaf(page)); ut_ad(dict_index_is_clust(index)); ut_ad(flags & BTR_KEEP_POS_FLAG); @@ -4827,7 +4505,6 @@ btr_cur_del_mark_set_clust_rec_log( mlog_close(mtr, log_ptr); } -#endif /* !UNIV_HOTBACKUP */ /****************************************************************//** Parses the redo log record for delete marking or unmarking of a clustered @@ -4909,7 +4586,6 @@ btr_cur_parse_del_mark_set_clust_rec( return(ptr); } -#ifndef UNIV_HOTBACKUP /***********************************************************//** Marks a clustered index record deleted. Writes an undo log record to undo log on this delete marking. Writes in the trx id field the id @@ -4971,27 +4647,17 @@ btr_cur_del_mark_set_clust_rec( btr_rec_set_deleted_flag(rec, page_zip, TRUE); - /* For intrinsic table, roll-ptr is not maintained as there is no UNDO - logging. Skip updating it. */ - if (dict_table_is_intrinsic(index->table)) { - return(err); - } - trx = thr_get_trx(thr); /* This function must not be invoked during rollback (of a TRX_STATE_PREPARE transaction or otherwise). */ ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE)); ut_ad(!trx->in_rollback); -#ifdef UNIV_DEBUG - { - rec_printer p(rec, offsets); - DBUG_PRINT("ib_cur", ("delete-mark clust %s (" IB_ID_FMT ") by " IB_ID_FMT ": %s", - index->table_name, index->id, - trx_get_id_for_print(trx), - p.str().c_str())); - } -#endif + DBUG_LOG("ib_cur", + "delete-mark clust " << index->table->name + << " (" << index->id << ") by " + << ib::hex(trx_get_id_for_print(trx)) << ": " + << rec_printer(rec, offsets).str()); if (dict_index_is_online_ddl(index)) { row_log_table_delete(rec, entry, index, offsets, NULL); @@ -5037,7 +4703,6 @@ btr_cur_del_mark_set_sec_rec_log( mlog_close(mtr, log_ptr); } -#endif /* !UNIV_HOTBACKUP */ /****************************************************************//** Parses the redo log record for delete marking or unmarking of a secondary @@ -5082,7 +4747,6 @@ btr_cur_parse_del_mark_set_sec_rec( return(ptr); } -#ifndef UNIV_HOTBACKUP /***********************************************************//** Sets a secondary index record delete mark to TRUE or FALSE. @return DB_SUCCESS, DB_LOCK_WAIT, or error number */ @@ -5175,16 +4839,9 @@ btr_cur_compress_if_useful( cursor position even if compression occurs */ mtr_t* mtr) /*!< in/out: mini-transaction */ { - /* Avoid applying compression as we don't accept lot of page garbage - given the workload of intrinsic table. */ - if (dict_table_is_intrinsic(cursor->index->table)) { - return(FALSE); - } - ut_ad(mtr_memo_contains_flagged( mtr, dict_index_get_lock(btr_cur_get_index(cursor)), - MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK) - || dict_table_is_intrinsic(cursor->index->table)); + MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK)); ut_ad(mtr_is_block_fix( mtr, btr_cur_get_block(cursor), MTR_MEMO_PAGE_X_FIX, cursor->index->table)); @@ -5368,8 +5025,7 @@ btr_cur_pessimistic_delete( || (flags & BTR_CREATE_FLAG)); ut_ad(mtr_memo_contains_flagged(mtr, dict_index_get_lock(index), MTR_MEMO_X_LOCK - | MTR_MEMO_SX_LOCK) - || dict_table_is_intrinsic(index->table)); + | MTR_MEMO_SX_LOCK)); ut_ad(mtr_is_block_fix(mtr, block, MTR_MEMO_PAGE_X_FIX, index->table)); ut_ad(mtr->is_named_space(index->space)); @@ -5683,7 +5339,6 @@ btr_estimate_n_rows_in_range_on_level( goto inexact; } - page = buf_block_get_frame(block); /* It is possible that the tree has been reorganized in the @@ -5919,7 +5574,6 @@ btr_estimate_n_rows_in_range_low( << " index: " << index->name; } - ut_ad(page_rec_is_supremum(btr_cur_get_rec(&cursor))); /* The range specified is wihout a right border, just @@ -6406,7 +6060,6 @@ btr_estimate_number_of_different_key_vals( } } - if (n_cols == dict_index_get_n_unique_in_tree(index)) { /* If there is more than one leaf page in the tree, @@ -6914,17 +6567,14 @@ struct btr_blob_log_check_t { ut_ad(m_mtr->memo_contains_page_flagged( *m_rec, - MTR_MEMO_PAGE_X_FIX | MTR_MEMO_PAGE_SX_FIX) - || dict_table_is_intrinsic(index->table)); + MTR_MEMO_PAGE_X_FIX | MTR_MEMO_PAGE_SX_FIX)); ut_ad(mtr_memo_contains_flagged(m_mtr, dict_index_get_lock(index), - MTR_MEMO_SX_LOCK | MTR_MEMO_X_LOCK) - || dict_table_is_intrinsic(index->table)); + MTR_MEMO_SX_LOCK | MTR_MEMO_X_LOCK)); } }; - /*******************************************************************//** Stores the fields in big_rec_vec to the tablespace and puts pointers to them in rec. The extern flags in rec will have to be set beforehand. @@ -6978,9 +6628,7 @@ btr_store_big_rec_extern_fields( ut_ad(rec_offs_any_extern(offsets)); ut_ad(btr_mtr); ut_ad(mtr_memo_contains_flagged(btr_mtr, dict_index_get_lock(index), - MTR_MEMO_X_LOCK - | MTR_MEMO_SX_LOCK) - || dict_table_is_intrinsic(index->table)); + MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK)); ut_ad(mtr_is_block_fix( btr_mtr, rec_block, MTR_MEMO_PAGE_X_FIX, index->table)); ut_ad(buf_block_get_frame(rec_block) == page_align(rec)); @@ -7479,9 +7127,7 @@ btr_free_externally_stored_field( ut_ad(dict_index_is_clust(index)); ut_ad(mtr_memo_contains_flagged(local_mtr, dict_index_get_lock(index), - MTR_MEMO_X_LOCK - | MTR_MEMO_SX_LOCK) - || dict_table_is_intrinsic(index->table)); + MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK)); ut_ad(mtr_is_page_fix( local_mtr, field_ref, MTR_MEMO_PAGE_X_FIX, index->table)); ut_ad(!rec || rec_offs_validate(rec, index, offsets)); @@ -8112,4 +7758,3 @@ btr_rec_copy_externally_stored_field( return(btr_copy_externally_stored_field(len, data, page_size, local_len, heap)); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/btr/btr0defragment.cc b/storage/innobase/btr/btr0defragment.cc index 71a93744ef4..7f06aed9ff0 100644 --- a/storage/innobase/btr/btr0defragment.cc +++ b/storage/innobase/btr/btr0defragment.cc @@ -26,7 +26,6 @@ Modified 30/07/2014 Jan Lindström jan.lindstrom@mariadb.com *******************************************************/ #include "btr0defragment.h" -#ifndef UNIV_HOTBACKUP #include "btr0btr.h" #include "btr0cur.h" #include "btr0sea.h" @@ -847,5 +846,3 @@ DECLARE_THREAD(btr_defragment_thread)( os_thread_exit(); OS_THREAD_DUMMY_RETURN; } - -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/btr/btr0pcur.cc b/storage/innobase/btr/btr0pcur.cc index a5da1b9fb0c..ecf3f7c07cb 100644 --- a/storage/innobase/btr/btr0pcur.cc +++ b/storage/innobase/btr/btr0pcur.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -119,23 +120,17 @@ btr_pcur_store_position( page = page_align(rec); offs = page_offset(rec); -#ifdef UNIV_DEBUG - if (dict_index_is_spatial(index)) { - /* For spatial index, when we do positioning on parent - buffer if necessary, it might not hold latches, but the - tree must be locked to prevent change on the page */ - ut_ad((mtr_memo_contains_flagged( - mtr, dict_index_get_lock(index), - MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK) - || mtr_memo_contains(mtr, block, MTR_MEMO_PAGE_S_FIX) - || mtr_memo_contains(mtr, block, MTR_MEMO_PAGE_X_FIX)) - && (block->page.buf_fix_count > 0)); - } else { - ut_ad(mtr_memo_contains(mtr, block, MTR_MEMO_PAGE_S_FIX) - || mtr_memo_contains(mtr, block, MTR_MEMO_PAGE_X_FIX) - || dict_table_is_intrinsic(index->table)); - } -#endif /* UNIV_DEBUG */ + ut_ad(block->page.buf_fix_count); + /* For spatial index, when we do positioning on parent + buffer if necessary, it might not hold latches, but the + tree must be locked to prevent change on the page */ + ut_ad(mtr_memo_contains_flagged(mtr, block, + MTR_MEMO_PAGE_S_FIX + | MTR_MEMO_PAGE_X_FIX) + || (dict_index_is_spatial(index) + && mtr_memo_contains_flagged( + mtr, dict_index_get_lock(index), + MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK))); if (page_is_empty(page)) { /* It must be an empty index tree; NOTE that in this case @@ -283,13 +278,11 @@ btr_pcur_restore_position_func( ut_a(cursor->old_rec); ut_a(cursor->old_n_fields); - /* Optimistic latching involves S/X latch not required for - intrinsic table instead we would prefer to search fresh. */ - if ((latch_mode == BTR_SEARCH_LEAF - || latch_mode == BTR_MODIFY_LEAF - || latch_mode == BTR_SEARCH_PREV - || latch_mode == BTR_MODIFY_PREV) - && !dict_table_is_intrinsic(cursor->btr_cur.index->table)) { + switch (latch_mode) { + case BTR_SEARCH_LEAF: + case BTR_MODIFY_LEAF: + case BTR_SEARCH_PREV: + case BTR_MODIFY_PREV: /* Try optimistic restoration. */ if (!buf_pool_is_obsolete(cursor->withdraw_clock) @@ -422,7 +415,6 @@ btr_pcur_move_to_next_page( buf_block_t* next_block; page_t* next_page; ulint mode; - dict_table_t* table = btr_pcur_get_btr_cur(cursor)->index->table; ut_ad(cursor->pos_state == BTR_PCUR_IS_POSITIONED); ut_ad(cursor->latch_mode != BTR_NO_LATCHES); @@ -444,12 +436,6 @@ btr_pcur_move_to_next_page( mode = BTR_MODIFY_LEAF; } - /* For intrinsic tables we avoid taking any latches as table is - accessed by only one thread at any given time. */ - if (dict_table_is_intrinsic(table)) { - mode = BTR_NO_LATCHES; - } - buf_block_t* block = btr_pcur_get_block(cursor); next_block = btr_block_get( @@ -523,32 +509,26 @@ btr_pcur_move_backward_from_page( prev_page_no = btr_page_get_prev(page, mtr); - /* For intrinsic table we don't do optimistic restore and so there is - no left block that is pinned that needs to be released. */ - if (!dict_table_is_intrinsic( - btr_cur_get_index(btr_pcur_get_btr_cur(cursor))->table)) { - - if (prev_page_no == FIL_NULL) { - } else if (btr_pcur_is_before_first_on_page(cursor)) { + if (prev_page_no == FIL_NULL) { + } else if (btr_pcur_is_before_first_on_page(cursor)) { - prev_block = btr_pcur_get_btr_cur(cursor)->left_block; + prev_block = btr_pcur_get_btr_cur(cursor)->left_block; - btr_leaf_page_release(btr_pcur_get_block(cursor), - latch_mode, mtr); + btr_leaf_page_release(btr_pcur_get_block(cursor), + latch_mode, mtr); - page_cur_set_after_last(prev_block, + page_cur_set_after_last(prev_block, btr_pcur_get_page_cur(cursor)); - } else { + } else { - /* The repositioned cursor did not end on an infimum - record on a page. Cursor repositioning acquired a latch - also on the previous page, but we do not need the latch: - release it. */ + /* The repositioned cursor did not end on an infimum + record on a page. Cursor repositioning acquired a latch + also on the previous page, but we do not need the latch: + release it. */ - prev_block = btr_pcur_get_btr_cur(cursor)->left_block; + prev_block = btr_pcur_get_btr_cur(cursor)->left_block; - btr_leaf_page_release(prev_block, latch_mode, mtr); - } + btr_leaf_page_release(prev_block, latch_mode, mtr); } cursor->latch_mode = latch_mode; @@ -610,7 +590,7 @@ btr_pcur_open_on_user_rec_func( mtr_t* mtr) /*!< in: mtr */ { btr_pcur_open_low(index, 0, tuple, mode, latch_mode, cursor, - file, line, mtr); + file, line, 0, mtr); if ((mode == PAGE_CUR_GE) || (mode == PAGE_CUR_G)) { diff --git a/storage/innobase/btr/btr0scrub.cc b/storage/innobase/btr/btr0scrub.cc index 8ed0117b36e..8051c517e3a 100644 --- a/storage/innobase/btr/btr0scrub.cc +++ b/storage/innobase/btr/btr0scrub.cc @@ -121,8 +121,8 @@ bool btr_scrub_lock_dict_func(ulint space, bool lock_to_close_table, const char * file, uint line) { - uint start = time(0); - uint last = start; + time_t start = time(0); + time_t last = start; while (mutex_enter_nowait(&(dict_sys->mutex))) { /* if we lock to close a table, we wait forever @@ -136,10 +136,11 @@ btr_scrub_lock_dict_func(ulint space, bool lock_to_close_table, } os_thread_sleep(250000); - uint now = time(0); + time_t now = time(0); + if (now >= last + 30) { fprintf(stderr, - "WARNING: %s:%u waited %u seconds for" + "WARNING: %s:%u waited %lu seconds for" " dict_sys lock, space: %lu" " lock_to_close_table: %u\n", file, line, now - start, space, @@ -366,12 +367,17 @@ btr_optimistic_scrub( /* We play safe and reset the free bits */ if (!dict_index_is_clust(index) && - page_is_leaf(buf_block_get_frame(block))) { + block != NULL) { + buf_frame_t* frame = buf_block_get_frame(block); + if (frame && + page_is_leaf(frame)) { ibuf_reset_free_bits(block); + } } scrub_data->scrub_stat.page_reorganizations++; + return DB_SUCCESS; } @@ -486,9 +492,13 @@ btr_pessimistic_scrub( /* We play safe and reset the free bits * NOTE: need to call this prior to btr_page_split_and_insert */ if (!dict_index_is_clust(index) && - page_is_leaf(buf_block_get_frame(block))) { + block != NULL) { + buf_frame_t* frame = buf_block_get_frame(block); + if (frame && + page_is_leaf(frame)) { - ibuf_reset_free_bits(block); + ibuf_reset_free_bits(block); + } } rec = btr_page_split_and_insert( @@ -787,11 +797,8 @@ btr_scrub_page( return BTR_SCRUB_SKIP_PAGE_AND_CLOSE_TABLE; } - buf_frame_t* frame = NULL; + buf_frame_t* frame = buf_block_get_frame(block); - if (block) { - frame = buf_block_get_frame(block); - } if (!frame || btr_page_get_index_id(frame) != scrub_data->current_index->id) { /* page has been reallocated to new index */ diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 4489775d46c..0af2a8b637c 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -1214,7 +1214,9 @@ retry: (buf_fix_count == 0 when DROP TABLE or similar is executing buf_LRU_drop_page_hash_for_tablespace()). */ ut_a(index == block->index); +#ifdef MYSQL_INDEX_DISABLE_AHI ut_ad(!index->disable_ahi); +#endif ut_ad(block->page.id.space() == index->space); ut_a(index_id == index->id); @@ -1437,7 +1439,10 @@ btr_search_build_page_hash_index( ulint offsets_[REC_OFFS_NORMAL_SIZE]; ulint* offsets = offsets_; - if (index->disable_ahi || !btr_search_enabled) { +#ifdef MYSQL_INDEX_DISABLE_AHI + if (index->disable_ahi) return; +#endif + if (!btr_search_enabled) { return; } @@ -1614,15 +1619,13 @@ btr_search_move_or_delete_hash_entries( buf_block_t* block, dict_index_t* index) { - /* AHI is disabled for intrinsic table as it depends on index-id - which is dynamically assigned for intrinsic table indexes and not - through a centralized index generator. */ - if (index->disable_ahi || !btr_search_enabled) { +#ifdef MYSQL_INDEX_DISABLE_AHI + if (index->disable_ahi) return; +#endif + if (!btr_search_enabled) { return; } - ut_ad(!dict_table_is_intrinsic(index->table)); - ut_ad(rw_lock_own(&(block->lock), RW_LOCK_X)); ut_ad(rw_lock_own(&(new_block->lock), RW_LOCK_X)); @@ -1681,7 +1684,10 @@ btr_search_update_hash_on_delete(btr_cur_t* cursor) mem_heap_t* heap = NULL; rec_offs_init(offsets_); - if (cursor->index->disable_ahi || !btr_search_enabled) { +#ifdef MYSQL_INDEX_DISABLE_AHI + if (cursor->index->disable_ahi) return; +#endif + if (!btr_search_enabled) { return; } @@ -1740,7 +1746,10 @@ btr_search_update_hash_node_on_insert(btr_cur_t* cursor) dict_index_t* index; rec_t* rec; - if (cursor->index->disable_ahi || !btr_search_enabled) { +#ifdef MYSQL_INDEX_DISABLE_AHI + if (cursor->index->disable_ahi) return; +#endif + if (!btr_search_enabled) { return; } @@ -1817,7 +1826,10 @@ btr_search_update_hash_on_insert(btr_cur_t* cursor) ulint* offsets = offsets_; rec_offs_init(offsets_); - if (cursor->index->disable_ahi || !btr_search_enabled) { +#ifdef MYSQL_INDEX_DISABLE_AHI + if (cursor->index->disable_ahi) return; +#endif + if (!btr_search_enabled) { return; } @@ -1839,7 +1851,9 @@ btr_search_update_hash_on_insert(btr_cur_t* cursor) rec = btr_cur_get_rec(cursor); +#ifdef MYSQL_INDEX_DISABLE_AHI ut_a(!index->disable_ahi); +#endif ut_a(index == cursor->index); ut_a(!dict_index_is_ibuf(index)); diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index f86f3bdef40..d6a2d9ba9e0 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -35,6 +35,7 @@ Created 11/5/1995 Heikki Tuuri #include "page0size.h" #include "buf0buf.h" +#include "os0api.h" #ifdef UNIV_NONINL #include "buf0buf.ic" @@ -49,7 +50,6 @@ Created 11/5/1995 Heikki Tuuri #include "fil0fil.h" #include "fil0crypt.h" #include "fsp0sysspace.h" -#ifndef UNIV_HOTBACKUP #include "buf0buddy.h" #include "lock0lock.h" #include "sync0rw.h" @@ -59,7 +59,6 @@ Created 11/5/1995 Heikki Tuuri #include "trx0purge.h" #include "log0log.h" #include "dict0stats_bg.h" -#endif /* !UNIV_HOTBACKUP */ #include "srv0srv.h" #include "srv0start.h" #include "dict0dict.h" @@ -87,7 +86,7 @@ Created 11/5/1995 Heikki Tuuri #include "lzo/lzo1x.h" #endif -#if defined(HAVE_LIBNUMA) && defined(WITH_NUMA) +#ifdef HAVE_LIBNUMA #include <numa.h> #include <numaif.h> struct set_numa_interleave_t @@ -128,7 +127,7 @@ struct set_numa_interleave_t #define NUMA_MEMPOLICY_INTERLEAVE_IN_SCOPE set_numa_interleave_t scoped_numa #else #define NUMA_MEMPOLICY_INTERLEAVE_IN_SCOPE -#endif /* HAVE_LIBNUMA && WITH_NUMA */ +#endif /* HAVE_LIBNUMA */ /* IMPLEMENTATION OF THE BUFFER POOL @@ -318,7 +317,7 @@ that the whole area may be needed in the near future, and issue the read requests for the whole area. */ -#if (!(defined(UNIV_HOTBACKUP) || defined(UNIV_INNOCHECKSUM))) +#ifndef UNIV_INNOCHECKSUM /** Value in microseconds */ static const int WAIT_FOR_READ = 100; static const int WAIT_FOR_WRITE = 100; @@ -407,6 +406,9 @@ buf_pool_register_chunk( chunk->blocks->frame, chunk)); } +/* prototypes for new functions added to ha_innodb.cc */ +trx_t* innobase_get_trx(); + /********************************************************************//** Check if page is maybe compressed, encrypted or both when we encounter corrupted page. Note that we can't be 100% sure if page is corrupted @@ -583,7 +585,7 @@ buf_block_alloc( return(block); } -#endif /* !UNIV_HOTBACKUP && !UNIV_INNOCHECKSUM */ +#endif /* !UNIV_INNOCHECKSUM */ /** Checks if a page contains only zeroes. @param[in] read_buf database page @@ -641,19 +643,24 @@ buf_page_is_checksum_valid_crc32( #endif /* UNIV_INNOCHECKSUM */ if (checksum_field1 != checksum_field2) { - return(false); + goto invalid; } if (checksum_field1 == crc32) { return(true); - } - - const uint32_t crc32_legacy = buf_calc_page_crc32(read_buf, true); + } else { + const uint32_t crc32_legacy = buf_calc_page_crc32(read_buf, true); - if (checksum_field1 == crc32_legacy) { - return(true); + if (checksum_field1 == crc32_legacy) { + return(true); + } } +invalid: + DBUG_LOG("checksum", "Page checksum crc32 not valid" + << " field1 " << checksum_field1 + << " field2 " << checksum_field2 + << " crc32 " << crc32); return(false); } @@ -725,6 +732,13 @@ buf_page_is_checksum_valid_innodb( if (checksum_field2 != mach_read_from_4(read_buf + FIL_PAGE_LSN) && checksum_field2 != old_checksum) { + DBUG_LOG("checksum", + "Page checksum crc32 not valid" + << " field1 " << checksum_field1 + << " field2 " << checksum_field2 + << " crc32 " << buf_calc_page_old_checksum(read_buf) + << " lsn " << mach_read_from_4( + read_buf + FIL_PAGE_LSN)); return(false); } @@ -734,6 +748,13 @@ buf_page_is_checksum_valid_innodb( (always equal to 0), to FIL_PAGE_SPACE_OR_CHKSUM */ if (checksum_field1 != 0 && checksum_field1 != new_checksum) { + DBUG_LOG("checksum", + "Page checksum crc32 not valid" + << " field1 " << checksum_field1 + << " field2 " << checksum_field2 + << " crc32 " << buf_calc_page_new_checksum(read_buf) + << " lsn " << mach_read_from_4( + read_buf + FIL_PAGE_LSN)); return(false); } @@ -763,6 +784,18 @@ buf_page_is_checksum_valid_none( #endif /* UNIV_INNOCHECKSUM */ ) { +#ifndef DBUG_OFF + if (checksum_field1 != checksum_field2 + && checksum_field1 != BUF_NO_CHECKSUM_MAGIC) { + DBUG_LOG("checksum", + "Page checksum crc32 not valid" + << " field1 " << checksum_field1 + << " field2 " << checksum_field2 + << " crc32 " << BUF_NO_CHECKSUM_MAGIC + << " lsn " << mach_read_from_4(read_buf + + FIL_PAGE_LSN)); + } +#endif #ifdef UNIV_INNOCHECKSUM if (is_log_enabled @@ -806,13 +839,43 @@ buf_page_is_corrupted( #endif /* UNIV_INNOCHECKSUM */ ) { - ulint page_encrypted = (mach_read_from_4(read_buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0); ulint checksum_field1; ulint checksum_field2; + bool no_checksum = false; + +#ifndef UNIV_INNOCHECKSUM + ulint space_id = mach_read_from_4( + read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + ulint page_type = mach_read_from_4( + read_buf + FIL_PAGE_TYPE); + no_checksum = (page_type == FIL_PAGE_PAGE_COMPRESSED || + page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); + fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); + + /* Page is encrypted if encryption information is found from + tablespace and page contains used key_version. This is true + also for pages first compressed and then encrypted. */ + if (crypt_data && + crypt_data->type != CRYPT_SCHEME_UNENCRYPTED && + fil_page_is_encrypted(read_buf)) { + no_checksum = true; + } + + /* Return early if there is no checksum or END_LSN */ + if (no_checksum) { + return (FALSE); + } +#else + if (mach_read_from_4(read_buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0 + || mach_read_from_2(read_buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + no_checksum= true; + } + +#endif DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", return(TRUE); ); - if (!page_encrypted && !page_size.is_compressed() + if (!no_checksum && !page_size.is_compressed() && memcmp(read_buf + FIL_PAGE_LSN + 4, read_buf + page_size.logical() - FIL_PAGE_END_LSN_OLD_CHKSUM + 4, 4)) { @@ -820,10 +883,17 @@ buf_page_is_corrupted( /* Stored log sequence numbers at the start and the end of page do not match */ +#ifndef UNIV_INNOCHECKSUM + ib::info() << "Log sequence number at the start " + << mach_read_from_4(read_buf + FIL_PAGE_LSN + 4) + << " and the end " + << mach_read_from_4(read_buf + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + 4) + << " do not match"; +#endif return(TRUE); } -#if !defined(UNIV_HOTBACKUP) && !defined(UNIV_INNOCHECKSUM) +#ifndef UNIV_INNOCHECKSUM if (check_lsn && recv_lsn_checks_on) { lsn_t current_lsn; const lsn_t page_lsn @@ -853,7 +923,7 @@ buf_page_is_corrupted( } } -#endif /* !UNIV_HOTBACKUP && !UNIV_INNOCHECKSUM */ +#endif /* !UNIV_INNOCHECKSUM */ /* Check whether the checksum fields have correct values */ @@ -873,9 +943,6 @@ buf_page_is_corrupted( page_size.physical())); #endif /* UNIV_INNOCHECKSUM */ } - if (page_encrypted) { - return (FALSE); - } checksum_field1 = mach_read_from_4( read_buf + FIL_PAGE_SPACE_OR_CHKSUM); @@ -907,6 +974,10 @@ buf_page_is_corrupted( || i >= FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID) && read_buf[i] != 0) { +#ifndef UNIV_INNOCHECKSUM + ib::info() << "Checksum fields zero but page is not empty."; +#endif + break; } } @@ -1200,9 +1271,7 @@ buf_page_print( const page_size_t& page_size, ulint flags) { -#ifndef UNIV_HOTBACKUP dict_index_t* index; -#endif /* !UNIV_HOTBACKUP */ if (!(flags & BUF_PAGE_PRINT_NO_FULL)) { @@ -1308,7 +1377,6 @@ buf_page_print( read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); } -#ifndef UNIV_HOTBACKUP if (mach_read_from_2(read_buf + TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_TYPE) == TRX_UNDO_INSERT) { fprintf(stderr, @@ -1319,7 +1387,6 @@ buf_page_print( fprintf(stderr, "InnoDB: Page may be an update undo log page\n"); } -#endif /* !UNIV_HOTBACKUP */ switch (fil_page_get_type(read_buf)) { index_id_t index_id; @@ -1330,7 +1397,6 @@ buf_page_print( "InnoDB: Page may be an index page where" " index id is " << index_id; -#ifndef UNIV_HOTBACKUP index = dict_index_find_on_id_low(index_id); if (index) { ib::info() @@ -1338,7 +1404,6 @@ buf_page_print( << " is " << index->name << " in table " << index->table->name; } -#endif /* !UNIV_HOTBACKUP */ break; case FIL_PAGE_INODE: fputs("InnoDB: Page may be an 'inode' page\n", stderr); @@ -1385,8 +1450,6 @@ buf_page_print( ut_ad(flags & BUF_PAGE_PRINT_NO_CRASH); } -#ifndef UNIV_HOTBACKUP - # ifdef PFS_GROUP_BUFFER_SYNC extern mysql_pfs_key_t buffer_block_mutex_key; @@ -1476,7 +1539,6 @@ buf_block_init( ut_d(block->page.file_page_was_freed = FALSE); block->index = NULL; - block->made_dirty_with_no_latch = false; block->skip_flush_check = false; ut_d(block->page.in_page_hash = FALSE); @@ -1555,7 +1617,7 @@ buf_chunk_init( return(NULL); } -#if defined(HAVE_LIBNUMA) && defined(WITH_NUMA) +#ifdef HAVE_LIBNUMA if (srv_numa_interleave) { struct bitmask *numa_mems_allowed = numa_get_mems_allowed(); int st = mbind(chunk->mem, chunk->mem_size(), @@ -1569,7 +1631,7 @@ buf_chunk_init( " (error: " << strerror(errno) << ")."; } } -#endif /* HAVE_LIBNUMA && WITH_NUMA */ +#endif /* HAVE_LIBNUMA */ /* Allocate the block descriptors from @@ -1714,12 +1776,25 @@ buf_chunk_not_freed( file pages. */ break; case BUF_BLOCK_FILE_PAGE: + if (srv_read_only_mode) { + /* The page cleaner is disabled in + read-only mode. No pages can be + dirtied, so all of them must be clean. */ + ut_ad(block->page.oldest_modification + == block->page.newest_modification); + ut_ad(block->page.oldest_modification == 0 + || block->page.oldest_modification + == recv_sys->recovered_lsn); + ut_ad(block->page.buf_fix_count == 0); + ut_ad(block->page.io_fix == BUF_IO_NONE); + break; + } + buf_page_mutex_enter(block); ready = buf_flush_ready_for_replace(&block->page); buf_page_mutex_exit(block); if (!ready) { - return(block); } @@ -1900,7 +1975,7 @@ buf_pool_init_instance( /* Initialize the temporal memory array and slots */ buf_pool->tmp_arr = (buf_tmp_array_t *)ut_malloc_nokey(sizeof(buf_tmp_array_t)); memset(buf_pool->tmp_arr, 0, sizeof(buf_tmp_array_t)); - ulint n_slots = srv_n_read_io_threads * srv_n_write_io_threads * (8 * OS_AIO_N_PENDING_IOS_PER_THREAD); + ulint n_slots = (srv_n_read_io_threads + srv_n_write_io_threads) * (8 * OS_AIO_N_PENDING_IOS_PER_THREAD); buf_pool->tmp_arr->n_slots = n_slots; buf_pool->tmp_arr->slots = (buf_tmp_buffer_t*)ut_malloc_nokey(sizeof(buf_tmp_buffer_t) * n_slots); memset(buf_pool->tmp_arr->slots, 0, (sizeof(buf_tmp_buffer_t) * n_slots)); @@ -1931,6 +2006,11 @@ buf_pool_free_instance( mutex_free(&buf_pool->zip_mutex); mutex_free(&buf_pool->flush_list_mutex); + if (buf_pool->flush_rbt) { + rbt_free(buf_pool->flush_rbt); + buf_pool->flush_rbt = NULL; + } + for (bpage = UT_LIST_GET_LAST(buf_pool->LRU); bpage != NULL; bpage = prev_bpage) { @@ -3078,18 +3158,14 @@ calc_buf_pool_size: /** This is the thread for resizing buffer pool. It waits for an event and when waked up either performs a resizing and sleeps again. -@param[in] arg a dummy parameter required by os_thread_create. @return this function does not return, calls os_thread_exit() */ extern "C" os_thread_ret_t -DECLARE_THREAD(buf_resize_thread)( - void* arg MY_ATTRIBUTE((unused))) +DECLARE_THREAD(buf_resize_thread)(void*) { my_thread_init(); - srv_buf_resize_thread_active = true; - while (srv_shutdown_state == SRV_SHUTDOWN_NONE) { os_event_wait(srv_buf_resize_event); os_event_reset(srv_buf_resize_event); @@ -3399,7 +3475,6 @@ page_found: buf_pool->watch[]. However, it is not in the critical code path as this function will be called only by the purge thread. */ - /* To obey latching order first release the hash_lock. */ rw_lock_x_unlock(*hash_lock); @@ -3848,7 +3923,6 @@ buf_block_init_low( buf_block_t* block) /*!< in: block to init */ { block->index = NULL; - block->made_dirty_with_no_latch = false; block->skip_flush_check = false; block->n_hash_helps = 0; @@ -3856,7 +3930,6 @@ buf_block_init_low( block->n_bytes = 0; block->left_side = TRUE; } -#endif /* !UNIV_HOTBACKUP */ /********************************************************************//** Decompress a block. @@ -3927,7 +4000,6 @@ buf_zip_decompress( return(FALSE); } -#ifndef UNIV_HOTBACKUP /** Get a buffer block from an adaptive hash index pointer. This function does not return if the block is not identified. @param[in] ptr pointer to within a page frame @@ -4117,9 +4189,6 @@ BUF_PEEK_IF_IN_POOL, BUF_GET_NO_LATCH, or BUF_GET_IF_IN_POOL_OR_WATCH @param[in] file file name @param[in] line line where called @param[in] mtr mini-transaction -@param[in] dirty_with_no_latch - mark page as dirty even if page - is being pinned without any latch @return pointer to the block or NULL */ buf_block_t* buf_page_get_gen( @@ -4131,8 +4200,7 @@ buf_page_get_gen( const char* file, ulint line, mtr_t* mtr, - dberr_t* err, - bool dirty_with_no_latch) + dberr_t* err) { buf_block_t* block; unsigned access_time; @@ -4415,11 +4483,9 @@ got_block: bpage = &block->page; if (fsp_is_system_temporary(page_id.space()) && buf_page_get_io_fix(bpage) != BUF_IO_NONE) { - /* This suggest that page is being flushed. + /* This suggests that the page is being flushed. Avoid returning reference to this page. - Instead wait for flush action to complete. - For normal page this sync is done using SX - lock but for intrinsic there is no latching. */ + Instead wait for the flush action to complete. */ buf_block_unfix(fix_block); os_thread_sleep(WAIT_FOR_WRITE); goto loop; @@ -4744,17 +4810,6 @@ got_block: and block->lock. */ buf_wait_for_read(fix_block); - /* Mark block as dirty if requested by caller. If not requested (false) - then we avoid updating the dirty state of the block and retain the - original one. This is reason why ? - Same block can be shared/pinned by 2 different mtrs. If first mtr - set the dirty state to true and second mtr mark it as false the last - updated dirty state is retained. Which means we can loose flushing of - a modified block. */ - if (dirty_with_no_latch) { - fix_block->made_dirty_with_no_latch = dirty_with_no_latch; - } - mtr_memo_type_t fix_type; switch (rw_latch) { @@ -5800,16 +5855,16 @@ buf_page_check_corrupt( { byte* dst_frame = (bpage->zip.data) ? bpage->zip.data : ((buf_block_t*) bpage)->frame; - unsigned key_version = bpage->key_version; bool page_compressed = bpage->page_encrypted; ulint stored_checksum = bpage->stored_checksum; - ulint calculated_checksum = bpage->stored_checksum; + ulint calculated_checksum = bpage->calculated_checksum; bool page_compressed_encrypted = bpage->page_compressed; ulint space_id = mach_read_from_4( dst_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); fil_space_t* space = fil_space_found_by_id(space_id); bool corrupted = true; + ulint key_version = bpage->key_version; if (key_version != 0 || page_compressed_encrypted) { bpage->encrypted = true; @@ -5899,8 +5954,7 @@ buf_page_io_complete( if (io_type == BUF_IO_READ) { ulint read_page_no; ulint read_space_id; - byte* frame; - bool compressed_page=false; + byte* frame = NULL; ut_ad(bpage->zip.data != NULL || ((buf_block_t*)bpage)->frame != NULL); @@ -5911,7 +5965,13 @@ buf_page_io_complete( } else { frame = ((buf_block_t*) bpage)->frame; } - goto corrupt; + + ib::info() << "Page " + << bpage->id + << " encryption error key_version " + << bpage->key_version; + + goto database_corrupted; } if (bpage->size.is_compressed()) { @@ -5923,8 +5983,12 @@ buf_page_io_complete( FALSE)) { buf_pool->n_pend_unzip--; - compressed_page = false; - goto corrupt; + + ib::info() << "Page " + << bpage->id + << " zip_decompress failure."; + + goto database_corrupted; } buf_pool->n_pend_unzip--; } else { @@ -5947,13 +6011,13 @@ buf_page_io_complete( } else if (read_space_id == 0 && read_page_no == 0) { /* This is likely an uninitialized page. */ - } else if ((bpage->id.space() != 0 + } else if ((bpage->id.space() != TRX_SYS_SPACE && bpage->id.space() != read_space_id) || bpage->id.page_no() != read_page_no) { /* We did not compare space_id to read_space_id - if bpage->space == 0, because the field on the - page may contain garbage in MySQL < 4.1.1, - which only supported bpage->space == 0. */ + in the system tablespace, because the field + was written as garbage before MySQL 4.1.1, + which did not support innodb_file_per_table. */ ib::error() << "Space id and page no stored in " "the page, read in are " @@ -5961,37 +6025,16 @@ buf_page_io_complete( << ", should be " << bpage->id; } -#ifdef MYSQL_COMPRESSION - compressed_page = Compression::is_compressed_page(frame); - - /* If the decompress failed then the most likely case is - that we are reading in a page for which this instance doesn't - support the compression algorithm. */ - if (compressed_page) { - - Compression::meta_t meta; - - Compression::deserialize_header(frame, &meta); - - ib::error() - << "Page " << bpage->id << " " - << "compressed with " - << Compression::to_string(meta) << " " - << "that is not supported by this instance"; - } -#endif /* MYSQL_COMPRESSION */ - /* From version 3.23.38 up we store the page checksum to the 4 first bytes of the page end lsn field */ - if (compressed_page - || buf_page_is_corrupted( + if (buf_page_is_corrupted( true, frame, bpage->size, fsp_is_checksum_disabled(bpage->id.space()))) { /* Not a real corruption if it was triggered by error injection */ DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", - if (bpage->space > TRX_SYS_SPACE + if (bpage->id.space() != TRX_SYS_SPACE && buf_mark_space_corrupt(bpage)) { ib::info() << "Simulated page corruption"; @@ -6013,12 +6056,12 @@ buf_page_io_complete( } goto page_not_corrupt; ;); -corrupt: +database_corrupted: bool corrupted = buf_page_check_corrupt(bpage); /* Compressed and encrypted pages are basically gibberish avoid printing the contents. */ - if (corrupted && !compressed_page) { + if (corrupted) { ib::error() << "Database page corruption on disk" @@ -6053,6 +6096,7 @@ corrupt: return(false); } else { corrupted = buf_page_check_corrupt(bpage); + ulint key_version = bpage->key_version; if (corrupted) { ib::fatal() @@ -6065,11 +6109,11 @@ corrupt: } ib_push_warning((void *)NULL, DB_DECRYPTION_FAILED, - "Table in tablespace %lu encrypted." + "Table in tablespace %u encrypted." "However key management plugin or used key_id %u is not found or" " used encryption algorithm or method does not match." " Can't continue opening the table.", - (ulint)bpage->space, bpage->key_version); + bpage->id.space(), key_version); buf_page_print(frame, bpage->size, BUF_PAGE_PRINT_NO_CRASH); @@ -6093,23 +6137,20 @@ corrupt: /* If space is being truncated then avoid ibuf operation. During re-init we have already freed ibuf entries. */ if (uncompressed -#ifdef MYSQL_COMPRESSION - && !Compression::is_compressed_page(frame) -#endif /* MYSQL_COMPRESSION */ && !recv_no_ibuf_operations && !Tablespace::is_undo_tablespace(bpage->id.space()) - && bpage->id.space() != srv_tmp_space.space_id() + && bpage->id.space() != SRV_TMP_SPACE_ID && !srv_is_tablespace_truncated(bpage->id.space()) && fil_page_get_type(frame) == FIL_PAGE_INDEX && page_is_leaf(frame)) { if (bpage && bpage->encrypted) { fprintf(stderr, - "InnoDB: Warning: Table in tablespace %lu encrypted." + "InnoDB: Warning: Table in tablespace %u encrypted." "However key management plugin or used key_id %u is not found or" " used encryption algorithm or method does not match." " Can't continue opening the table.\n", - (ulint)bpage->space, bpage->key_version); + bpage->id.space(), bpage->key_version); } else { ibuf_merge_or_delete_for_page( (buf_block_t*) bpage, bpage->id, @@ -7260,57 +7301,6 @@ buf_pool_check_no_pending_io(void) return(pending_io); } -#if 0 -Code currently not used -/*********************************************************************//** -Gets the current length of the free list of buffer blocks. -@return length of the free list */ -ulint -buf_get_free_list_len(void) -/*=======================*/ -{ - ulint len; - - buf_pool_mutex_enter(buf_pool); - - len = UT_LIST_GET_LEN(buf_pool->free); - - buf_pool_mutex_exit(buf_pool); - - return(len); -} -#endif - -#else /* !UNIV_HOTBACKUP */ - -/** Inits a page to the buffer buf_pool, for use in mysqlbackup --restore. -@param[in] page_id page id -@param[in] page_size page size -@param[in,out] block block to init */ -void -buf_page_init_for_backup_restore( - const page_id_t& page_id, - const page_size_t& page_size, - buf_block_t* block) -{ - block->page.state = BUF_BLOCK_FILE_PAGE; - block->page.id = page_id; - block->page.size.copy_from(page_size); - - page_zip_des_init(&block->page.zip); - - /* We assume that block->page.data has been allocated - with page_size == univ_page_size. */ - if (page_size.is_compressed()) { - page_zip_set_size(&block->page.zip, page_size.physical()); - block->page.zip.data = block->frame + page_size.logical(); - } else { - page_zip_set_size(&block->page.zip, 0); - } -} - -#endif /* !UNIV_HOTBACKUP */ - /** Print the given page_id_t object. @param[in,out] out the output stream @param[in] page_id the page_id_t object to be printed @@ -7440,12 +7430,12 @@ buf_page_encrypt_before_write( return src_frame; } - if (crypt_data != NULL && crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) { + if (crypt_data != NULL && crypt_data->not_encrypted()) { /* Encryption is disabled */ encrypted = false; } - if (!srv_encrypt_tables && (crypt_data == NULL || crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { + if (!srv_encrypt_tables && (crypt_data == NULL || crypt_data->is_default_encryption())) { /* Encryption is disabled */ encrypted = false; } @@ -7550,6 +7540,35 @@ buf_page_decrypt_after_read( bool page_compressed_encrypted = fil_page_is_compressed_encrypted(dst_frame); buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); bool success = true; + ulint space_id = mach_read_from_4( + dst_frame + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); + + /* Page is encrypted if encryption information is found from + tablespace and page contains used key_version. This is true + also for pages first compressed and then encrypted. */ + if (!crypt_data || + (crypt_data && + crypt_data->type == CRYPT_SCHEME_UNENCRYPTED && + key_version != 0)) { + byte* frame = NULL; + + if (bpage->size.is_compressed()) { + frame = bpage->zip.data; + } else { + frame = ((buf_block_t*) bpage)->frame; + } + + /* If page is not corrupted at this point, page can't be + encrypted, thus set key_version to 0. If page is corrupted, + we assume at this point that it is encrypted as page + contained key_version != 0. Note that page could still be + really corrupted. This we will find out after decrypt by + checking page checksums. */ + if (!buf_page_is_corrupted(false, frame, bpage->size, false)) { + key_version = 0; + } + } /* If page is encrypted read post-encryption checksum */ if (!page_compressed_encrypted && key_version != 0) { @@ -7567,7 +7586,6 @@ buf_page_decrypt_after_read( bpage->key_version = key_version; bpage->page_encrypted = page_compressed_encrypted; bpage->page_compressed = page_compressed; - bpage->space = bpage->id.space(); if (page_compressed) { /* the page we read is unencrypted */ @@ -7653,5 +7671,30 @@ buf_page_decrypt_after_read( return (success); } -#endif /* !UNIV_INNOCHECKSUM */ +/** +Should we punch hole to deallocate unused portion of the page. +@param[in] bpage Page control block +@return true if punch hole should be used, false if not */ +bool +buf_page_should_punch_hole( + const buf_page_t* bpage) +{ + return (bpage->real_size != bpage->size.physical()); +} + +/** +Calculate the length of trim (punch_hole) operation. +@param[in] bpage Page control block +@param[in] write_length Write length +@return length of the trim or zero. */ +ulint +buf_page_get_trim_length( + const buf_page_t* bpage, + ulint write_length) +{ + return (bpage->size.physical() - write_length); +} + + +#endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index ff1d2057e6a..4f83921a553 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -40,8 +40,6 @@ Created 2011/12/19 #include "fil0crypt.h" #include "fil0pagecompress.h" -#ifndef UNIV_HOTBACKUP - /** The doublewrite buffer */ buf_dblwr_t* buf_dblwr = NULL; @@ -382,8 +380,6 @@ buf_dblwr_init_or_load_pages( IORequest read_request(IORequest::READ); - read_request.disable_compression(); - err = os_file_read( read_request, file, read_buf, TRX_SYS_PAGE_NO * UNIV_PAGE_SIZE, @@ -498,11 +494,6 @@ buf_dblwr_init_or_load_pages( IORequest write_request(IORequest::WRITE); - /* Recovered data file pages are written out - as uncompressed. */ - - write_request.disable_compression(); - err = os_file_write( write_request, path, file, page, source_page_no * UNIV_PAGE_SIZE, @@ -518,8 +509,9 @@ buf_dblwr_init_or_load_pages( return(err); } - } else { - + } else if (memcmp(field_ref_zero, page + FIL_PAGE_LSN, 8)) { + /* Each valid page header must contain + a nonzero FIL_PAGE_LSN field. */ recv_dblwr.add(page); } @@ -553,11 +545,9 @@ buf_dblwr_process(void) for (recv_dblwr_t::list::iterator i = recv_dblwr.pages.begin(); i != recv_dblwr.pages.end(); ++i, ++page_no_dblwr) { - bool is_compressed = false; - - const byte* page = *i; - ulint page_no = page_get_page_no(page); - ulint space_id = page_get_space_id(page); + byte* page = *i; + ulint page_no = page_get_page_no(page); + ulint space_id = page_get_space_id(page); fil_space_t* space = fil_space_get(space_id); @@ -569,163 +559,130 @@ buf_dblwr_process(void) fil_space_open_if_needed(space); + const page_id_t page_id(space_id, page_no); + if (page_no >= space->size) { - /* Do not report the warning if the tablespace is - schedule for truncate or was truncated and we have live - MLOG_TRUNCATE record in redo. */ - bool skip_warning = - srv_is_tablespace_truncated(space_id) - || srv_was_tablespace_truncated(space); - - if (!skip_warning) { - ib::warn() << "Page " << page_no_dblwr - << " in the doublewrite buffer is" - " not within space bounds: page " - << page_id_t(space_id, page_no); + /* Do not report the warning if the tablespace + is scheduled for truncation or was truncated + and we have parsed an MLOG_TRUNCATE record. */ + if (!srv_is_tablespace_truncated(space_id) + && !srv_was_tablespace_truncated(space)) { + ib::warn() << "A copy of page " << page_id + << " in the doublewrite buffer slot " + << page_no_dblwr + << " is not within space bounds"; } - } else { - const page_size_t page_size(space->flags); - const page_id_t page_id(space_id, page_no); + continue; + } - /* We want to ensure that for partial reads the - unread portion of the page is NUL. */ - memset(read_buf, 0x0, page_size.physical()); + const page_size_t page_size(space->flags); + ut_ad(!buf_page_is_zeroes(page, page_size)); - IORequest request; + /* We want to ensure that for partial reads the + unread portion of the page is NUL. */ + memset(read_buf, 0x0, page_size.physical()); - request.dblwr_recover(); + IORequest request; - /* Read in the actual page from the file */ - dberr_t err = fil_io( - request, true, - page_id, page_size, - 0, page_size.physical(), read_buf, NULL, NULL); + request.dblwr_recover(); - if (err != DB_SUCCESS) { + /* Read in the actual page from the file */ + dberr_t err = fil_io( + request, true, + page_id, page_size, + 0, page_size.physical(), read_buf, NULL); - ib::warn() - << "Double write buffer recovery: " - << page_id << " read failed with " - << "error: " << ut_strerr(err); - } + if (err != DB_SUCCESS) { + ib::warn() + << "Double write buffer recovery: " + << page_id << " read failed with " + << "error: " << ut_strerr(err); + } - /* Is page compressed ? */ - is_compressed = fil_page_is_compressed_encrypted(read_buf) | - fil_page_is_compressed(read_buf); + const bool is_all_zero = buf_page_is_zeroes( + read_buf, page_size); - /* If page was compressed, decompress it before we - check checksum. */ - if (is_compressed) { - fil_decompress_page(NULL, read_buf, UNIV_PAGE_SIZE, NULL, true); + if (is_all_zero) { + /* We will check if the copy in the + doublewrite buffer is valid. If not, we will + ignore this page (there should be redo log + records to initialize it). */ + } else { + if (fil_page_is_compressed_encrypted(read_buf) || + fil_page_is_compressed(read_buf)) { + /* Decompress the page before + validating the checksum. */ + fil_decompress_page( + NULL, read_buf, UNIV_PAGE_SIZE, + NULL, true); } - if (err != DB_SUCCESS) { - ib::warn() - << "Double write buffer recovery: " - << page_id << " read failed with " - << "error: " << ut_strerr(err); + if (fil_space_verify_crypt_checksum( + read_buf, page_size) + || !buf_page_is_corrupted( + true, read_buf, page_size, false)) { + /* The page is good; there is no need + to consult the doublewrite buffer. */ + continue; } - if (fil_space_verify_crypt_checksum(read_buf, page_size)) { - - /* page is encrypted and checksum is OK */ - } else if (buf_page_is_corrupted( - true, read_buf, page_size, - fsp_is_checksum_disabled(space_id))) { - - ib::warn() << "Database page corruption or" - << " a failed file read of page " - << page_id - << ". Trying to recover it from the" - << " doublewrite buffer."; - - /* Is page compressed ? */ - is_compressed = fil_page_is_compressed_encrypted(page) | - fil_page_is_compressed(page); - - /* If page was compressed, decompress it before we - check checksum. */ - if (is_compressed) { - fil_decompress_page(NULL, (byte*)page, UNIV_PAGE_SIZE, NULL, true); - } - - if (fil_space_verify_crypt_checksum(page, page_size)) { - /* the doublewrite buffer page is encrypted and OK */ - } else if (buf_page_is_corrupted( - true, page, page_size, - fsp_is_checksum_disabled(space_id))) { - - ib::error() << "Dump of the page:"; - - buf_page_print( - read_buf, page_size, - BUF_PAGE_PRINT_NO_CRASH); - ib::error() << "Dump of corresponding" - " page in doublewrite buffer:"; - - buf_page_print( - page, page_size, - BUF_PAGE_PRINT_NO_CRASH); - - ib::fatal() << "The page in the" - " doublewrite buffer is" - " corrupt. Cannot continue" - " operation. You can try to" - " recover the database with" - " innodb_force_recovery=6"; - } - } else if (buf_page_is_zeroes(read_buf, page_size) - && !buf_page_is_zeroes(page, page_size) - && !buf_page_is_corrupted( - true, page, page_size, - fsp_is_checksum_disabled(space_id))) { - - /* Database page contained only zeroes, while - a valid copy is available in dblwr buffer. */ - - } else { - - bool t1 = buf_page_is_zeroes( - read_buf, page_size); - - bool t2 = buf_page_is_zeroes(page, page_size); - - bool t3 = buf_page_is_corrupted( - true, page, page_size, - fsp_is_checksum_disabled(space_id)); - - if (t1 && !(t2 || t3)) { + /* We intentionally skip this message for + is_all_zero pages. */ + ib::info() + << "Trying to recover page " << page_id + << " from the doublewrite buffer."; + } - /* Database page contained only - zeroes, while a valid copy is - available in dblwr buffer. */ + /* Next, validate the doublewrite page. */ + if (fil_page_is_compressed_encrypted(page) || + fil_page_is_compressed(page)) { + /* Decompress the page before + validating the checksum. */ + fil_decompress_page( + NULL, page, UNIV_PAGE_SIZE, NULL, true); + } - } else { - continue; - } + if (!fil_space_verify_crypt_checksum(page, page_size) + && buf_page_is_corrupted(true, page, page_size, false)) { + if (!is_all_zero) { + ib::warn() << "A doublewrite copy of page " + << page_id << " is corrupted."; } + /* Theoretically we could have another good + copy for this page in the doublewrite + buffer. If not, we will report a fatal error + for a corrupted page somewhere else if that + page was truly needed. */ + continue; + } - /* Recovered data file pages are written out - as uncompressed. */ - - IORequest write_request(IORequest::WRITE); + if (page_no == 0) { + /* Check the FSP_SPACE_FLAGS. */ + ulint flags = fsp_header_get_flags(page); + if (!fsp_flags_is_valid(flags) + && fsp_flags_convert_from_101(flags) + == ULINT_UNDEFINED) { + ib::warn() << "Ignoring a doublewrite copy" + " of page " << page_id + << " due to invalid flags " + << ib::hex(flags); + continue; + } + /* The flags on the page should be converted later. */ + } - write_request.disable_compression(); + /* Write the good page from the doublewrite buffer to + the intended position. */ - /* Write the good page from the doublewrite - buffer to the intended position. */ + IORequest write_request(IORequest::WRITE); - fil_io(write_request, true, - page_id, page_size, - 0, page_size.physical(), - const_cast<byte*>(page), NULL, NULL); + fil_io(write_request, true, page_id, page_size, + 0, page_size.physical(), + const_cast<byte*>(page), NULL); - ib::info() - << "Recovered page " - << page_id - << " from the doublewrite buffer."; - } + ib::info() << "Recovered page " << page_id + << " from the doublewrite buffer."; } recv_dblwr.pages.clear(); @@ -955,7 +912,7 @@ buf_dblwr_write_block_to_datafile( type |= IORequest::DO_NOT_WAKE; } - IORequest request(type); + IORequest request(type, const_cast<buf_page_t*>(bpage)); /* We request frame here to get correct buffer in case of encryption and/or page compression */ @@ -967,7 +924,7 @@ buf_dblwr_write_block_to_datafile( fil_io(request, sync, bpage->id, bpage->size, 0, bpage->size.physical(), (void*) frame, - (void*) bpage, NULL); + (void*) bpage); } else { ut_ad(!bpage->size.is_compressed()); @@ -981,8 +938,8 @@ buf_dblwr_write_block_to_datafile( buf_dblwr_check_page_lsn(block->frame); fil_io(request, - sync, bpage->id, bpage->size, 0, bpage->size.physical(), - frame, block, (ulint *)&bpage->write_size); + sync, bpage->id, bpage->size, 0, bpage->real_size, + frame, block); } } @@ -1084,7 +1041,7 @@ try_again: fil_io(IORequestWrite, true, page_id_t(TRX_SYS_SPACE, buf_dblwr->block1), univ_page_size, - 0, len, (void*) write_buf, NULL, NULL); + 0, len, (void*) write_buf, NULL); if (buf_dblwr->first_free <= TRX_SYS_DOUBLEWRITE_BLOCK_SIZE) { /* No unwritten pages in the second block. */ @@ -1100,7 +1057,7 @@ try_again: fil_io(IORequestWrite, true, page_id_t(TRX_SYS_SPACE, buf_dblwr->block2), univ_page_size, - 0, len, (void*) write_buf, NULL, NULL); + 0, len, (void*) write_buf, NULL); flush: /* increment the doublewrite flushed pages counter */ @@ -1195,7 +1152,6 @@ try_again: } else { ut_a(buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); - UNIV_MEM_ASSERT_RW(frame, bpage->size.logical()); @@ -1336,7 +1292,6 @@ retry: 0, univ_page_size.physical(), (void *)(buf_dblwr->write_buf + univ_page_size.physical() * i), - NULL, NULL); } else { /* It is a regular page. Write it directly to the @@ -1348,7 +1303,6 @@ retry: 0, univ_page_size.physical(), (void*) frame, - NULL, NULL); } @@ -1360,4 +1314,3 @@ retry: blocks. Next do the write to the intended position. */ buf_dblwr_write_block_to_datafile(bpage, sync); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/buf/buf0dump.cc b/storage/innobase/buf/buf0dump.cc index 7042e25d5ba..9110345fa24 100644 --- a/storage/innobase/buf/buf0dump.cc +++ b/storage/innobase/buf/buf0dump.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -53,8 +54,8 @@ enum status_severity { /* Flags that tell the buffer pool dump/load thread which action should it take after being waked up. */ -static ibool buf_dump_should_start = FALSE; -static ibool buf_load_should_start = FALSE; +static volatile bool buf_dump_should_start; +static volatile bool buf_load_should_start; static ibool buf_load_abort_flag = FALSE; @@ -78,7 +79,7 @@ void buf_dump_start() /*============*/ { - buf_dump_should_start = TRUE; + buf_dump_should_start = true; os_event_set(srv_buf_dump_event); } @@ -91,7 +92,7 @@ void buf_load_start() /*============*/ { - buf_load_should_start = TRUE; + buf_load_should_start = true; os_event_set(srv_buf_dump_event); } @@ -790,10 +791,7 @@ again. @return this function does not return, it calls os_thread_exit() */ extern "C" os_thread_ret_t -DECLARE_THREAD(buf_dump_thread)( -/*============================*/ - void* arg MY_ATTRIBUTE((unused))) /*!< in: a dummy parameter - required by os_thread_create */ +DECLARE_THREAD(buf_dump_thread)(void*) { ut_ad(!srv_read_only_mode); /* JAN: TODO: MySQL 5.7 PSI @@ -801,8 +799,6 @@ DECLARE_THREAD(buf_dump_thread)( pfs_register_thread(buf_dump_thread_key); #endif */ /* UNIV_PFS_THREAD */ - srv_buf_dump_thread_active = TRUE; - buf_dump_status(STATUS_VERBOSE, "Dumping of buffer pool not started"); buf_load_status(STATUS_VERBOSE, "Loading of buffer pool not started"); @@ -815,15 +811,18 @@ DECLARE_THREAD(buf_dump_thread)( os_event_wait(srv_buf_dump_event); if (buf_dump_should_start) { - buf_dump_should_start = FALSE; + buf_dump_should_start = false; buf_dump(TRUE /* quit on shutdown */); } if (buf_load_should_start) { - buf_load_should_start = FALSE; + buf_load_should_start = false; buf_load(); } + if (buf_dump_should_start || buf_load_should_start) { + continue; + } os_event_reset(srv_buf_dump_event); } @@ -832,7 +831,7 @@ DECLARE_THREAD(buf_dump_thread)( keep going even if we are in a shutdown state */); } - srv_buf_dump_thread_active = FALSE; + srv_buf_dump_thread_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 26e0da4e371..ed975702fa3 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. Copyright (c) 2013, 2014, Fusion-io This program is free software; you can redistribute it and/or modify it under @@ -41,7 +41,6 @@ Created 11/11/1995 Heikki Tuuri #include "srv0start.h" #include "srv0srv.h" #include "page0zip.h" -#ifndef UNIV_HOTBACKUP #include "ut0byte.h" #include "page0page.h" #include "fil0fil.h" @@ -796,6 +795,8 @@ buf_flush_write_complete( flush_type = buf_page_get_flush_type(bpage); buf_pool->n_flush[flush_type]--; + ut_ad(buf_pool_mutex_own(buf_pool)); + if (buf_pool->n_flush[flush_type] == 0 && buf_pool->init_flush[flush_type] == FALSE) { @@ -806,7 +807,6 @@ buf_flush_write_complete( buf_dblwr_update(bpage, flush_type); } -#endif /* !UNIV_HOTBACKUP */ /** Calculate the checksum of a page from compressed table and update the page. @@ -995,7 +995,6 @@ buf_flush_init_for_writing( checksum); } -#ifndef UNIV_HOTBACKUP /********************************************************************//** Does an asynchronous write of a buffer page. NOTE: in simulated aio and also when the doublewrite buffer is used, we must call @@ -1011,7 +1010,7 @@ buf_flush_write_block_low( { page_t* frame = NULL; ulint space_id = bpage->id.space(); - atomic_writes_t awrites = fil_space_get_atomic_writes(space_id); + bool atomic_writes = fil_space_get_atomic_writes(space_id); #ifdef UNIV_DEBUG buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); @@ -1089,18 +1088,18 @@ buf_flush_write_block_low( || buf_dblwr == NULL || srv_read_only_mode || fsp_is_system_temporary(bpage->id.space()) - || awrites == ATOMIC_WRITES_ON) { + || atomic_writes) { ut_ad(!srv_read_only_mode || fsp_is_system_temporary(bpage->id.space())); ulint type = IORequest::WRITE | IORequest::DO_NOT_WAKE; - IORequest request(type); + IORequest request(type, bpage); fil_io(request, sync, bpage->id, bpage->size, 0, bpage->size.physical(), - frame, bpage, NULL); + frame, bpage); } else { if (flush_type == BUF_FLUSH_SINGLE_PAGE) { buf_dblwr_write_single_page(bpage, sync); @@ -2264,8 +2263,6 @@ buf_flush_single_page_from_LRU( scanned); } - - ut_ad(!buf_pool_mutex_own(buf_pool)); return(freed); } @@ -2521,7 +2518,6 @@ page_cleaner_flush_pages_recommendation( lsn_avg_rate = (lsn_avg_rate + lsn_rate) / 2; - /* aggregate stats of all slots */ mutex_enter(&page_cleaner->mutex); @@ -3237,7 +3233,8 @@ DECLARE_THREAD(buf_flush_page_cleaner_coordinator)( if (ret_sleep == OS_SYNC_TIME_EXCEEDED) { ulint curr_time = ut_time_ms(); - if (curr_time > next_loop_time + 3000) { + if (curr_time > next_loop_time + 3000 + && !(test_flags & TEST_SIGINT)) { if (warn_count == 0) { ib::info() << "page_cleaner: 1000ms" " intended loop took " @@ -3678,7 +3675,6 @@ buf_flush_validate( } #endif /* UNIV_DEBUG || UNIV_BUF_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ /******************************************************************//** Check if there are any dirty pages that belong to a space id in the flush @@ -3767,9 +3763,7 @@ FlushObserver::FlushObserver( m_removed->at(i) = 0; } -#ifdef FLUSH_LIST_OBSERVER_DEBUG - ib::info() << "FlushObserver constructor: " << m_trx->id; -#endif /* FLUSH_LIST_OBSERVER_DEBUG */ + DBUG_LOG("flush", "FlushObserver(): trx->id=" << m_trx->id); } /** FlushObserver deconstructor */ @@ -3780,9 +3774,7 @@ FlushObserver::~FlushObserver() UT_DELETE(m_flushed); UT_DELETE(m_removed); -#ifdef FLUSH_LIST_OBSERVER_DEBUG - ib::info() << "FlushObserver deconstructor: " << m_trx->id; -#endif /* FLUSH_LIST_OBSERVER_DEBUG */ + DBUG_LOG("flush", "~FlushObserver(): trx->id=" << m_trx->id); } /** Check whether trx is interrupted @@ -3815,10 +3807,7 @@ FlushObserver::notify_flush( m_stage->inc(); } -#ifdef FLUSH_LIST_OBSERVER_DEBUG - ib::info() << "Flush <" << bpage->id.space() - << ", " << bpage->id.page_no() << ">"; -#endif /* FLUSH_LIST_OBSERVER_DEBUG */ + DBUG_LOG("flush", "Flush " << bpage->id); } /** Notify observer of a remove @@ -3833,10 +3822,7 @@ FlushObserver::notify_remove( m_removed->at(buf_pool->instance_no)++; -#ifdef FLUSH_LIST_OBSERVER_DEBUG - ib::info() << "Remove <" << bpage->id.space() - << ", " << bpage->id.page_no() << ">"; -#endif /* FLUSH_LIST_OBSERVER_DEBUG */ + DBUG_LOG("flush", "Remove " << bpage->id); } /** Flush dirty pages and wait. */ diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 5d2077763a6..0169a63e972 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -28,7 +28,6 @@ Created 11/5/1995 Heikki Tuuri #include "buf0lru.ic" #endif /* UNIV_NOINL */ -#ifndef UNIV_HOTBACKUP #include "ut0byte.h" #include "ut0rnd.h" #include "sync0rw.h" @@ -80,10 +79,6 @@ static const ulint BUF_LRU_DROP_SEARCH_SIZE = 1024; during LRU eviction. */ static const ulint BUF_LRU_SEARCH_SCAN_THRESHOLD = 100; -/** We scan these many blocks when looking for a clean page to evict -during LRU eviction. */ -#define BUF_LRU_SEARCH_SCAN_THRESHOLD 100 - /** If we switch on the InnoDB monitor because there are too few available frames in the buffer pool, we set this to TRUE */ static bool buf_lru_switched_on_innodb_mon = false; @@ -2791,4 +2786,3 @@ buf_LRU_print(void) } } #endif /* UNIV_DEBUG_PRINT || UNIV_DEBUG || UNIV_BUF_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/buf/buf0mtflu.cc b/storage/innobase/buf/buf0mtflu.cc index 117de5cc948..7c15b12950e 100644 --- a/storage/innobase/buf/buf0mtflu.cc +++ b/storage/innobase/buf/buf0mtflu.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2014, Fusion-io. All Rights Reserved. -Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -118,15 +118,65 @@ typedef struct wrk_itm mem_heap_t *rheap; } wrk_t; -typedef struct thread_data +struct thread_data_t { os_thread_id_t wthread_id; /*!< Identifier */ wthr_status_t wt_status; /*!< Worker thread status */ -} thread_data_t; +}; -/* Thread syncronization data */ -typedef struct thread_sync +/** Flush dirty pages when multi-threaded flush is used. */ +extern "C" UNIV_INTERN +os_thread_ret_t +DECLARE_THREAD(mtflush_io_thread)(void* arg); + +/** Thread syncronization data */ +struct thread_sync_t { + /** Constructor */ + thread_sync_t(ulint n_threads, mem_heap_t* wheap, mem_heap_t* rheap) : + thread_global_mtx(), n_threads(n_threads), + wq(ib_wqueue_create()), + wr_cq(ib_wqueue_create()), + rd_cq(ib_wqueue_create()), + wheap(wheap), rheap(rheap), gwt_status(), + thread_data(static_cast<thread_data_t*>( + mem_heap_zalloc(wheap, n_threads + * sizeof *thread_data))) + { + ut_a(wq); + ut_a(wr_cq); + ut_a(rd_cq); + ut_a(thread_data); + + mutex_create(LATCH_ID_MTFLUSH_THREAD_MUTEX, + &thread_global_mtx); + + /* Create threads for page-compression-flush */ + for(ulint i = 0; i < n_threads; i++) { + thread_data[i].wt_status = WTHR_INITIALIZED; + os_thread_create(mtflush_io_thread, this, + &thread_data[i].wthread_id); + } + } + + /** Destructor */ + ~thread_sync_t() + { + ut_a(ib_wqueue_is_empty(wq)); + ut_a(ib_wqueue_is_empty(wr_cq)); + ut_a(ib_wqueue_is_empty(rd_cq)); + + /* Free all queues */ + ib_wqueue_free(wq); + ib_wqueue_free(wr_cq); + ib_wqueue_free(rd_cq); + + mutex_free(&thread_global_mtx); + + mem_heap_free(rheap); + mem_heap_free(wheap); + } + /* Global variables used by all threads */ ib_mutex_t thread_global_mtx; /*!< Mutex used protecting below variables */ @@ -142,31 +192,19 @@ typedef struct thread_sync /* Variables used by only one thread at a time */ thread_data_t* thread_data; /*!< Thread specific data */ +}; -} thread_sync_t; - -static int mtflush_work_initialized = -1; -static thread_sync_t* mtflush_ctx=NULL; +static thread_sync_t* mtflush_ctx; static ib_mutex_t mtflush_mtx; /******************************************************************//** -Set multi-threaded flush work initialized. */ -static inline -void -buf_mtflu_work_init(void) -/*=====================*/ -{ - mtflush_work_initialized = 1; -} - -/******************************************************************//** Return true if multi-threaded flush is initialized @return true if initialized */ bool buf_mtflu_init_done(void) /*=====================*/ { - return(mtflush_work_initialized == 1); + return(mtflush_ctx != NULL); } /******************************************************************//** @@ -307,15 +345,10 @@ mtflush_service_io( } } -/******************************************************************//** -Thead used to flush dirty pages when multi-threaded flush is -used. -@return a dummy parameter*/ +/** Flush dirty pages when multi-threaded flush is used. */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(mtflush_io_thread)( -/*==============================*/ - void * arg) +DECLARE_THREAD(mtflush_io_thread)(void* arg) { thread_sync_t *mtflush_io = ((thread_sync_t *)arg); thread_data_t *this_thread_data = NULL; @@ -438,29 +471,10 @@ buf_mtflu_io_thread_exit(void) ib_wqueue_nowait(mtflush_io->wq); } - mutex_enter(&mtflush_mtx); - - ut_a(ib_wqueue_is_empty(mtflush_io->wq)); - ut_a(ib_wqueue_is_empty(mtflush_io->wr_cq)); - ut_a(ib_wqueue_is_empty(mtflush_io->rd_cq)); - - /* Free all queues */ - ib_wqueue_free(mtflush_io->wq); - ib_wqueue_free(mtflush_io->wr_cq); - ib_wqueue_free(mtflush_io->rd_cq); + mtflush_ctx->~thread_sync_t(); + mtflush_ctx = NULL; - mtflush_io->wq = NULL; - mtflush_io->wr_cq = NULL; - mtflush_io->rd_cq = NULL; - mtflush_work_initialized = 0; - - /* Free heap */ - mem_heap_free(mtflush_io->wheap); - mem_heap_free(mtflush_io->rheap); - - mutex_exit(&mtflush_mtx); mutex_free(&mtflush_mtx); - mutex_free(&mtflush_io->thread_global_mtx); } /******************************************************************//** @@ -472,7 +486,6 @@ buf_mtflu_handler_init( ulint n_threads, /*!< in: Number of threads to create */ ulint wrk_cnt) /*!< in: Number of work items */ { - ulint i; mem_heap_t* mtflush_heap; mem_heap_t* mtflush_heap2; @@ -484,43 +497,10 @@ buf_mtflu_handler_init( mtflush_heap2 = mem_heap_create(0); ut_a(mtflush_heap2 != NULL); - mtflush_ctx = (thread_sync_t *)mem_heap_alloc(mtflush_heap, - sizeof(thread_sync_t)); - memset(mtflush_ctx, 0, sizeof(thread_sync_t)); - ut_a(mtflush_ctx != NULL); - mtflush_ctx->thread_data = (thread_data_t*)mem_heap_alloc( - mtflush_heap, sizeof(thread_data_t) * n_threads); - ut_a(mtflush_ctx->thread_data); - memset(mtflush_ctx->thread_data, 0, sizeof(thread_data_t) * n_threads); - - mtflush_ctx->n_threads = n_threads; - mtflush_ctx->wq = ib_wqueue_create(); - ut_a(mtflush_ctx->wq); - mtflush_ctx->wr_cq = ib_wqueue_create(); - ut_a(mtflush_ctx->wr_cq); - mtflush_ctx->rd_cq = ib_wqueue_create(); - ut_a(mtflush_ctx->rd_cq); - mtflush_ctx->wheap = mtflush_heap; - mtflush_ctx->rheap = mtflush_heap2; - - mutex_create(LATCH_ID_MTFLUSH_THREAD_MUTEX, &mtflush_ctx->thread_global_mtx); mutex_create(LATCH_ID_MTFLUSH_MUTEX, &mtflush_mtx); - /* Create threads for page-compression-flush */ - for(i=0; i < n_threads; i++) { - os_thread_id_t new_thread_id; - - mtflush_ctx->thread_data[i].wt_status = WTHR_INITIALIZED; - - os_thread_create( - mtflush_io_thread, - ((void *) mtflush_ctx), - &new_thread_id); - - mtflush_ctx->thread_data[i].wthread_id = new_thread_id; - } - - buf_mtflu_work_init(); + mtflush_ctx = new (mem_heap_zalloc(mtflush_heap, sizeof *mtflush_ctx)) + thread_sync_t(n_threads, mtflush_heap, mtflush_heap2); return((void *)mtflush_ctx); } diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index ea3c1ceccf9..4d68ad5ac51 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -182,7 +182,7 @@ buf_read_page_low( *err = fil_io( request, sync, page_id, page_size, 0, page_size.physical(), - dst, bpage, NULL); + dst, bpage); if (sync) { thd_wait_end(NULL); diff --git a/storage/innobase/data/data0data.cc b/storage/innobase/data/data0data.cc index 4b788c8952c..0ccf5868d14 100644 --- a/storage/innobase/data/data0data.cc +++ b/storage/innobase/data/data0data.cc @@ -31,7 +31,6 @@ Created 5/30/1994 Heikki Tuuri #include "data0data.ic" #endif -#ifndef UNIV_HOTBACKUP #include "rem0rec.h" #include "rem0cmp.h" #include "page0page.h" @@ -40,8 +39,6 @@ Created 5/30/1994 Heikki Tuuri #include "btr0cur.h" #include "row0upd.h" -#endif /* !UNIV_HOTBACKUP */ - #ifdef UNIV_DEBUG /** Dummy variable to catch access to uninitialized fields. In the debug version, dtuple_create() will make all fields of dtuple_t point @@ -54,7 +51,6 @@ ulint data_dummy; # endif /* !UNIV_DEBUG_VALGRIND */ #endif /* UNIV_DEBUG */ -#ifndef UNIV_HOTBACKUP /** Compare two data tuples. @param[in] tuple1 first data tuple @param[in] tuple2 second data tuple @@ -159,7 +155,6 @@ dump: return(TRUE); } -#endif /* !UNIV_HOTBACKUP */ #ifdef UNIV_DEBUG /**********************************************************//** @@ -255,7 +250,6 @@ dtuple_validate( } #endif /* UNIV_DEBUG */ -#ifndef UNIV_HOTBACKUP /*************************************************************//** Pretty prints a dfield value according to its data type. */ void @@ -848,4 +842,3 @@ dfield_t::clone( return(obj); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/data/data0type.cc b/storage/innobase/data/data0type.cc index 8fb3761531c..cad165d1489 100644 --- a/storage/innobase/data/data0type.cc +++ b/storage/innobase/data/data0type.cc @@ -31,7 +31,6 @@ Created 1/16/1996 Heikki Tuuri #include "data0type.ic" #endif -#ifndef UNIV_HOTBACKUP /* At the database startup we store the default-charset collation number of this MySQL installation to this global variable. If we have < 4.1.2 format column definitions, or records in the insert buffer, we use this @@ -78,7 +77,6 @@ dtype_get_at_most_n_mbchars( return(data_len); } -#endif /* UNIV_HOTBACKUP */ /*********************************************************************//** Checks if a data main type is a string type. Also a BLOB is considered a @@ -174,14 +172,11 @@ dtype_validate( ut_a((type->prtype & DATA_MYSQL_TYPE_MASK) < DATA_N_SYS_COLS); } -#ifndef UNIV_HOTBACKUP ut_a(dtype_get_mbminlen(type) <= dtype_get_mbmaxlen(type)); -#endif /* !UNIV_HOTBACKUP */ return(TRUE); } -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Prints a data type structure. */ void @@ -299,4 +294,3 @@ dtype_print( fprintf(stderr, " len %lu", (ulong) len); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/dict/dict0crea.cc b/storage/innobase/dict/dict0crea.cc index 31952424119..51c721bfc93 100644 --- a/storage/innobase/dict/dict0crea.cc +++ b/storage/innobase/dict/dict0crea.cc @@ -377,75 +377,6 @@ dict_build_table_def_step( return(err); } -/** Build a tablespace to store various objects. -@param[in,out] tablespace Tablespace object describing what to build. -@return DB_SUCCESS or error code. */ -dberr_t -dict_build_tablespace( - Tablespace* tablespace) -{ - dberr_t err = DB_SUCCESS; - mtr_t mtr; - ulint space = 0; - - ut_ad(mutex_own(&dict_sys->mutex)); - ut_ad(tablespace); - - DBUG_EXECUTE_IF("out_of_tablespace_disk", - return(DB_OUT_OF_FILE_SPACE);); - /* Get a new space id. */ - dict_hdr_get_new_id(NULL, NULL, &space, NULL, false); - if (space == ULINT_UNDEFINED) { - return(DB_ERROR); - } - tablespace->set_space_id(space); - - Datafile* datafile = tablespace->first_datafile(); - - /* We create a new generic empty tablespace. - We initially let it be 4 pages: - - page 0 is the fsp header and an extent descriptor page, - - page 1 is an ibuf bitmap page, - - page 2 is the first inode page, - - page 3 will contain the root of the clustered index of the - first table we create here. */ - - err = fil_ibd_create( - space, - tablespace->name(), - datafile->filepath(), - tablespace->flags(), - FIL_IBD_FILE_INITIAL_SIZE, - tablespace->encryption_mode(), - tablespace->key_id()); - - if (err != DB_SUCCESS) { - return(err); - } - - /* Update SYS_TABLESPACES and SYS_DATAFILES */ - err = dict_replace_tablespace_and_filepath( - tablespace->space_id(), tablespace->name(), - datafile->filepath(), tablespace->flags()); - if (err != DB_SUCCESS) { - os_file_delete(innodb_data_file_key, datafile->filepath()); - return(err); - } - - mtr_start(&mtr); - mtr.set_named_space(space); - - /* Once we allow temporary general tablespaces, we must do this; - mtr_set_log_mode(&mtr, MTR_LOG_NO_REDO); */ - ut_a(!FSP_FLAGS_GET_TEMPORARY(tablespace->flags())); - - fsp_header_init(space, FIL_IBD_FILE_INITIAL_SIZE, &mtr); - - mtr_commit(&mtr); - - return(err); -} - /** Builds a tablespace to contain a table, using file-per-table=1. @param[in,out] table Table to build in its own tablespace. @param[in] node Table create node @@ -461,7 +392,7 @@ dict_build_tablespace_for_table( bool needs_file_per_table; char* filepath; - ut_ad(mutex_own(&dict_sys->mutex) || dict_table_is_intrinsic(table)); + ut_ad(mutex_own(&dict_sys->mutex)); needs_file_per_table = DICT_TF2_FLAG_IS_SET(table, DICT_TF2_USE_FILE_PER_TABLE); @@ -473,6 +404,7 @@ dict_build_tablespace_for_table( DICT_TF2_FTS_AUX_HEX_NAME);); if (needs_file_per_table) { + ut_ad(!dict_table_is_temporary(table)); /* This table will need a new tablespace. */ ut_ad(dict_table_get_format(table) <= UNIV_FORMAT_MAX); @@ -493,23 +425,10 @@ dict_build_tablespace_for_table( table->space = static_cast<unsigned int>(space); /* Determine the tablespace flags. */ - bool is_temp = dict_table_is_temporary(table); - bool is_encrypted = dict_table_is_encrypted(table); bool has_data_dir = DICT_TF_HAS_DATA_DIR(table->flags); - ulint fsp_flags = dict_tf_to_fsp_flags(table->flags, - is_temp, - is_encrypted); - - /* Determine the full filepath */ - if (is_temp) { - /* Temporary table filepath contains a full path - and a filename without the extension. */ - ut_ad(table->dir_path_of_temp_table); - filepath = fil_make_filepath( - table->dir_path_of_temp_table, - NULL, IBD, false); + ulint fsp_flags = dict_tf_to_fsp_flags(table->flags); - } else if (has_data_dir) { + if (has_data_dir) { ut_ad(table->data_dir_path); filepath = fil_make_filepath( table->data_dir_path, @@ -545,7 +464,6 @@ dict_build_tablespace_for_table( mtr_start(&mtr); mtr.set_named_space(table->space); - dict_disable_redo_if_temporary(table, &mtr); bool ret = fsp_header_init(table->space, FIL_IBD_FILE_INITIAL_SIZE, @@ -556,23 +474,14 @@ dict_build_tablespace_for_table( return(DB_ERROR); } } else { - /* We do not need to build a tablespace for this table. It - is already built. Just find the correct tablespace ID. */ - - if (DICT_TF_HAS_SHARED_SPACE(table->flags)) { - ut_ad(table->tablespace != NULL); - - ut_ad(table->space == fil_space_get_id_by_name( - table->tablespace())); - } else if (dict_table_is_temporary(table)) { + if (dict_table_is_temporary(table)) { /* Use the shared temporary tablespace. Note: The temp tablespace supports all non-Compressed row formats whereas the system tablespace only supports Redundant and Compact */ ut_ad(dict_tf_get_rec_format(table->flags) != REC_FORMAT_COMPRESSED); - table->space = static_cast<uint32_t>( - srv_tmp_space.space_id()); + table->space = SRV_TMP_SPACE_ID; } else { /* Create in the system tablespace. */ ut_ad(table->space == srv_sys_space.space_id()); @@ -920,7 +829,7 @@ dict_build_index_def( dict_index_t* index, /*!< in/out: index */ trx_t* trx) /*!< in/out: InnoDB transaction handle */ { - ut_ad(mutex_own(&dict_sys->mutex) || dict_table_is_intrinsic(table)); + ut_ad(mutex_own(&dict_sys->mutex)); if (trx->table_id == 0) { /* Record only the first table id. */ @@ -930,18 +839,7 @@ dict_build_index_def( ut_ad((UT_LIST_GET_LEN(table->indexes) > 0) || dict_index_is_clust(index)); - if (!dict_table_is_intrinsic(table)) { - dict_hdr_get_new_id(NULL, &index->id, NULL, table, false); - } else { - /* Index are re-loaded in process of creation using id. - If same-id is used for all indexes only first index will always - be retrieved when expected is iterative return of all indexes*/ - if (UT_LIST_GET_LEN(table->indexes) > 0) { - index->id = UT_LIST_GET_LAST(table->indexes)->id + 1; - } else { - index->id = 1; - } - } + dict_hdr_get_new_id(NULL, &index->id, NULL, table, false); /* Inherit the space id from the table; we store all indexes of a table in the same tablespace */ @@ -1061,8 +959,7 @@ dict_create_index_tree_in_mem( mtr_t mtr; ulint page_no = FIL_NULL; - ut_ad(mutex_own(&dict_sys->mutex) - || dict_table_is_intrinsic(index->table)); + ut_ad(mutex_own(&dict_sys->mutex)); if (index->type == DICT_FTS) { /* FTS index does not need an index tree */ @@ -1175,8 +1072,7 @@ dict_drop_index_tree_in_mem( const dict_index_t* index, /*!< in: index */ ulint page_no) /*!< in: index page-no */ { - ut_ad(mutex_own(&dict_sys->mutex) - || dict_table_is_intrinsic(index->table)); + ut_ad(mutex_own(&dict_sys->mutex)); ut_ad(dict_table_is_temporary(index->table)); ulint root_page_no = page_no; @@ -1295,8 +1191,7 @@ dict_truncate_index_tree_in_mem( bool truncate; ulint space = index->space; - ut_ad(mutex_own(&dict_sys->mutex) - || dict_table_is_intrinsic(index->table)); + ut_ad(mutex_own(&dict_sys->mutex)); ut_ad(dict_table_is_temporary(index->table)); ulint type = index->type; @@ -1784,6 +1679,11 @@ dict_create_or_check_foreign_constraint_tables(void) return(DB_SUCCESS); } + if (srv_read_only_mode + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO) { + return(DB_READ_ONLY); + } + trx = trx_allocate_for_mysql(); trx_set_dict_operation(trx, TRX_DICT_OP_TABLE); @@ -1913,11 +1813,9 @@ dict_create_or_check_sys_virtual() return(DB_SUCCESS); } - if (srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO - || srv_read_only_mode) { - ib::error() << "Cannot create sys_virtual system tables;" - " running in read-only mode."; - return(DB_ERROR); + if (srv_read_only_mode + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO) { + return(DB_READ_ONLY); } trx = trx_allocate_for_mysql(); @@ -2112,7 +2010,7 @@ dict_foreign_def_get( char* fk_def = (char *)mem_heap_alloc(foreign->heap, 4*1024); const char* tbname; char tablebuf[MAX_TABLE_NAME_LEN + 1] = ""; - int i; + unsigned i; char* bufend; tbname = dict_remove_db_name(foreign->id); @@ -2130,7 +2028,7 @@ dict_foreign_def_get( strlen(foreign->foreign_col_names[i]), trx->mysql_thd); strcat(fk_def, buf); - if (i < foreign->n_fields-1) { + if (i < static_cast<unsigned>(foreign->n_fields-1)) { strcat(fk_def, (char *)","); } } @@ -2154,7 +2052,7 @@ dict_foreign_def_get( trx->mysql_thd); buf[bufend - buf] = '\0'; strcat(fk_def, buf); - if (i < foreign->n_fields-1) { + if (i < (uint)foreign->n_fields-1) { strcat(fk_def, (char *)","); } } @@ -2203,7 +2101,6 @@ Add a foreign key definition to the data dictionary tables. dberr_t dict_create_add_foreign_to_dictionary( /*==================================*/ - dict_table_t* table, /*!< in: table */ const char* name, /*!< in: table name */ const dict_foreign_t* foreign,/*!< in: foreign key */ trx_t* trx) /*!< in/out: dictionary transaction */ @@ -2245,8 +2142,7 @@ dict_create_add_foreign_to_dictionary( char* fk_def; innobase_convert_name(tablename, MAX_TABLE_NAME_LEN, - table->name.m_name, strlen(table->name.m_name), - trx->mysql_thd); + name, strlen(name), trx->mysql_thd); innobase_convert_name(buf, MAX_TABLE_NAME_LEN, foreign->id, strlen(foreign->id), trx->mysql_thd); @@ -2277,8 +2173,7 @@ dict_create_add_foreign_to_dictionary( char* fk_def; innobase_convert_name(tablename, MAX_TABLE_NAME_LEN, - table->name.m_name, strlen(table->name.m_name), - trx->mysql_thd); + name, strlen(name), trx->mysql_thd); innobase_convert_name(buf, MAX_TABLE_NAME_LEN, foreign->id, strlen(foreign->id), trx->mysql_thd); fk_def = dict_foreign_def_get((dict_foreign_t*)foreign, trx); @@ -2509,12 +2404,7 @@ dict_create_add_foreigns_to_dictionary( dict_foreign_t* foreign; dberr_t error; - ut_ad(mutex_own(&dict_sys->mutex) - || dict_table_is_intrinsic(table)); - - if (dict_table_is_intrinsic(table)) { - goto exit_loop; - } + ut_ad(mutex_own(&dict_sys->mutex)); if (NULL == dict_table_get_low("SYS_FOREIGN")) { @@ -2532,7 +2422,7 @@ dict_create_add_foreigns_to_dictionary( ut_ad(foreign->id != NULL); error = dict_create_add_foreign_to_dictionary( - (dict_table_t*)table, table->name.m_name, foreign, trx); + table->name.m_name, foreign, trx); if (error != DB_SUCCESS) { @@ -2540,7 +2430,6 @@ dict_create_add_foreigns_to_dictionary( } } -exit_loop: trx->op_info = "committing foreign key definitions"; if (trx_is_started(trx)) { @@ -2579,9 +2468,15 @@ dict_create_or_check_sys_tablespace(void) if (sys_tablespaces_err == DB_SUCCESS && sys_datafiles_err == DB_SUCCESS) { + srv_sys_tablespaces_open = true; return(DB_SUCCESS); } + if (srv_read_only_mode + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO) { + return(DB_READ_ONLY); + } + trx = trx_allocate_for_mysql(); trx_set_dict_operation(trx, TRX_DICT_OP_TABLE); @@ -2654,6 +2549,7 @@ dict_create_or_check_sys_tablespace(void) if (err == DB_SUCCESS) { ib::info() << "Tablespace and datafile system tables created."; + srv_sys_tablespaces_open = true; } /* Note: The master thread has not been started at this point. */ @@ -2796,14 +2692,6 @@ dict_table_assign_new_id( dict_table_t* table, trx_t* trx) { - if (dict_table_is_intrinsic(table)) { - /* There is no significance of this table->id (if table is - intrinsic) so assign it default instead of something meaningful - to avoid confusion.*/ - table->id = ULINT_UNDEFINED; - } else { - dict_hdr_get_new_id(&table->id, NULL, NULL, table, false); - } - + dict_hdr_get_new_id(&table->id, NULL, NULL, table, false); trx->table_id = table->id; } diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 3d527c6b017..943b79c3514 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -55,7 +55,6 @@ Issue a warning that the row is too big. */ void ib_warn_row_too_big(const dict_table_t* table); -#ifndef UNIV_HOTBACKUP #include "btr0btr.h" #include "btr0cur.h" #include "btr0sea.h" @@ -502,21 +501,15 @@ dict_table_close( indexes after an aborted online index creation */ { - if (!dict_locked && !dict_table_is_intrinsic(table)) { + if (!dict_locked) { mutex_enter(&dict_sys->mutex); } - ut_ad(mutex_own(&dict_sys->mutex) || dict_table_is_intrinsic(table)); + ut_ad(mutex_own(&dict_sys->mutex)); ut_a(table->get_ref_count() > 0); table->release(); - /* Intrinsic table is not added to dictionary cache so skip other - cache specific actions. */ - if (dict_table_is_intrinsic(table)) { - return; - } - /* Force persistent stats re-read upon next open of the table so that FLUSH TABLE can be used to forcibly fetch stats from disk if they have been manually modified. We reset table->stat_initialized @@ -557,7 +550,6 @@ dict_table_close( } } } -#endif /* !UNIV_HOTBACKUP */ /********************************************************************//** Closes the only open handle to a table and drops a table while assuring @@ -660,41 +652,6 @@ dict_table_get_col_name( return(s); } -/**********************************************************************//** -Returns a column's name. -@return column name. NOTE: not guaranteed to stay valid if table is -modified in any way (columns added, etc.). */ -UNIV_INTERN -const char* -dict_table_get_col_name_for_mysql( -/*==============================*/ - const dict_table_t* table, /*!< in: table */ - const char* col_name)/*! in: MySQL table column name */ -{ - ulint i; - const char* s; - - ut_ad(table); - ut_ad(col_name); - ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); - - s = table->col_names; - if (s) { - /* If we have many virtual columns MySQL key_part->fieldnr - could be larger than number of columns in InnoDB table - when creating new indexes. */ - for (i = 0; i < table->n_def; i++) { - - if (!innobase_strcasecmp(s, col_name)) { - break; /* Found */ - } - s += strlen(s) + 1; - } - } - - return(s); -} - /** Returns a virtual column's name. @param[in] table target table @param[in] col_nr virtual column number (nth virtual column) @@ -794,7 +751,6 @@ dict_table_get_nth_v_col_mysql( return(dict_table_get_nth_v_col(table, i)); } -#ifndef UNIV_HOTBACKUP /** Allocate and init the autoinc latch of a given table. This function must not be called concurrently on the same table object. @param[in,out] table_void table whose autoinc latch to create */ @@ -823,7 +779,6 @@ dict_index_zip_pad_alloc( mutex_create(LATCH_ID_ZIP_PAD_MUTEX, index->zip_pad.mutex); } - /********************************************************************//** Acquire the autoinc lock. */ void @@ -851,20 +806,6 @@ dict_index_zip_pad_lock( mutex_enter(index->zip_pad.mutex); } - -/********************************************************************//** -Unconditionally set the autoinc counter. */ -void -dict_table_autoinc_initialize( -/*==========================*/ - dict_table_t* table, /*!< in/out: table */ - ib_uint64_t value) /*!< in: next value to assign to a row */ -{ - ut_ad(dict_table_autoinc_own(table)); - - table->autoinc = value; -} - /** Get all the FTS indexes on a table. @param[in] table table @param[out] indexes all FTS indexes on this table @@ -890,75 +831,6 @@ dict_table_get_all_fts_indexes( return(ib_vector_size(indexes)); } -/** Store autoinc value when the table is evicted. -@param[in] table table evicted */ -void -dict_table_autoinc_store( - const dict_table_t* table) -{ - ut_ad(mutex_own(&dict_sys->mutex)); - - if (table->autoinc != 0) { - ut_ad(dict_sys->autoinc_map->find(table->id) - == dict_sys->autoinc_map->end()); - - dict_sys->autoinc_map->insert( - std::pair<table_id_t, ib_uint64_t>( - table->id, table->autoinc)); - } -} - -/** Restore autoinc value when the table is loaded. -@param[in] table table loaded */ -void -dict_table_autoinc_restore( - dict_table_t* table) -{ - ut_ad(mutex_own(&dict_sys->mutex)); - - autoinc_map_t::iterator it; - it = dict_sys->autoinc_map->find(table->id); - - if (it != dict_sys->autoinc_map->end()) { - table->autoinc = it->second; - ut_ad(table->autoinc != 0); - - dict_sys->autoinc_map->erase(it); - } -} - -/********************************************************************//** -Reads the next autoinc value (== autoinc counter value), 0 if not yet -initialized. -@return value for a new row, or 0 */ -ib_uint64_t -dict_table_autoinc_read( -/*====================*/ - const dict_table_t* table) /*!< in: table */ -{ - ut_ad(dict_table_autoinc_own(table)); - - return(table->autoinc); -} - -/********************************************************************//** -Updates the autoinc counter if the value supplied is greater than the -current value. */ -void -dict_table_autoinc_update_if_greater( -/*=================================*/ - - dict_table_t* table, /*!< in/out: table */ - ib_uint64_t value) /*!< in: value which was assigned to a row */ -{ - ut_ad(dict_table_autoinc_own(table)); - - if (value > table->autoinc) { - - table->autoinc = value; - } -} - /********************************************************************//** Release the autoinc lock. */ void @@ -968,7 +840,6 @@ dict_table_autoinc_unlock( { mutex_exit(table->autoinc_mutex); } -#endif /* !UNIV_HOTBACKUP */ /** Looks for column n in an index. @param[in] index index @@ -1027,7 +898,6 @@ dict_index_get_nth_col_or_prefix_pos( return(ULINT_UNDEFINED); } -#ifndef UNIV_HOTBACKUP /** Returns TRUE if the index contains a column or a prefix of that column. @param[in] index index @param[in] n column number @@ -1253,8 +1123,6 @@ dict_init(void) } mutex_create(LATCH_ID_DICT_FOREIGN_ERR, &dict_foreign_err_mutex); - - dict_sys->autoinc_map = new autoinc_map_t(); } /**********************************************************************//** @@ -1368,7 +1236,6 @@ dict_table_open_on_name( DBUG_RETURN(table); } -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Adds system columns to a table object. */ @@ -1388,18 +1255,12 @@ dict_table_add_system_columns( (so that they can be indexed by the numerical value of DATA_ROW_ID, etc.) and as the last columns of the table memory object. The clustered index will not always physically contain all system - columns. - Intrinsic table don't need DB_ROLL_PTR as UNDO logging is turned off - for these tables. */ + columns. */ dict_mem_table_add_col(table, heap, "DB_ROW_ID", DATA_SYS, DATA_ROW_ID | DATA_NOT_NULL, DATA_ROW_ID_LEN); -#if (DATA_ITT_N_SYS_COLS != 2) -#error "DATA_ITT_N_SYS_COLS != 2" -#endif - #if DATA_ROW_ID != 0 #error "DATA_ROW_ID != 0" #endif @@ -1410,23 +1271,20 @@ dict_table_add_system_columns( #error "DATA_TRX_ID != 1" #endif - if (!dict_table_is_intrinsic(table)) { - dict_mem_table_add_col(table, heap, "DB_ROLL_PTR", DATA_SYS, - DATA_ROLL_PTR | DATA_NOT_NULL, - DATA_ROLL_PTR_LEN); + dict_mem_table_add_col(table, heap, "DB_ROLL_PTR", DATA_SYS, + DATA_ROLL_PTR | DATA_NOT_NULL, + DATA_ROLL_PTR_LEN); #if DATA_ROLL_PTR != 2 #error "DATA_ROLL_PTR != 2" #endif - /* This check reminds that if a new system column is added to - the program, it should be dealt with here */ + /* This check reminds that if a new system column is added to + the program, it should be dealt with here */ #if DATA_N_SYS_COLS != 3 #error "DATA_N_SYS_COLS != 3" #endif - } } -#ifndef UNIV_HOTBACKUP /** Mark if table has big rows. @param[in,out] table table handler */ void @@ -1526,8 +1384,6 @@ dict_table_add_to_cache( UT_LIST_ADD_FIRST(dict_sys->table_non_LRU, table); } - dict_table_autoinc_restore(table); - ut_ad(dict_lru_validate()); dict_sys->size += mem_heap_get_size(table->heap) @@ -1750,6 +1606,7 @@ struct dict_foreign_remove_partial if (table != NULL) { table->referenced_set.erase(foreign); } + dict_foreign_free(foreign); } }; @@ -1839,25 +1696,17 @@ dict_table_rename_in_cache( ib::info() << "Delete of " << filepath << " failed."; } - ut_free(filepath); } else if (dict_table_is_file_per_table(table)) { - if (table->dir_path_of_temp_table != NULL) { - ib::error() << "Trying to rename a TEMPORARY TABLE " - << old_name - << " ( " << table->dir_path_of_temp_table - << " )"; - return(DB_ERROR); - } - char* new_path = NULL; char* old_path = fil_space_get_first_path(table->space); + ut_ad(!dict_table_is_temporary(table)); + if (DICT_TF_HAS_DATA_DIR(table->flags)) { new_path = os_file_make_new_pathname( old_path, new_name); - err = RemoteDatafile::create_link_file( new_name, new_path); @@ -2262,10 +2111,6 @@ dict_table_remove_from_cache_low( ut_ad(dict_lru_validate()); - if (lru_evict) { - dict_table_autoinc_store(table); - } - if (lru_evict && table->drop_aborted) { /* Do as dict_table_try_drop_aborted() does. */ @@ -2654,7 +2499,7 @@ dict_index_add_to_cache_w_vcol( ulint i; ut_ad(index); - ut_ad(mutex_own(&dict_sys->mutex) || dict_table_is_intrinsic(table)); + ut_ad(mutex_own(&dict_sys->mutex)); ut_ad(index->n_def == index->n_fields); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); ut_ad(!dict_index_is_online_ddl(index)); @@ -2687,9 +2532,10 @@ dict_index_add_to_cache_w_vcol( new_index->n_fields = new_index->n_def; new_index->trx_id = index->trx_id; new_index->set_committed(index->is_committed()); - new_index->allow_duplicates = index->allow_duplicates; new_index->nulls_equal = index->nulls_equal; +#ifdef MYSQL_INDEX_DISABLE_AHI new_index->disable_ahi = index->disable_ahi; +#endif if (dict_index_too_big_for_tree(table, new_index, strict)) { @@ -2771,39 +2617,7 @@ dict_index_add_to_cache_w_vcol( rw_lock_create(index_tree_rw_lock_key, &new_index->lock, SYNC_INDEX_TREE); - /* Intrinsic table are not added to dictionary cache instead are - cached to session specific thread cache. */ - if (!dict_table_is_intrinsic(table)) { - dict_sys->size += mem_heap_get_size(new_index->heap); - } - - /* Check if key part of the index is unique. */ - if (dict_table_is_intrinsic(table)) { - - new_index->rec_cache.fixed_len_key = true; - for (i = 0; i < new_index->n_uniq; i++) { - - const dict_field_t* field; - field = dict_index_get_nth_field(new_index, i); - - if (!field->fixed_len) { - new_index->rec_cache.fixed_len_key = false; - break; - } - } - - new_index->rec_cache.key_has_null_cols = false; - for (i = 0; i < new_index->n_uniq; i++) { - - const dict_field_t* field; - field = dict_index_get_nth_field(new_index, i); - - if (!(field->col->prtype & DATA_NOT_NULL)) { - new_index->rec_cache.key_has_null_cols = true; - break; - } - } - } + dict_sys->size += mem_heap_get_size(new_index->heap); dict_mem_index_free(index); @@ -2926,7 +2740,6 @@ dict_index_remove_from_cache_low( size = mem_heap_get_size(index->heap); - ut_ad(!dict_table_is_intrinsic(table)); ut_ad(dict_sys->size >= size); dict_sys->size -= size; @@ -2963,7 +2776,7 @@ dict_index_find_cols( ut_ad(table != NULL && index != NULL); ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); - ut_ad(mutex_own(&dict_sys->mutex) || dict_table_is_intrinsic(table)); + ut_ad(mutex_own(&dict_sys->mutex)); for (ulint i = 0; i < index->n_fields; i++) { ulint j; @@ -3045,7 +2858,6 @@ found: return(TRUE); } -#endif /* !UNIV_HOTBACKUP */ /*******************************************************************//** Adds a column to index. */ @@ -3128,7 +2940,6 @@ dict_index_add_col( } } -#ifndef UNIV_HOTBACKUP /*******************************************************************//** Copies fields contained in index2 to index1. */ static @@ -3285,7 +3096,7 @@ dict_index_build_internal_clust( ut_ad(dict_index_is_clust(index)); ut_ad(!dict_index_is_ibuf(index)); - ut_ad(mutex_own(&dict_sys->mutex) || dict_table_is_intrinsic(table)); + ut_ad(mutex_own(&dict_sys->mutex)); ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); /* Create a new index object with certainly enough fields */ @@ -3342,7 +3153,6 @@ dict_index_build_internal_clust( new_index, table, dict_table_get_sys_col(table, DATA_TRX_ID), 0); - for (i = 0; i < trx_id_pos; i++) { ulint fixed_size = dict_col_get_fixed_size( @@ -3379,16 +3189,9 @@ dict_index_build_internal_clust( } } - /* UNDO logging is turned-off for intrinsic table and so - DATA_ROLL_PTR system columns are not added as default system - columns to such tables. */ - if (!dict_table_is_intrinsic(table)) { - - dict_index_add_col( - new_index, table, - dict_table_get_sys_col(table, DATA_ROLL_PTR), - 0); - } + dict_index_add_col( + new_index, table, + dict_table_get_sys_col(table, DATA_ROLL_PTR), 0); /* Remember the table columns already contained in new_index */ indexed = static_cast<ibool*>( @@ -3451,7 +3254,7 @@ dict_index_build_internal_non_clust( ut_ad(table && index); ut_ad(!dict_index_is_clust(index)); ut_ad(!dict_index_is_ibuf(index)); - ut_ad(mutex_own(&dict_sys->mutex) || dict_table_is_intrinsic(table)); + ut_ad(mutex_own(&dict_sys->mutex)); ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); /* The clustered index should be the first in the list of indexes */ @@ -3667,7 +3470,6 @@ dict_foreign_find( return(NULL); } - /*********************************************************************//** Tries to find an index whose first fields are the columns in the array, in the same order and is not marked for deletion and is not the same @@ -3716,7 +3518,6 @@ dict_foreign_find_index( if (types_idx != index && !(index->type & DICT_FTS) && !dict_index_is_spatial(index) - && !dict_index_has_virtual(index) && !index->to_be_dropped && dict_foreign_qualify_index( table, col_names, columns, n_cols, @@ -3847,14 +3648,12 @@ dict_foreign_add_to_cache( } if (for_in_cache) { - /* Free the foreign object */ dict_foreign_free(foreign); } else { for_in_cache = foreign; } - if (ref_table && !for_in_cache->referenced_table) { ulint index_error; ulint err_col; @@ -3877,10 +3676,9 @@ dict_foreign_add_to_cache( "referenced table do not match" " the ones in table."); - if (for_in_cache == foreign) { - mem_heap_free(foreign->heap); - } - + if (for_in_cache == foreign) { + dict_foreign_free(foreign); + } DBUG_RETURN(DB_CANNOT_ADD_CONSTRAINT); } @@ -3934,7 +3732,8 @@ dict_foreign_add_to_cache( elements removed must be one */ } - mem_heap_free(foreign->heap); + + dict_foreign_free(foreign); } DBUG_RETURN(DB_CANNOT_ADD_CONSTRAINT); @@ -4226,12 +4025,28 @@ dict_scan_col( break; } } + + for (i = 0; i < dict_table_get_n_v_cols(table); i++) { + + const char* col_name = dict_table_get_v_col_name( + table, i); + + if (0 == innobase_strcasecmp(col_name, *name)) { + /* Found */ + dict_v_col_t * vcol; + *success = TRUE; + vcol = dict_table_get_nth_v_col(table, i); + *column = &vcol->m_col; + strcpy((char*) *name, col_name); + + break; + } + } } return(ptr); } - /*********************************************************************//** Open a table from its database and table name, this is currently used by foreign constraint parser to get the referenced table. @@ -5211,9 +5026,7 @@ col_loop1: for (i = 0; i < foreign->n_fields; i++) { foreign->foreign_col_names[i] = mem_heap_strdup( - foreign->heap, - dict_table_get_col_name(table, - dict_col_get_no(columns[i]))); + foreign->heap, column_names[i]); } ptr = dict_scan_table_name(cs, ptr, &referenced_table, name, @@ -5531,6 +5344,7 @@ try_find_index: ref_column_names, i, foreign->foreign_index, TRUE, FALSE, &index_error, &err_col, &err_index); + if (!index) { mutex_enter(&dict_foreign_err_mutex); dict_foreign_error_report_low(ef, create_name); @@ -6527,13 +6341,10 @@ dict_set_merge_threshold_all_debug( } #endif /* UNIV_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ -/**********************************************************************//** -Inits dict_ind_redundant. */ +/** Initialize dict_ind_redundant. */ void -dict_ind_init(void) -/*===============*/ +dict_ind_init() { dict_table_t* table; @@ -6551,17 +6362,11 @@ dict_ind_init(void) dict_ind_redundant->cached = TRUE; } -#ifndef UNIV_HOTBACKUP -/**********************************************************************//** -Frees dict_ind_redundant. */ -static +/** Free dict_ind_redundant. */ void -dict_ind_free(void) -/*===============*/ +dict_ind_free() { - dict_table_t* table; - - table = dict_ind_redundant->table; + dict_table_t* table = dict_ind_redundant->table; dict_mem_index_free(dict_ind_redundant); dict_ind_redundant = NULL; dict_mem_table_free(table); @@ -7084,8 +6889,6 @@ dict_close(void) therefore we don't delete the individual elements. */ hash_table_free(dict_sys->table_id_hash); - dict_ind_free(); - mutex_free(&dict_sys->mutex); rw_lock_free(dict_operation_lock); @@ -7095,8 +6898,6 @@ dict_close(void) mutex_free(&dict_foreign_err_mutex); - delete dict_sys->autoinc_map; - ut_ad(dict_sys->size == 0); ut_free(dict_sys); @@ -7237,8 +7038,6 @@ dict_foreign_qualify_index( field = dict_index_get_nth_field(index, i); col_no = dict_col_get_no(field->col); - ut_ad(!dict_col_is_virtual(field->col)); - if (field->prefix_len != 0) { /* We do not accept column prefix indexes here */ @@ -7264,6 +7063,15 @@ dict_foreign_qualify_index( ? col_names[col_no] : dict_table_get_col_name(table, col_no); + if (dict_col_is_virtual(field->col)) { + for (ulint j = 0; j < table->n_v_def; j++) { + col_name = dict_table_get_v_col_name(table, j); + if (innobase_strcasecmp(field->name,col_name) == 0) { + break; + } + } + } + if (0 != innobase_strcasecmp(columns[i], col_name)) { return(false); } @@ -7409,7 +7217,6 @@ dict_index_zip_failure( dict_index_zip_pad_unlock(index); } - /*********************************************************************//** Return the optimal page size, for which page will likely compress. @return page size beyond which page might not compress */ @@ -7442,78 +7249,6 @@ dict_index_zip_pad_optimal_page_size( return(ut_max(sz, min_sz)); } -/** Convert a 32 bit integer table flags to the 32 bit FSP Flags. -Fsp Flags are written into the tablespace header at the offset -FSP_SPACE_FLAGS and are also stored in the fil_space_t::flags field. -The following chart shows the translation of the low order bit. -Other bits are the same. - Low order bit - | REDUNDANT | COMPACT | COMPRESSED | DYNAMIC -dict_table_t::flags | 0 | 1 | 1 | 1 -fil_space_t::flags | 0 | 0 | 1 | 1 -@param[in] table_flags dict_table_t::flags -@param[in] is_temp whether the tablespace is temporary -@param[in] is_encrypted whether the tablespace is encrypted -@return tablespace flags (fil_space_t::flags) */ -ulint -dict_tf_to_fsp_flags( - ulint table_flags, - bool is_temp, - bool is_encrypted) -{ - DBUG_EXECUTE_IF("dict_tf_to_fsp_flags_failure", - return(ULINT_UNDEFINED);); - - bool has_atomic_blobs = - DICT_TF_HAS_ATOMIC_BLOBS(table_flags); - page_size_t page_size = dict_tf_get_page_size(table_flags); - bool has_data_dir = DICT_TF_HAS_DATA_DIR(table_flags); - bool is_shared = DICT_TF_HAS_SHARED_SPACE(table_flags); - bool page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); - ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); - ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); - - ut_ad(!page_size.is_compressed() || has_atomic_blobs); - - /* General tablespaces that are not compressed do not get the - flags for dynamic row format (POST_ANTELOPE & ATOMIC_BLOBS) */ - if (is_shared && !page_size.is_compressed()) { - has_atomic_blobs = false; - } - - ulint fsp_flags = fsp_flags_init(page_size, - has_atomic_blobs, - has_data_dir, - is_shared, - is_temp, - 0, - 0, - 0, - is_encrypted); - - /* In addition, tablespace flags also contain if the page - compression is used for this table. */ - if (page_compression) { - fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION(fsp_flags, page_compression); - } - - /* In addition, tablespace flags also contain page compression level - if page compression is used for this table. */ - if (page_compression && page_compression_level) { - fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(fsp_flags, page_compression_level); - } - - /* In addition, tablespace flags also contain flag if atomic writes - is used for this table */ - if (atomic_writes) { - fsp_flags |= FSP_FLAGS_SET_ATOMIC_WRITES(fsp_flags, atomic_writes); - } - - ut_ad(fsp_flags_is_valid(fsp_flags)); - - return(fsp_flags); -} - /*************************************************************//** Convert table flag to row format string. @return row format name. */ @@ -7625,7 +7360,6 @@ dict_space_get_id( return(id); } -#endif /* !UNIV_HOTBACKUP */ /** Determine the extent size (in pages) for the given table @param[in] table the table whose extent size is being diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index f38ad85e903..db14983cd8f 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, MariaDB Corporation. +Copyright (c) 2016, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -493,7 +493,7 @@ err_len: } /* This receives a dict_foreign_t* that points to a stack variable. - So mem_heap_free(foreign->heap) is not used as elsewhere. + So dict_foreign_free(foreign) is not used as elsewhere. Since the heap used here is freed elsewhere, foreign->heap is not assigned. */ foreign->id = mem_heap_strdupl(heap, (const char*) field, len); @@ -811,102 +811,6 @@ dict_get_first_path( return(filepath); } -/** Gets the space name from SYS_TABLESPACES for a given space ID. -@param[in] space_id Tablespace ID -@param[in] callers_heap A heap to allocate from, may be NULL -@return Tablespace name (caller is responsible to free it) -@retval NULL if no dictionary entry was found. */ -static -char* -dict_space_get_name( - ulint space_id, - mem_heap_t* callers_heap) -{ - mtr_t mtr; - dict_table_t* sys_tablespaces; - dict_index_t* sys_index; - dtuple_t* tuple; - dfield_t* dfield; - byte* buf; - btr_pcur_t pcur; - const rec_t* rec; - const byte* field; - ulint len; - char* space_name = NULL; - mem_heap_t* heap = mem_heap_create(1024); - - ut_ad(mutex_own(&dict_sys->mutex)); - - sys_tablespaces = dict_table_get_low("SYS_TABLESPACES"); - if (sys_tablespaces == NULL) { - ut_a(!srv_sys_tablespaces_open); - return(NULL); - } - - sys_index = UT_LIST_GET_FIRST(sys_tablespaces->indexes); - - ut_ad(!dict_table_is_comp(sys_tablespaces)); - ut_ad(name_of_col_is(sys_tablespaces, sys_index, - DICT_FLD__SYS_TABLESPACES__SPACE, "SPACE")); - ut_ad(name_of_col_is(sys_tablespaces, sys_index, - DICT_FLD__SYS_TABLESPACES__NAME, "NAME")); - - tuple = dtuple_create(heap, 1); - dfield = dtuple_get_nth_field(tuple, DICT_FLD__SYS_TABLESPACES__SPACE); - - buf = static_cast<byte*>(mem_heap_alloc(heap, 4)); - mach_write_to_4(buf, space_id); - - dfield_set_data(dfield, buf, 4); - dict_index_copy_types(tuple, sys_index, 1); - - mtr_start(&mtr); - - btr_pcur_open_on_user_rec(sys_index, tuple, PAGE_CUR_GE, - BTR_SEARCH_LEAF, &pcur, &mtr); - - rec = btr_pcur_get_rec(&pcur); - - /* Get the tablespace name from this SYS_TABLESPACES record. */ - if (btr_pcur_is_on_user_rec(&pcur)) { - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_TABLESPACES__SPACE, &len); - ut_a(len == 4); - - if (space_id == mach_read_from_4(field)) { - /* A record for this space ID was found. */ - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_TABLESPACES__NAME, &len); - - ut_ad(len > 0); - ut_ad(len < OS_FILE_MAX_PATH); - - if (len > 0 && len != UNIV_SQL_NULL) { - /* Found a tablespace name. */ - if (callers_heap == NULL) { - space_name = mem_strdupl( - reinterpret_cast< - const char*>(field), - len); - } else { - space_name = mem_heap_strdupl( - callers_heap, - reinterpret_cast< - const char*>(field), - len); - } - ut_ad(space_name); - } - } - } - - btr_pcur_close(&pcur); - mtr_commit(&mtr); - mem_heap_free(heap); - - return(space_name); -} - /** Update the record for space_id in SYS_TABLESPACES to this filepath. @param[in] space_id Tablespace ID @param[in] filepath Tablespace filepath @@ -1144,106 +1048,6 @@ dict_sys_tablespaces_rec_read( return(true); } -/** Load and check each general tablespace mentioned in the SYS_TABLESPACES. -Ignore system and file-per-table tablespaces. -If it is valid, add it to the file_system list. -@param[in] validate true when the previous shutdown was not clean -@return the highest space ID found. */ -UNIV_INLINE -ulint -dict_check_sys_tablespaces( - bool validate) -{ - ulint max_space_id = 0; - btr_pcur_t pcur; - const rec_t* rec; - mtr_t mtr; - - DBUG_ENTER("dict_check_sys_tablespaces"); - - ut_ad(rw_lock_own(dict_operation_lock, RW_LOCK_X)); - ut_ad(mutex_own(&dict_sys->mutex)); - - /* Before traversing it, let's make sure we have - SYS_TABLESPACES and SYS_DATAFILES loaded. */ - dict_table_get_low("SYS_TABLESPACES"); - dict_table_get_low("SYS_DATAFILES"); - - mtr_start(&mtr); - - for (rec = dict_startscan_system(&pcur, &mtr, SYS_TABLESPACES); - rec != NULL; - rec = dict_getnext_system(&pcur, &mtr)) - { - char space_name[NAME_LEN]; - ulint space_id = 0; - ulint fsp_flags; - - if (!dict_sys_tablespaces_rec_read(rec, &space_id, - space_name, &fsp_flags)) { - continue; - } - - /* Ignore system and file-per-table tablespaces. */ - if (is_system_tablespace(space_id) - || !fsp_is_shared_tablespace(fsp_flags)) { - continue; - } - - /* Ignore tablespaces that already are in the tablespace - cache. */ - if (fil_space_for_table_exists_in_mem( - space_id, space_name, false, true, NULL, 0, NULL)) { - /* Recovery can open a datafile that does not - match SYS_DATAFILES. If they don't match, update - SYS_DATAFILES. */ - char *dict_path = dict_get_first_path(space_id); - char *fil_path = fil_space_get_first_path(space_id); - if (dict_path && fil_path - && strcmp(dict_path, fil_path)) { - dict_update_filepath(space_id, fil_path); - } - ut_free(dict_path); - ut_free(fil_path); - continue; - } - - /* Set the expected filepath from the data dictionary. - If the file is found elsewhere (from an ISL or the default - location) or this path is the same file but looks different, - fil_ibd_open() will update the dictionary with what is - opened. */ - char* filepath = dict_get_first_path(space_id); - - validate = true; /* Encryption */ - - /* Check that the .ibd file exists. */ - dberr_t err = fil_ibd_open( - validate, - !srv_read_only_mode && srv_log_file_size != 0, - FIL_TYPE_TABLESPACE, - space_id, - fsp_flags, - space_name, - filepath, - NULL); - - if (err != DB_SUCCESS) { - ib::warn() << "Ignoring tablespace " - << id_name_t(space_name) - << " because it could not be opened."; - } - - max_space_id = ut_max(max_space_id, space_id); - - ut_free(filepath); - } - - mtr_commit(&mtr); - - DBUG_RETURN(max_space_id); -} - /** Read and return 5 integer fields from a SYS_TABLES record. @param[in] rec A record of SYS_TABLES @param[in] name Table Name, the same as SYS_TABLES.NAME @@ -1363,7 +1167,6 @@ dict_check_sys_tables( rec = dict_getnext_system(&pcur, &mtr)) { const byte* field; ulint len; - char* space_name; table_name_t table_name; table_id_t table_id; ulint space_id; @@ -1395,39 +1198,24 @@ dict_check_sys_tables( } if (flags2 & DICT_TF2_DISCARDED) { - ib::info() << "Ignoring tablespace " << table_name + ib::info() << "Ignoring tablespace for " << table_name << " because the DISCARD flag is set ."; ut_free(table_name.m_name); continue; } /* If the table is not a predefined tablespace then it must - be in a file-per-table or shared tablespace. + be in a file-per-table tablespace. Note that flags2 is not available for REDUNDANT tables, so don't check those. */ - ut_ad(DICT_TF_HAS_SHARED_SPACE(flags) - || !DICT_TF_GET_COMPACT(flags) + ut_ad(!DICT_TF_GET_COMPACT(flags) || flags2 & DICT_TF2_USE_FILE_PER_TABLE); - /* Look up the tablespace name in the data dictionary if this - is a shared tablespace. For file-per-table, the table_name - and the tablespace_name are the same. - Some hidden tables like FTS AUX tables may not be found in - the dictionary since they can always be found in the default - location. If so, then dict_space_get_name() will return NULL, - the space name must be the table_name, and the filepath can be - discovered in the default location.*/ - char* shared_space_name = dict_space_get_name(space_id, NULL); - space_name = shared_space_name == NULL - ? table_name.m_name - : shared_space_name; - /* Now that we have the proper name for this tablespace, - whether it is a shared tablespace or a single table - tablespace, look to see if it is already in the tablespace - cache. */ + look to see if it is already in the tablespace cache. */ if (fil_space_for_table_exists_in_mem( - space_id, space_name, false, true, NULL, 0, NULL)) { + space_id, table_name.m_name, + false, true, NULL, 0, NULL, flags)) { /* Recovery can open a datafile that does not match SYS_DATAFILES. If they don't match, update SYS_DATAFILES. */ @@ -1440,7 +1228,6 @@ dict_check_sys_tables( ut_free(dict_path); ut_free(fil_path); ut_free(table_name.m_name); - ut_free(shared_space_name); continue; } @@ -1452,33 +1239,26 @@ dict_check_sys_tables( char* filepath = dict_get_first_path(space_id); /* Check that the .ibd file exists. */ - bool is_temp = flags2 & DICT_TF2_TEMPORARY; - bool is_encrypted = flags2 & DICT_TF2_ENCRYPTION; - ulint fsp_flags = dict_tf_to_fsp_flags(flags, - is_temp, - is_encrypted); validate = true; /* Encryption */ dberr_t err = fil_ibd_open( validate, !srv_read_only_mode && srv_log_file_size != 0, FIL_TYPE_TABLESPACE, - space_id, - fsp_flags, - space_name, + space_id, dict_tf_to_fsp_flags(flags), + table_name.m_name, filepath, NULL); if (err != DB_SUCCESS) { - ib::warn() << "Ignoring tablespace " - << id_name_t(space_name) + ib::warn() << "Ignoring tablespace for " + << table_name << " because it could not be opened."; } max_space_id = ut_max(max_space_id, space_id); ut_free(table_name.m_name); - ut_free(shared_space_name); ut_free(filepath); } @@ -1488,7 +1268,6 @@ dict_check_sys_tables( } /** Check each tablespace found in the data dictionary. -Look at each general tablespace found in SYS_TABLESPACES. Then look at each table defined in SYS_TABLES that has a space_id > 0 to find all the file-per-table tablespaces. @@ -1521,16 +1300,10 @@ dict_check_tablespaces_and_store_max_id( fil_set_max_space_id_if_bigger(max_space_id); - /* Open all general tablespaces found in SYS_TABLESPACES. */ - ulint max1 = dict_check_sys_tablespaces(validate); - /* Open all tablespaces referenced in SYS_TABLES. This will update SYS_TABLESPACES and SYS_DATAFILES if it finds any file-per-table tablespaces not already there. */ - ulint max2 = dict_check_sys_tables(validate); - - /* Store the max space_id found */ - max_space_id = ut_max(max1, max2); + max_space_id = dict_check_sys_tables(validate); fil_set_max_space_id_if_bigger(max_space_id); mutex_exit(&dict_sys->mutex); @@ -2737,9 +2510,9 @@ dict_get_and_save_data_dir_path( dict_table_t* table, bool dict_mutex_own) { - bool is_temp = DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY); + ut_ad(!dict_table_is_temporary(table)); - if (!is_temp && !table->data_dir_path && table->space) { + if (!table->data_dir_path && table->space) { char* path = fil_space_get_first_path(table->space); if (!dict_mutex_own) { @@ -2770,72 +2543,6 @@ dict_get_and_save_data_dir_path( } } -/** Make sure the tablespace name is saved in dict_table_t if the table -uses a general tablespace. -Try to read it from the fil_system_t first, then from SYS_TABLESPACES. -@param[in] table Table object -@param[in] dict_mutex_own) true if dict_sys->mutex is owned already */ -void -dict_get_and_save_space_name( - dict_table_t* table, - bool dict_mutex_own) -{ - /* Do this only for general tablespaces. */ - if (!DICT_TF_HAS_SHARED_SPACE(table->flags)) { - return; - } - - bool use_cache = true; - if (table->tablespace != NULL) { - - if (srv_sys_tablespaces_open - && dict_table_has_temp_general_tablespace_name( - table->tablespace)) { - /* We previous saved the temporary name, - get the real one now. */ - use_cache = false; - } else { - /* Keep and use this name */ - return; - } - } - - if (use_cache) { - fil_space_t* space = fil_space_acquire_silent(table->space); - - if (space != NULL) { - /* Use this name unless it is a temporary general - tablespace name and we can now replace it. */ - if (!srv_sys_tablespaces_open - || !dict_table_has_temp_general_tablespace_name( - space->name)) { - - /* Use this tablespace name */ - table->tablespace = mem_heap_strdup( - table->heap, space->name); - - fil_space_release(space); - return; - } - fil_space_release(space); - } - } - - /* Read it from the dictionary. */ - if (srv_sys_tablespaces_open) { - if (!dict_mutex_own) { - dict_mutex_enter_for_mysql(); - } - - table->tablespace = dict_space_get_name( - table->space, table->heap); - - if (!dict_mutex_own) { - dict_mutex_exit_for_mysql(); - } - } -} - /** Loads a table definition and also all its index definitions, and also the cluster definition if the table is a member in a cluster. Also loads all foreign key constraints where the foreign key is in the table or where @@ -2900,6 +2607,8 @@ dict_load_tablespace( mem_heap_t* heap, dict_err_ignore_t ignore_err) { + ut_ad(!dict_table_is_temporary(table)); + /* The system tablespace is always available. */ if (is_system_tablespace(table->space)) { return; @@ -2912,43 +2621,12 @@ dict_load_tablespace( return; } - if (dict_table_is_temporary(table)) { - /* Do not bother to retry opening temporary tables. */ - table->ibd_file_missing = TRUE; - return; - } - - /* A file-per-table table name is also the tablespace name. - A general tablespace name is not the same as the table name. - Use the general tablespace name if it can be read from the - dictionary, if not use 'innodb_general_##. */ - char* shared_space_name = NULL; - char* space_name; - if (DICT_TF_HAS_SHARED_SPACE(table->flags)) { - if (srv_sys_tablespaces_open) { - shared_space_name = - dict_space_get_name(table->space, NULL); - - } else { - /* Make the temporary tablespace name. */ - shared_space_name = static_cast<char*>( - ut_malloc_nokey( - strlen(general_space_name) + 20)); - - sprintf(shared_space_name, "%s_" ULINTPF, - general_space_name, - static_cast<ulint>(table->space)); - } - space_name = shared_space_name; - } else { - space_name = table->name.m_name; - } + char* space_name = table->name.m_name; /* The tablespace may already be open. */ if (fil_space_for_table_exists_in_mem( table->space, space_name, false, - true, heap, table->id, table)) { - ut_free(shared_space_name); + true, heap, table->id, table, table->flags)) { return; } @@ -2973,37 +2651,20 @@ dict_load_tablespace( table->data_dir_path, table->name.m_name, IBD, true); } - - } else if (DICT_TF_HAS_SHARED_SPACE(table->flags)) { - /* Set table->tablespace from either - fil_system or SYS_TABLESPACES */ - dict_get_and_save_space_name(table, true); - - /* Set the filepath from either - fil_system or SYS_DATAFILES. */ - filepath = dict_get_first_path(table->space); - if (filepath == NULL) { - ib::warn() << "Could not find the filepath" - " for table " << table->name << - ", space ID " << table->space; - } } /* Try to open the tablespace. We set the 2nd param (fix_dict) to false because we do not have an x-lock on dict_operation_lock */ - ulint fsp_flags = dict_tf_to_fsp_flags(table->flags, - false, - dict_table_is_encrypted(table)); dberr_t err = fil_ibd_open( true, false, FIL_TYPE_TABLESPACE, table->space, - fsp_flags, space_name, filepath, table); + dict_tf_to_fsp_flags(table->flags), + space_name, filepath, table); if (err != DB_SUCCESS) { /* We failed to find a sensible tablespace file */ table->ibd_file_missing = TRUE; } - ut_free(shared_space_name); ut_free(filepath); } @@ -3179,7 +2840,7 @@ err_exit: ib::error() << "Table " << table->name << " in InnoDB" " data dictionary contains invalid flags." " SYS_TABLES.MIX_LEN=" << table->flags2; - table->flags2 &= ~(DICT_TF2_TEMPORARY|DICT_TF2_INTRINSIC); + table->flags2 &= ~DICT_TF2_TEMPORARY; dict_table_remove_from_cache(table); table = NULL; err = DB_FAIL; diff --git a/storage/innobase/dict/dict0mem.cc b/storage/innobase/dict/dict0mem.cc index a8f8e345728..9463c5103ca 100644 --- a/storage/innobase/dict/dict0mem.cc +++ b/storage/innobase/dict/dict0mem.cc @@ -24,10 +24,8 @@ Data dictionary memory object creation Created 1/8/1996 Heikki Tuuri ***********************************************************************/ -#ifndef UNIV_HOTBACKUP #include "ha_prototypes.h" #include <mysql_com.h> -#endif /* !UNIV_HOTBACKUP */ #include "dict0mem.h" @@ -41,11 +39,7 @@ Created 1/8/1996 Heikki Tuuri #include "dict0dict.h" #include "fts0priv.h" #include "ut0crc32.h" - -#ifndef UNIV_HOTBACKUP -# include "lock0lock.h" -#endif /* !UNIV_HOTBACKUP */ - +#include "lock0lock.h" #include "sync0sync.h" #include <iostream> @@ -152,21 +146,12 @@ dict_mem_table_create( dict_table_stats_lock() will not be noop. */ dict_table_stats_latch_create(table, true); -#ifndef UNIV_HOTBACKUP table->autoinc_lock = static_cast<ib_lock_t*>( mem_heap_alloc(heap, lock_get_size())); /* lazy creation of table autoinc latch */ dict_table_autoinc_create_lazy(table); - table->autoinc = 0; - table->sess_row_id = 0; - table->sess_trx_id = 0; - - /* The number of transactions that are either waiting on the - AUTOINC lock or have been granted the lock. */ - table->n_waiting_or_granted_auto_inc_locks = 0; - /* If the table has an FTS index or we are in the process of building one, create the table->fts */ if (dict_table_has_fts_index(table) @@ -177,11 +162,6 @@ dict_mem_table_create( } else { table->fts = NULL; } -#endif /* !UNIV_HOTBACKUP */ - - if (DICT_TF_HAS_SHARED_SPACE(table->flags)) { - dict_get_and_save_space_name(table, true); - } new(&table->foreign_set) dict_foreign_set(); new(&table->referenced_set) dict_foreign_set(); @@ -209,10 +189,8 @@ dict_mem_table_free( fts_free(table); } } -#ifndef UNIV_HOTBACKUP - dict_table_autoinc_destroy(table); -#endif /* UNIV_HOTBACKUP */ + dict_table_autoinc_destroy(table); dict_mem_table_free_foreign_vcol_set(table); dict_table_stats_latch_destroy(table); @@ -440,7 +418,6 @@ dict_mem_table_add_s_col( table->s_cols->push_back(s_col); } - /**********************************************************************//** Renames a column of a table in the data dictionary cache. */ static MY_ATTRIBUTE((nonnull)) @@ -640,10 +617,8 @@ dict_mem_fill_column_struct( ulint prtype, /*!< in: precise type */ ulint col_len) /*!< in: column length */ { -#ifndef UNIV_HOTBACKUP ulint mbminlen; ulint mbmaxlen; -#endif /* !UNIV_HOTBACKUP */ column->ind = (unsigned int) col_pos; column->ord_part = 0; @@ -651,10 +626,9 @@ dict_mem_fill_column_struct( column->mtype = (unsigned int) mtype; column->prtype = (unsigned int) prtype; column->len = (unsigned int) col_len; -#ifndef UNIV_HOTBACKUP - dtype_get_mblen(mtype, prtype, &mbminlen, &mbmaxlen); + + dtype_get_mblen(mtype, prtype, &mbminlen, &mbmaxlen); dict_col_set_mbminmaxlen(column, mbminlen, mbmaxlen); -#endif /* !UNIV_HOTBACKUP */ } /**********************************************************************//** @@ -701,7 +675,6 @@ dict_mem_index_create( return(index); } -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Creates and initializes a foreign constraint memory object. @return own: foreign constraint struct */ @@ -961,8 +934,6 @@ dict_mem_table_free_foreign_vcol_set( } } -#endif /* !UNIV_HOTBACKUP */ - /**********************************************************************//** Adds a field definition to an index. NOTE: does not take a copy of the column name if the field is a column. The memory occupied @@ -1158,4 +1129,3 @@ dict_mem_table_is_system( return true; } } - diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index ff5162a68c4..87502ef130c 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -23,8 +23,6 @@ Code used for calculating and manipulating table statistics. Created Jan 06, 2010 Vasil Dimov *******************************************************/ -#ifndef UNIV_HOTBACKUP - #include "univ.i" #include "ut0ut.h" @@ -700,7 +698,10 @@ void dict_stats_copy( /*============*/ dict_table_t* dst, /*!< in/out: destination table */ - const dict_table_t* src) /*!< in: source table */ + const dict_table_t* src, /*!< in: source table */ + bool reset_ignored_indexes) /*!< in: if true, set ignored indexes + to have the same statistics as if + the table was empty */ { dst->stats_last_recalc = src->stats_last_recalc; dst->stat_n_rows = src->stat_n_rows; @@ -719,7 +720,16 @@ dict_stats_copy( && (src_idx = dict_table_get_next_index(src_idx)))) { if (dict_stats_should_ignore_index(dst_idx)) { - continue; + if (reset_ignored_indexes) { + /* Reset index statistics for all ignored indexes, + unless they are FT indexes (these have no statistics)*/ + if (dst_idx->type & DICT_FTS) { + continue; + } + dict_stats_empty_index(dst_idx, true); + } else { + continue; + } } ut_ad(!dict_index_is_ibuf(dst_idx)); @@ -818,7 +828,7 @@ dict_stats_snapshot_create( t = dict_stats_table_clone_create(table); - dict_stats_copy(t, table); + dict_stats_copy(t, table, false); t->stat_persistent = table->stat_persistent; t->stats_auto_recalc = table->stats_auto_recalc; @@ -3283,13 +3293,10 @@ dict_stats_update( dict_table_stats_lock(table, RW_X_LATCH); - /* Initialize all stats to dummy values before - copying because dict_stats_table_clone_create() does - skip corrupted indexes so our dummy object 't' may - have less indexes than the real object 'table'. */ - dict_stats_empty_table(table, true); - - dict_stats_copy(table, t); + /* Pass reset_ignored_indexes=true as parameter + to dict_stats_copy. This will cause statictics + for corrupted indexes to be set to empty values */ + dict_stats_copy(table, t, true); dict_stats_assert_initialized(table); @@ -4281,5 +4288,3 @@ test_dict_stats_all() #endif /* UNIV_ENABLE_UNIT_TEST_DICT_STATS */ /* @} */ - -#endif /* UNIV_HOTBACKUP */ diff --git a/storage/innobase/dict/dict0stats_bg.cc b/storage/innobase/dict/dict0stats_bg.cc index 9d04cee4165..974ec9b5c41 100644 --- a/storage/innobase/dict/dict0stats_bg.cc +++ b/storage/innobase/dict/dict0stats_bg.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -42,8 +43,9 @@ Created Apr 25, 2012 Vasil Dimov #define SHUTTING_DOWN() (srv_shutdown_state != SRV_SHUTDOWN_NONE) -/** Event to wake up the stats thread */ -os_event_t dict_stats_event = NULL; +/** Event to wake up dict_stats_thread on dict_stats_recalc_pool_add() +or shutdown. Not protected by any mutex. */ +os_event_t dict_stats_event; /** Variable to initiate shutdown the dict stats thread. Note we don't use 'srv_shutdown_state' because we want to shutdown dict stats thread @@ -399,10 +401,7 @@ statistics. @return this function does not return, it calls os_thread_exit() */ extern "C" os_thread_ret_t -DECLARE_THREAD(dict_stats_thread)( -/*==============================*/ - void* arg MY_ATTRIBUTE((unused))) /*!< in: a dummy parameter - required by os_thread_create */ +DECLARE_THREAD(dict_stats_thread)(void*) { ut_a(!srv_read_only_mode); @@ -412,8 +411,6 @@ DECLARE_THREAD(dict_stats_thread)( */ #endif /* UNIV_PFS_THREAD */ - srv_dict_stats_thread_active = TRUE; - while (!dict_stats_start_shutdown) { /* Wake up periodically even if not signaled. This is @@ -445,7 +442,7 @@ DECLARE_THREAD(dict_stats_thread)( os_event_reset(dict_stats_event); } - srv_dict_stats_thread_active = FALSE; + srv_dict_stats_thread_active = false; os_event_set(dict_stats_shutdown_event); my_thread_end(); diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 9062bf1586b..ac1978f2673 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -1,6 +1,6 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -24,14 +24,16 @@ Modified Jan Lindström jan.lindstrom@mariadb.com *******************************************************/ #include "fil0fil.h" +#include "mach0data.h" +#include "page0size.h" +#include "page0zip.h" +#ifndef UNIV_INNOCHECKSUM #include "fil0crypt.h" #include "srv0srv.h" #include "srv0start.h" -#include "mach0data.h" #include "log0recv.h" #include "mtr0mtr.h" #include "mtr0log.h" -#include "page0zip.h" #include "ut0ut.h" #include "btr0scrub.h" #include "fsp0fsp.h" @@ -41,14 +43,10 @@ Modified Jan Lindström jan.lindstrom@mariadb.com #include <my_crypt.h> /** Mutex for keys */ -UNIV_INTERN ib_mutex_t fil_crypt_key_mutex; +static ib_mutex_t fil_crypt_key_mutex; static bool fil_crypt_threads_inited = false; -#ifdef UNIV_PFS_MUTEX -UNIV_INTERN mysql_pfs_key_t fil_crypt_key_mutex_key; -#endif - /** Is encryption enabled/disabled */ UNIV_INTERN ulong srv_encrypt_tables = 0; @@ -56,26 +54,22 @@ UNIV_INTERN ulong srv_encrypt_tables = 0; UNIV_INTERN uint srv_n_fil_crypt_threads = 0; /** No of key rotation threads started */ -static uint srv_n_fil_crypt_threads_started = 0; +UNIV_INTERN uint srv_n_fil_crypt_threads_started = 0; /** At this age or older a space/page will be rotated */ UNIV_INTERN uint srv_fil_crypt_rotate_key_age = 1; /** Event to signal FROM the key rotation threads. */ -UNIV_INTERN os_event_t fil_crypt_event; +static os_event_t fil_crypt_event; /** Event to signal TO the key rotation threads. */ UNIV_INTERN os_event_t fil_crypt_threads_event; /** Event for waking up threads throttle */ -UNIV_INTERN os_event_t fil_crypt_throttle_sleep_event; +static os_event_t fil_crypt_throttle_sleep_event; /** Mutex for key rotation threads */ -UNIV_INTERN ib_mutex_t fil_crypt_threads_mutex; - -#ifdef UNIV_PFS_MUTEX -UNIV_INTERN mysql_pfs_key_t fil_crypt_threads_mutex_key; -#endif +static ib_mutex_t fil_crypt_threads_mutex; /** Variable ensuring only 1 thread at time does initial conversion */ static bool fil_crypt_start_converting = false; @@ -95,17 +89,6 @@ extern uint srv_background_scrub_data_check_interval; static fil_crypt_stat_t crypt_stat; static ib_mutex_t crypt_stat_mutex; -#ifdef UNIV_PFS_MUTEX -UNIV_INTERN mysql_pfs_key_t fil_crypt_stat_mutex_key; -#endif - -/** - * key for crypt data mutex -*/ -#ifdef UNIV_PFS_MUTEX -UNIV_INTERN mysql_pfs_key_t fil_crypt_data_mutex_key; -#endif - static bool fil_crypt_needs_rotation( /*=====================*/ @@ -138,6 +121,25 @@ fil_space_crypt_cleanup() /*=====================*/ { os_event_destroy(fil_crypt_throttle_sleep_event); + mutex_free(&fil_crypt_key_mutex); + mutex_free(&crypt_stat_mutex); +} + +/** +Get latest key version from encryption plugin. +@return key version or ENCRYPTION_KEY_VERSION_INVALID */ +uint +fil_space_crypt_struct::key_get_latest_version(void) +{ + uint key_version = key_found; + + if (is_key_found()) { + key_version = encryption_key_get_latest_version(key_id); + srv_stats.n_key_requests.inc(); + key_found = key_version; + } + + return key_version; } /****************************************************************** @@ -148,20 +150,25 @@ fil_crypt_get_latest_key_version( /*=============================*/ fil_space_crypt_t* crypt_data) /*!< in: crypt data */ { - uint rc = encryption_key_get_latest_version(crypt_data->key_id); + ut_ad(crypt_data != NULL); - if (fil_crypt_needs_rotation(crypt_data->encryption, - crypt_data->min_key_version, - rc, srv_fil_crypt_rotate_key_age)) { - os_event_set(fil_crypt_threads_event); + uint key_version = crypt_data->key_get_latest_version(); + + if (crypt_data->is_key_found()) { + + if (fil_crypt_needs_rotation(crypt_data->encryption, + crypt_data->min_key_version, + key_version, + srv_fil_crypt_rotate_key_age)) { + os_event_set(fil_crypt_threads_event); + } } - return rc; + return key_version; } /****************************************************************** Mutex helper for crypt_data->scheme */ -static void crypt_data_scheme_locker( /*=====================*/ @@ -181,37 +188,47 @@ crypt_data_scheme_locker( /****************************************************************** Create a fil_space_crypt_t object @return crypt object */ -UNIV_INTERN +static fil_space_crypt_t* fil_space_create_crypt_data( /*========================*/ - fil_encryption_t encrypt_mode, /*!< in: encryption mode */ - uint key_id) /*!< in: encryption key id */ + uint type, + fil_encryption_t encrypt_mode, + uint min_key_version, + uint key_id, + ulint offset) { const uint sz = sizeof(fil_space_crypt_t); - fil_space_crypt_t* crypt_data = - static_cast<fil_space_crypt_t*>(malloc(sz)); - - memset(crypt_data, 0, sz); + void* buf = ut_zalloc_nokey(sz); + fil_space_crypt_t* crypt_data = NULL; - if (encrypt_mode == FIL_SPACE_ENCRYPTION_OFF || - (!srv_encrypt_tables && encrypt_mode == FIL_SPACE_ENCRYPTION_DEFAULT)) { - crypt_data->type = CRYPT_SCHEME_UNENCRYPTED; - } else { - crypt_data->type = CRYPT_SCHEME_1; - crypt_data->min_key_version = encryption_key_get_latest_version(key_id); + if (buf) { + crypt_data = new(buf) + fil_space_crypt_struct( + type, + min_key_version, + key_id, + offset, + encrypt_mode); } - mutex_create(LATCH_ID_FIL_CRYPT_DATA_MUTEX, &crypt_data->mutex); - crypt_data->locker = crypt_data_scheme_locker; - my_random_bytes(crypt_data->iv, sizeof(crypt_data->iv)); - crypt_data->encryption = encrypt_mode; - crypt_data->inited = true; - crypt_data->key_id = key_id; return crypt_data; } /****************************************************************** +Create a fil_space_crypt_t object +@return crypt object */ +UNIV_INTERN +fil_space_crypt_t* +fil_space_create_crypt_data( +/*========================*/ + fil_encryption_t encrypt_mode, /*!< in: encryption mode */ + uint key_id) /*!< in: encryption key id */ +{ + return (fil_space_create_crypt_data(0, encrypt_mode, 0, key_id, 0)); +} + +/****************************************************************** Merge fil_space_crypt_t object */ UNIV_INTERN void @@ -233,7 +250,7 @@ fil_space_merge_crypt_data( dst->type = src->type; dst->min_key_version = src->min_key_version; dst->keyserver_requests += src->keyserver_requests; - dst->inited = src->inited; + dst->closing = src->closing; mutex_exit(&dst->mutex); } @@ -249,11 +266,6 @@ fil_space_read_crypt_data( const byte* page, /*!< in: page 0 */ ulint offset) /*!< in: offset */ { - if (memcmp(page + offset, EMPTY_PATTERN, MAGIC_SZ) == 0) { - /* Crypt data is not stored. */ - return NULL; - } - if (memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) { /* Crypt data is not stored. */ return NULL; @@ -305,18 +317,12 @@ fil_space_read_crypt_data( fil_encryption_t encryption = (fil_encryption_t)mach_read_from_1( page + offset + MAGIC_SZ + 2 + iv_length + 8); - const uint sz = sizeof(fil_space_crypt_t) + iv_length; - crypt_data = static_cast<fil_space_crypt_t*>(malloc(sz)); - memset(crypt_data, 0, sz); - + crypt_data = fil_space_create_crypt_data(encryption, key_id); + /* We need to overwrite these as above function will initialize + members */ crypt_data->type = type; crypt_data->min_key_version = min_key_version; - crypt_data->key_id = key_id; crypt_data->page0_offset = offset; - crypt_data->encryption = encryption; - mutex_create(LATCH_ID_FIL_CRYPT_DATA_MUTEX, &crypt_data->mutex); - crypt_data->locker = crypt_data_scheme_locker; - crypt_data->inited = true; memcpy(crypt_data->iv, page + offset + MAGIC_SZ + 2, iv_length); return crypt_data; @@ -331,19 +337,11 @@ fil_space_destroy_crypt_data( fil_space_crypt_t **crypt_data) /*!< out: crypt data */ { if (crypt_data != NULL && (*crypt_data) != NULL) { - /* Make sure that this thread owns the crypt_data - and make it unawailable, this does not fully - avoid the race between drop table and crypt thread */ mutex_enter(&fil_crypt_threads_mutex); - mutex_enter(&(*crypt_data)->mutex); - (*crypt_data)->inited = false; - mutex_exit(&(*crypt_data)->mutex); - /* JAN: TODO: - mutex_free(& (*crypt_data)->mutex); - memset(*crypt_data, 0, sizeof(fil_space_crypt_t)); - free(*crypt_data); - (*crypt_data) = NULL; - */ + fil_space_crypt_t* c = *crypt_data; + c->~fil_space_crypt_struct(); + ut_free(c); + *crypt_data = NULL; mutex_exit(&fil_crypt_threads_mutex); } } @@ -491,6 +489,7 @@ fil_parse_write_crypt_data( } fil_space_crypt_t* crypt_data = fil_space_create_crypt_data(encryption, key_id); + /* Need to overwrite these as above will initialize fields. */ crypt_data->page0_offset = offset; crypt_data->min_key_version = min_key_version; crypt_data->encryption = encryption; @@ -630,10 +629,11 @@ fil_space_encrypt( ulint orig_page_type = mach_read_from_2(src_frame+FIL_PAGE_TYPE); - if (orig_page_type==FIL_PAGE_TYPE_FSP_HDR - || orig_page_type==FIL_PAGE_TYPE_XDES) { - /* File space header or extent descriptor do not need to be - encrypted. */ + if (orig_page_type == FIL_PAGE_TYPE_FSP_HDR || + orig_page_type == FIL_PAGE_TYPE_XDES || + orig_page_type == FIL_PAGE_RTREE) { + /* File space header, extent descriptor or spatial index + are not encrypted. */ return src_frame; } @@ -644,7 +644,7 @@ fil_space_encrypt( return src_frame; } - ut_a(crypt_data != NULL && crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF); + ut_a(crypt_data != NULL && crypt_data->is_encrypted()); byte* tmp = fil_encrypt_buf(crypt_data, space, offset, lsn, src_frame, page_size, dst_frame); @@ -723,7 +723,7 @@ fil_space_check_encryption_read( return false; } - if (crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) { + if (crypt_data->not_encrypted()) { return false; } @@ -776,7 +776,7 @@ fil_space_decrypt( return false; } - ut_a(crypt_data != NULL && crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF); + ut_a(crypt_data != NULL && crypt_data->is_encrypted()); /* read space & lsn */ ulint header_len = FIL_PAGE_DATA; @@ -913,81 +913,6 @@ fil_crypt_calculate_checksum( return checksum; } -/********************************************************************* -Verify checksum for a page (iff it's encrypted) -NOTE: currently this function can only be run in single threaded mode -as it modifies srv_checksum_algorithm (temporarily) -@return true if page is encrypted AND OK, false otherwise */ -UNIV_INTERN -bool -fil_space_verify_crypt_checksum( -/*============================*/ - const byte* src_frame, /*!< in: page the verify */ - const page_size_t& page_size) /*!< in: page size */ -{ - // key version - uint key_version = mach_read_from_4( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - - if (key_version == 0) { - return false; // unencrypted page - } - - /* "trick" the normal checksum routines by storing the post-encryption - * checksum into the normal checksum field allowing for reuse of - * the normal routines */ - - // post encryption checksum - ib_uint32_t stored_post_encryption = mach_read_from_4( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); - - // save pre encryption checksum for restore in end of this function - ib_uint32_t stored_pre_encryption = mach_read_from_4( - src_frame + FIL_PAGE_SPACE_OR_CHKSUM); - - ib_uint32_t checksum_field2 = mach_read_from_4( - src_frame + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); - - /** prepare frame for usage of normal checksum routines */ - mach_write_to_4(const_cast<byte*>(src_frame) + FIL_PAGE_SPACE_OR_CHKSUM, - stored_post_encryption); - - /* NOTE: this function is (currently) only run when restoring - * dblwr-buffer, server is single threaded so it's safe to modify - * srv_checksum_algorithm */ - srv_checksum_algorithm_t save_checksum_algorithm = - (srv_checksum_algorithm_t)srv_checksum_algorithm; - - if (!page_size.is_compressed() && - (save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB || - save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB)) { - /* handle ALGORITHM_INNODB specially, - * "downgrade" to ALGORITHM_INNODB and store BUF_NO_CHECKSUM_MAGIC - * checksum_field2 is sort of pointless anyway... - */ - srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB; - mach_write_to_4(const_cast<byte*>(src_frame) + - UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM, - BUF_NO_CHECKSUM_MAGIC); - } - - /* verify checksums */ - ibool corrupted = buf_page_is_corrupted(false, src_frame, page_size, false); - - /** restore frame & algorithm */ - srv_checksum_algorithm = save_checksum_algorithm; - - mach_write_to_4(const_cast<byte*>(src_frame) + - FIL_PAGE_SPACE_OR_CHKSUM, - stored_pre_encryption); - - mach_write_to_4(const_cast<byte*>(src_frame) + - UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM, - checksum_field2); - - return (!corrupted); -} - /***********************************************************************/ /** A copy of global key state */ @@ -1008,20 +933,13 @@ Copy global key state */ static void fil_crypt_get_key_state( /*====================*/ - key_state_t *new_state) /*!< out: key state */ + key_state_t* new_state, /*!< out: key state */ + fil_space_crypt_t* crypt_data) /*!< in, out: crypt_data */ { if (srv_encrypt_tables) { - new_state->key_version = - encryption_key_get_latest_version(new_state->key_id); + new_state->key_version = crypt_data->key_get_latest_version(); new_state->rotate_key_age = srv_fil_crypt_rotate_key_age; - if (new_state->key_version == ENCRYPTION_KEY_VERSION_INVALID) { - ib::error() << "Used key_id " - << new_state->key_id - << " can't be found from key file."; - } - - ut_a(new_state->key_version != ENCRYPTION_KEY_VERSION_INVALID); ut_a(new_state->key_version != ENCRYPTION_KEY_NOT_ENCRYPTED); } else { new_state->key_version = 0; @@ -1081,9 +999,7 @@ fil_crypt_is_closing( fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); if (crypt_data) { - mutex_enter(&crypt_data->mutex); - closing = crypt_data->closing; - mutex_exit(&crypt_data->mutex); + closing = crypt_data->is_closing(false); } return closing; @@ -1183,10 +1099,11 @@ fil_crypt_start_encrypting_space( /* 3 - compute location to store crypt data */ byte* frame = buf_block_get_frame(block); - ulint maxsize = 0; ut_ad(crypt_data); - crypt_data->page0_offset = - fsp_header_get_crypt_offset(page_size, &maxsize); + crypt_data->page0_offset = FSP_HEADER_OFFSET + + fsp_header_get_encryption_offset(page_size); + const ulint maxsize = page_size.logical() + - crypt_data->page0_offset - FIL_PAGE_DATA_END; /* 4 - write crypt data to page 0 */ fil_space_write_crypt_data_low(crypt_data, @@ -1284,17 +1201,28 @@ struct rotate_thread_t { uint estimated_max_iops; /*!< estimation of max iops */ uint allocated_iops; /*!< allocated iops */ uint cnt_waited; /*!< #times waited during this slot */ - uint sum_waited_us; /*!< wait time during this slot */ + uintmax_t sum_waited_us; /*!< wait time during this slot */ fil_crypt_stat_t crypt_stat; // statistics btr_scrub_t scrub_data; /* thread local data used by btr_scrub-functions * when iterating pages of tablespace */ - /* check if this thread should shutdown */ + /** @return whether this thread should terminate */ bool should_shutdown() const { - return ! (srv_shutdown_state == SRV_SHUTDOWN_NONE && - thread_no < srv_n_fil_crypt_threads); + switch (srv_shutdown_state) { + case SRV_SHUTDOWN_NONE: + case SRV_SHUTDOWN_CLEANUP: + return thread_no >= srv_n_fil_crypt_threads; + case SRV_SHUTDOWN_EXIT_THREADS: + /* srv_init_abort() must have been invoked */ + case SRV_SHUTDOWN_FLUSH_PHASE: + return true; + case SRV_SHUTDOWN_LAST_PHASE: + break; + } + ut_ad(0); + return true; } }; @@ -1342,6 +1270,18 @@ fil_crypt_space_needs_rotation( } return false; } + + crypt_data->key_get_latest_version(); + + if (!crypt_data->is_key_found()) { + return false; + } + } + + /* If used key_id is not found from encryption plugin we can't + continue to rotate the tablespace */ + if (!crypt_data->is_key_found()) { + return false; } mutex_enter(&crypt_data->mutex); @@ -1355,7 +1295,7 @@ fil_crypt_space_needs_rotation( } /* prevent threads from starting to rotate space */ - if (crypt_data->closing) { + if (crypt_data->is_closing(true)) { break; } @@ -1364,13 +1304,13 @@ fil_crypt_space_needs_rotation( } /* No need to rotate space if encryption is disabled */ - if (crypt_data->encryption == FIL_SPACE_ENCRYPTION_OFF) { + if (crypt_data->not_encrypted()) { break; } if (crypt_data->key_id != key_state->key_id) { key_state->key_id= crypt_data->key_id; - fil_crypt_get_key_state(key_state); + fil_crypt_get_key_state(key_state, crypt_data); } bool need_key_rotation = fil_crypt_needs_rotation( @@ -1383,12 +1323,14 @@ fil_crypt_space_needs_rotation( time_t diff = time(0) - crypt_data->rotate_state.scrubbing. last_scrub_completed; + bool need_scrubbing = crypt_data->rotate_state.scrubbing.is_active && diff >= (time_t) srv_background_scrub_data_interval; - if (need_key_rotation == false && need_scrubbing == false) + if (need_key_rotation == false && need_scrubbing == false) { break; + } mutex_exit(&crypt_data->mutex); /* NOTE! fil_decr_pending_ops is performed outside */ @@ -1604,8 +1546,9 @@ fil_crypt_find_space_to_rotate( os_event_wait_time(fil_crypt_threads_event, 1000000); } - if (state->should_shutdown()) + if (state->should_shutdown()) { return false; + } if (state->first) { state->first = false; @@ -1667,7 +1610,7 @@ fil_crypt_start_rotate_space( crypt_data->rotate_state.start_time = time(0); if (crypt_data->type == CRYPT_SCHEME_UNENCRYPTED && - crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF && + crypt_data->is_encrypted() && key_state->key_version != 0) { /* this is rotation unencrypted => encrypted */ crypt_data->type = CRYPT_SCHEME_1; @@ -1704,7 +1647,7 @@ fil_crypt_find_page_to_rotate( mutex_enter(&crypt_data->mutex); ut_ad(key_state->key_id == crypt_data->key_id); - if (crypt_data->closing == false && + if (!crypt_data->is_closing(true) && crypt_data->rotate_state.next_offset < crypt_data->rotate_state.max_offset) { @@ -1968,7 +1911,7 @@ fil_crypt_rotate_page( /* statistics */ state->crypt_stat.pages_modified++; } else { - if (crypt_data->encryption != FIL_SPACE_ENCRYPTION_OFF) { + if (crypt_data->is_encrypted()) { ut_a(kv >= crypt_data->min_key_version || (kv == 0 && key_state->key_version == 0)); @@ -2154,10 +2097,9 @@ fil_crypt_flush_space( if (block && err == DB_SUCCESS) { byte* frame = buf_block_get_frame(block); - ulint maxsize=0; - crypt_data->page0_offset = - fsp_header_get_crypt_offset(page_size, &maxsize); + crypt_data->page0_offset = FSP_HEADER_OFFSET + + fsp_header_get_encryption_offset(page_size); fil_space_write_crypt_data(space, frame, crypt_data->page0_offset, @@ -2181,7 +2123,7 @@ fil_crypt_complete_rotate_space( fil_space_crypt_t *crypt_data = fil_space_get_crypt_data(space); /* Space might already be dropped */ - if (crypt_data != NULL && crypt_data->inited) { + if (crypt_data != NULL && !crypt_data->is_closing(false)) { mutex_enter(&crypt_data->mutex); /** @@ -2449,25 +2391,19 @@ fil_crypt_threads_init() } /********************************************************************* -End threads for key rotation */ -UNIV_INTERN -void -fil_crypt_threads_end() -/*===================*/ -{ - /* stop threads */ - fil_crypt_set_thread_cnt(0); -} - -/********************************************************************* Clean up key rotation threads resources */ UNIV_INTERN void fil_crypt_threads_cleanup() /*=======================*/ { + if (!fil_crypt_threads_inited) { + return; + } + ut_a(!srv_n_fil_crypt_threads_started); os_event_destroy(fil_crypt_event); os_event_destroy(fil_crypt_threads_event); + mutex_free(&fil_crypt_threads_mutex); fil_crypt_threads_inited = false; } @@ -2477,7 +2413,8 @@ UNIV_INTERN void fil_space_crypt_mark_space_closing( /*===============================*/ - ulint space) /*!< in: Space id */ + ulint space, /*!< in: tablespace id */ + fil_space_crypt_t* crypt_data) /*!< in: crypt_data or NULL */ { if (!fil_crypt_threads_inited) { return; @@ -2485,7 +2422,9 @@ fil_space_crypt_mark_space_closing( mutex_enter(&fil_crypt_threads_mutex); - fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); + if (!crypt_data) { + crypt_data = fil_space_get_crypt_data(space); + } if (crypt_data == NULL) { mutex_exit(&fil_crypt_threads_mutex); @@ -2514,13 +2453,13 @@ fil_space_crypt_close_tablespace( fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space); - if (crypt_data == NULL || !crypt_data->inited) { + if (crypt_data == NULL || crypt_data->is_closing(false)) { mutex_exit(&fil_crypt_threads_mutex); return; } - uint start = time(0); - uint last = start; + time_t start = time(0); + time_t last = start; mutex_enter(&crypt_data->mutex); mutex_exit(&fil_crypt_threads_mutex); @@ -2542,7 +2481,7 @@ fil_space_crypt_close_tablespace( cnt = crypt_data->rotate_state.active_threads; flushing = crypt_data->rotate_state.flushing; - uint now = time(0); + time_t now = time(0); if (now >= last + 30) { ib::warn() << "Waited " @@ -2568,6 +2507,8 @@ fil_space_crypt_get_status( { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(id); + memset(status, 0, sizeof(*status)); + if (crypt_data != NULL) { status->space = id; status->scheme = crypt_data->type; @@ -2588,6 +2529,7 @@ fil_space_crypt_get_status( } else { status->rotating = false; } + mutex_exit(&crypt_data->mutex); if (srv_encrypt_tables || crypt_data->min_key_version) { @@ -2597,7 +2539,6 @@ fil_space_crypt_get_status( status->current_key_version = 0; } } else { - memset(status, 0, sizeof(*status)); if (srv_encrypt_tables) { os_event_set(fil_crypt_threads_event); } @@ -2630,6 +2571,7 @@ fil_space_get_scrub_status( struct fil_space_scrub_status_t* status) /*!< out: status */ { fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(id); + memset(status, 0, sizeof(*status)); if (crypt_data != NULL) { @@ -2654,10 +2596,99 @@ fil_space_get_scrub_status( } else { status->scrubbing = false; } + mutex_exit(&crypt_data->mutex); - } else { - memset(status, 0, sizeof(*status)); } return crypt_data == NULL ? 1 : 0; } +#endif /* UNIV_INNOCHECKSUM */ + +/********************************************************************* +Verify checksum for a page (iff it's encrypted) +NOTE: currently this function can only be run in single threaded mode +as it modifies srv_checksum_algorithm (temporarily) +@param[in] src_fame page to verify +@param[in] page_size page_size +@param[in] page_no page number of given read_buf +@param[in] strict_check true if strict-check option is enabled +@return true if page is encrypted AND OK, false otherwise */ +UNIV_INTERN +bool +fil_space_verify_crypt_checksum( +/*============================*/ + const byte* src_frame, /*!< in: page the verify */ + const page_size_t& page_size /*!< in: page size */ +#ifdef UNIV_INNOCHECKSUM + ,uintmax_t page_no, + bool strict_check +#endif /* UNIV_INNOCHECKSUM */ +) +{ + // key version + uint key_version = mach_read_from_4( + src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + + if (key_version == 0) { + return false; // unencrypted page + } + + /* "trick" the normal checksum routines by storing the post-encryption + * checksum into the normal checksum field allowing for reuse of + * the normal routines */ + + // post encryption checksum + ib_uint32_t stored_post_encryption = mach_read_from_4( + src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); + + // save pre encryption checksum for restore in end of this function + ib_uint32_t stored_pre_encryption = mach_read_from_4( + src_frame + FIL_PAGE_SPACE_OR_CHKSUM); + + ib_uint32_t checksum_field2 = mach_read_from_4( + src_frame + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); + + /** prepare frame for usage of normal checksum routines */ + mach_write_to_4(const_cast<byte*>(src_frame) + FIL_PAGE_SPACE_OR_CHKSUM, + stored_post_encryption); + + /* NOTE: this function is (currently) only run when restoring + * dblwr-buffer, server is single threaded so it's safe to modify + * srv_checksum_algorithm */ + srv_checksum_algorithm_t save_checksum_algorithm = + (srv_checksum_algorithm_t)srv_checksum_algorithm; + + if (!page_size.is_compressed() && + (save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB || + save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB)) { + /* handle ALGORITHM_INNODB specially, + * "downgrade" to ALGORITHM_INNODB and store BUF_NO_CHECKSUM_MAGIC + * checksum_field2 is sort of pointless anyway... + */ + srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB; + mach_write_to_4(const_cast<byte*>(src_frame) + + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM, + BUF_NO_CHECKSUM_MAGIC); + } + + /* verify checksums */ + bool corrupted = buf_page_is_corrupted(false, src_frame, + page_size, false +#ifdef UNIV_INNOCHECKSUM + ,page_no, strict_check, false, NULL +#endif /* UNIV_INNOCHECKSUM */ + ); + + /** restore frame & algorithm */ + srv_checksum_algorithm = save_checksum_algorithm; + + mach_write_to_4(const_cast<byte*>(src_frame) + + FIL_PAGE_SPACE_OR_CHKSUM, + stored_pre_encryption); + + mach_write_to_4(const_cast<byte*>(src_frame) + + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM, + checksum_field2); + + return (!corrupted); +} diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index cfb4903bb7a..14dad77f332 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -29,7 +29,6 @@ Created 10/25/1995 Heikki Tuuri #include "fsp0pagecompress.h" #include "fil0crypt.h" -#ifndef UNIV_HOTBACKUP #include "btr0btr.h" #include "buf0buf.h" #include "dict0boot.h" @@ -51,15 +50,15 @@ Created 10/25/1995 Heikki Tuuri #include "srv0start.h" #include "trx0purge.h" #include "ut0new.h" -# include "buf0lru.h" -# include "ibuf0ibuf.h" -# include "os0event.h" -# include "sync0sync.h" -#endif /* !UNIV_HOTBACKUP */ +#include "buf0lru.h" +#include "ibuf0ibuf.h" +#include "os0event.h" +#include "sync0sync.h" #include "buf0flu.h" #include "srv0start.h" #include "trx0purge.h" #include "ut0new.h" +#include "os0api.h" /** Tries to close a file in the LRU list. The caller must hold the fil_sys mutex. @@ -130,18 +129,13 @@ out of the LRU-list and keep a count of pending operations. When an operation completes, we decrement the count and return the file node to the LRU-list if the count drops to zero. */ -/** This tablespace name is used internally during recovery to open a -general tablespace before the data dictionary are recovered and available. */ -const char general_space_name[] = "innodb_general"; - /** Reference to the server data directory. Usually it is the current working directory ".", but in the MySQL Embedded Server Library it is an absolute path. */ const char* fil_path_to_mysql_datadir; -Folder folder_mysql_datadir; /** Common InnoDB file extentions */ -const char* dot_ext[] = { "", ".ibd", ".isl", ".cfg", ".cfp" }; +const char* dot_ext[] = { "", ".ibd", ".isl", ".cfg" }; /** The number of fsyncs done to the log */ ulint fil_n_log_flushes = 0; @@ -161,11 +155,6 @@ fil_addr_t fil_addr_null = {FIL_NULL, 0}; initialized. */ fil_system_t* fil_system = NULL; -#ifdef UNIV_HOTBACKUP -static ulint srv_data_read; -static ulint srv_data_written; -#endif /* UNIV_HOTBACKUP */ - /** Determine if user has explicitly disabled fsync(). */ #ifndef _WIN32 # define fil_buffering_disabled(s) \ @@ -185,7 +174,7 @@ fil_is_user_tablespace_id( ulint space_id) { return(space_id > srv_undo_tablespaces_open - && space_id != srv_tmp_space.space_id()); + && space_id != SRV_TMP_SPACE_ID); } #ifdef UNIV_DEBUG @@ -227,12 +216,13 @@ fil_space_belongs_in_lru( const fil_space_t* space) /*!< in: file space */ { switch (space->purpose) { + case FIL_TYPE_TEMPORARY: case FIL_TYPE_LOG: return(false); case FIL_TYPE_TABLESPACE: - case FIL_TYPE_TEMPORARY: - case FIL_TYPE_IMPORT: return(fil_is_user_tablespace_id(space->id)); + case FIL_TYPE_IMPORT: + return(true); } ut_ad(0); @@ -291,7 +281,7 @@ fil_read( void* buf) { return(fil_io(IORequestRead, true, page_id, page_size, - byte_offset, len, buf, NULL, NULL)); + byte_offset, len, buf, NULL)); } /** Writes data to a space from a buffer. Remember that the possible incomplete @@ -319,7 +309,7 @@ fil_write( ut_ad(!srv_read_only_mode); return(fil_io(IORequestWrite, true, page_id, page_size, - byte_offset, len, buf, NULL, NULL)); + byte_offset, len, buf, NULL)); } /*******************************************************************//** @@ -384,7 +374,6 @@ fil_space_get( return(space); } -#ifndef UNIV_HOTBACKUP /** Returns the latch of a file space. @param[in] id space id @param[out] flags tablespace flags @@ -456,7 +445,6 @@ fil_space_set_imported( mutex_exit(&fil_system->mutex); } -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Checks if all the file nodes in a space are flushed. The caller must hold @@ -484,42 +472,13 @@ fil_space_is_flushed( return(true); } -#if !defined(NO_FALLOCATE) && defined(UNIV_LINUX) - -#include <sys/ioctl.h> -/** FusionIO atomic write control info */ -#define DFS_IOCTL_ATOMIC_WRITE_SET _IOW(0x95, 2, uint) - -/** -Try and enable FusionIO atomic writes. -@param[in] file OS file handle -@return true if successful */ -bool -fil_fusionio_enable_atomic_write(os_file_t file) -{ - if (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT) { - - uint atomic = 1; - - ut_a(file != -1); - - if (ioctl(file, DFS_IOCTL_ATOMIC_WRITE_SET, &atomic) != -1) { - - return(true); - } - } - - return(false); -} -#endif /* !NO_FALLOCATE && UNIV_LINUX */ /** Append a file to the chain of files of a space. @param[in] name file name of a file that is not open @param[in] size file size in entire database blocks @param[in,out] space tablespace from fil_space_create() @param[in] is_raw whether this is a raw device or partition -@param[in] punch_hole true if supported for this node -@param[in] atomic_write true if the file has atomic write enabled +@param[in] atomic_write true if the file could use atomic write @param[in] max_pages maximum number of pages in file, ULINT_MAX means the file size is unlimited. @return pointer to the file name @@ -531,7 +490,6 @@ fil_node_create_low( ulint size, fil_space_t* space, bool is_raw, - bool punch_hole, bool atomic_write, ulint max_pages = ULINT_MAX) { @@ -546,6 +504,8 @@ fil_node_create_low( node = reinterpret_cast<fil_node_t*>(ut_zalloc_nokey(sizeof(*node))); + node->handle = OS_FILE_CLOSED; + node->name = mem_strdup(name); ut_a(!is_raw || srv_start_raw_disk_in_use); @@ -567,40 +527,6 @@ fil_node_create_low( node->space = space; - os_file_stat_t stat_info; - -#ifdef UNIV_DEBUG - dberr_t err = -#endif /* UNIV_DEBUG */ - - os_file_get_status( - node->name, &stat_info, false, - fsp_is_system_temporary(space->id) ? true : srv_read_only_mode); - - ut_ad(err == DB_SUCCESS); - - node->block_size = stat_info.block_size; - - /* In this debugging mode, we can overcome the limitation of some - OSes like Windows that support Punch Hole but have a hole size - effectively too large. By setting the block size to be half the - page size, we can bypass one of the checks that would normally - turn Page Compression off. This execution mode allows compression - to be tested even when full punch hole support is not available. */ - DBUG_EXECUTE_IF("ignore_punch_hole", - node->block_size = ut_min(stat_info.block_size, - static_cast<size_t>(UNIV_PAGE_SIZE / 2)); - ); - - if (!IORequest::is_punch_hole_supported() - || !punch_hole - || node->block_size >= srv_page_size) { - - fil_no_punch_hole(node); - } else { - node->punch_hole = punch_hole; - } - node->atomic_write = atomic_write; UT_LIST_ADD_LAST(space->chain, node); @@ -615,7 +541,7 @@ fil_node_create_low( an integer @param[in,out] space space where to append @param[in] is_raw true if a raw device or a raw disk partition -@param[in] atomic_write true if the file has atomic write enabled +@param[in] atomic_write true if the file could use atomic write @param[in] max_pages maximum number of pages in file, ULINT_MAX means the file size is unlimited. @return pointer to the file name @@ -632,8 +558,7 @@ fil_node_create( fil_node_t* node; node = fil_node_create_low( - name, size, space, is_raw, IORequest::is_punch_hole_supported(), - atomic_write, max_pages); + name, size, space, is_raw, atomic_write, max_pages); return(node == NULL ? NULL : node->name); } @@ -649,17 +574,12 @@ fil_node_open_file( { os_offset_t size_bytes; bool success; - byte* buf2; - byte* page; - ulint flags; - ulint min_size; - ulint space_id; bool read_only_mode; fil_space_t* space = node->space; ut_ad(mutex_own(&fil_system->mutex)); ut_a(node->n_pending == 0); - ut_a(!node->is_open); + ut_a(!node->is_open()); read_only_mode = !fsp_is_system_temporary(space->id) && srv_read_only_mode; @@ -698,133 +618,92 @@ retry: size_bytes = os_file_get_size(node->handle); ut_a(size_bytes != (os_offset_t) -1); -#ifdef UNIV_HOTBACKUP - if (space->id == 0) { - node->size = (ulint) (size_bytes / UNIV_PAGE_SIZE); + ut_a(space->purpose != FIL_TYPE_LOG); + const page_size_t page_size(space->flags); + const ulint psize = page_size.physical(); + const ulint min_size = FIL_IBD_FILE_INITIAL_SIZE + * psize; + + if (size_bytes < min_size) { + ib::error() << "The size of the file " << node->name + << " is only " << size_bytes + << " bytes, should be at least " << min_size; os_file_close(node->handle); - goto add_size; + return(false); } -#endif /* UNIV_HOTBACKUP */ - ut_a(space->purpose != FIL_TYPE_LOG); /* Read the first page of the tablespace */ - buf2 = static_cast<byte*>(ut_malloc_nokey(2 * UNIV_PAGE_SIZE)); + byte* buf2 = static_cast<byte*>(ut_malloc_nokey(2 * psize)); /* Align the memory for file i/o if we might have O_DIRECT set */ - page = static_cast<byte*>(ut_align(buf2, UNIV_PAGE_SIZE)); - ut_ad(page == page_align(page)); + byte* page = static_cast<byte*>(ut_align(buf2, psize)); IORequest request(IORequest::READ); success = os_file_read( request, - node->handle, page, 0, UNIV_PAGE_SIZE); - - space_id = fsp_header_get_space_id(page); - flags = fsp_header_get_flags(page); - - /* Close the file now that we have read the space id from it */ - + node->handle, page, 0, psize); + srv_stats.page0_read.add(1); + + const ulint space_id + = fsp_header_get_space_id(page); + ulint flags = fsp_header_get_flags(page); + const ulint size = fsp_header_get_field( + page, FSP_SIZE); + const ulint free_limit = fsp_header_get_field( + page, FSP_FREE_LIMIT); + const ulint free_len = flst_get_len( + FSP_HEADER_OFFSET + FSP_FREE + page); + ut_free(buf2); os_file_close(node->handle); - const page_size_t page_size(flags); - - min_size = FIL_IBD_FILE_INITIAL_SIZE * page_size.physical(); - - if (size_bytes < min_size) { - - ib::error() << "The size of tablespace " << space_id << " file " - << node->name << " is only " << size_bytes - << ", should be at least " << min_size << "!"; - - ut_error; - } - - if (space->flags != flags) { - ulint sflags = (space->flags & ~FSP_FLAGS_MASK_DATA_DIR); - ulint fflags = (flags & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - - /* DATA_DIR option is on different place on MariaDB - compared to MySQL. If this is the difference. Fix - it. */ - - if (sflags == fflags) { - ib::warn() - << "Tablespace " << space_id - << " flags " << space->flags - << " in the data dictionary but in file " << node->name - << " are " << flags - << ". Temporally corrected because DATA_DIR option to " - << space->flags; - - flags = space->flags; - } else { - ib::fatal() - << "Table flags are " - << ib::hex(space->flags) << " in the data" - " dictionary but the flags in file " - << node->name << " are " << ib::hex(flags) - << "!"; + if (!fsp_flags_is_valid(flags)) { + ulint cflags = fsp_flags_convert_from_101(flags); + if (cflags == ULINT_UNDEFINED) { + ib::error() + << "Expected tablespace flags " + << ib::hex(flags) + << " but found " << ib::hex(flags) + << " in the file " << node->name; + return(false); } - } - - { - ulint size = fsp_header_get_field( - page, FSP_SIZE); - ulint free_limit = fsp_header_get_field( - page, FSP_FREE_LIMIT); - ulint free_len = flst_get_len( - FSP_HEADER_OFFSET + FSP_FREE + page); - - ut_ad(space->free_limit == 0 - || space->free_limit == free_limit); - ut_ad(space->free_len == 0 - || space->free_len == free_len); - space->size_in_header = size; - space->free_limit = free_limit; - space->free_len = free_len; + flags = cflags; } - ut_free(buf2); - -#ifdef MYSQL_ENCRYPTION - /* For encrypted tablespace, we need to check the - encrytion key and iv(initial vector) is readed. */ - if (FSP_FLAGS_GET_ENCRYPTION(flags) - && !recv_recovery_is_on()) { - if (space->encryption_type != Encryption::AES) { - ib::error() - << "Can't read encryption" - << " key from file " - << node->name << "!"; - return(false); - } + if (UNIV_UNLIKELY(space_id != space->id)) { + ib::error() + << "Expected tablespace id " << space->id + << " but found " << space_id + << "in the file" << node->name; + return(false); } -#endif + + ut_ad(space->free_limit == 0 + || space->free_limit == free_limit); + ut_ad(space->free_len == 0 + || space->free_len == free_len); + space->size_in_header = size; + space->free_limit = free_limit; + space->free_len = free_len; if (node->size == 0) { ulint extent_size; - extent_size = page_size.physical() * FSP_EXTENT_SIZE; + extent_size = psize * FSP_EXTENT_SIZE; /* After apply-incremental, tablespaces are not extended to a whole megabyte. Do not cut off valid data. */ -#ifndef UNIV_HOTBACKUP + /* Truncate the size to a multiple of extent size. */ if (size_bytes >= extent_size) { size_bytes = ut_2pow_round(size_bytes, extent_size); } -#endif /* !UNIV_HOTBACKUP */ - node->size = (ulint) - (size_bytes / page_size.physical()); -#ifdef UNIV_HOTBACKUP -add_size: -#endif /* UNIV_HOTBACKUP */ + node->size = static_cast<ulint>(size_bytes / psize); space->size += node->size; } } @@ -847,11 +726,30 @@ add_size: node->handle = os_file_create( innodb_data_file_key, node->name, OS_FILE_OPEN, OS_FILE_AIO, OS_DATA_FILE, read_only_mode, &success); - } - ut_a(success); + if (!space->atomic_write_tested) + { + const page_size_t page_size(space->flags); + + space->atomic_write_tested= 1; + /* + Atomic writes is supported if the file can be used + with atomic_writes (not log file), O_DIRECT is + used (tested in ha_innodbc.cc) and the file is + device and file system that supports atomic writes + for the given block size + */ + space->atomic_write_supported= + srv_use_atomic_writes && + node->atomic_write && + my_test_if_atomic_write(node->handle, + page_size.physical()) ? + true : false; + } + } - node->is_open = true; + ut_a(success); + ut_a(node->is_open()); fil_system->n_open++; fil_n_file_opened++; @@ -875,22 +773,22 @@ fil_node_close_file( bool ret; ut_ad(mutex_own(&(fil_system->mutex))); - ut_a(node->is_open); + ut_a(node->is_open()); ut_a(node->n_pending == 0); ut_a(node->n_pending_flushes == 0); ut_a(!node->being_extended); -#ifndef UNIV_HOTBACKUP ut_a(node->modification_counter == node->flush_counter || node->space->purpose == FIL_TYPE_TEMPORARY - || srv_fast_shutdown == 2); -#endif /* !UNIV_HOTBACKUP */ + || srv_fast_shutdown == 2 + || !srv_was_started); ret = os_file_close(node->handle); ut_a(ret); /* printf("Closing file %s\n", node->name); */ - node->is_open = false; + node->handle = OS_FILE_CLOSED; + ut_ad(!node->is_open()); ut_a(fil_system->n_open > 0); fil_system->n_open--; fil_n_file_opened--; @@ -968,6 +866,341 @@ fil_try_to_close_file_in_LRU( return(false); } +/** Flush any writes cached by the file system. +@param[in,out] space tablespace */ +static +void +fil_flush_low(fil_space_t* space) +{ + ut_ad(mutex_own(&fil_system->mutex)); + ut_ad(space); + ut_ad(!space->stop_new_ops); + + if (fil_buffering_disabled(space)) { + + /* No need to flush. User has explicitly disabled + buffering. */ + ut_ad(!space->is_in_unflushed_spaces); + ut_ad(fil_space_is_flushed(space)); + ut_ad(space->n_pending_flushes == 0); + +#ifdef UNIV_DEBUG + for (fil_node_t* node = UT_LIST_GET_FIRST(space->chain); + node != NULL; + node = UT_LIST_GET_NEXT(chain, node)) { + ut_ad(node->modification_counter + == node->flush_counter); + ut_ad(node->n_pending_flushes == 0); + } +#endif /* UNIV_DEBUG */ + + return; + } + + /* Prevent dropping of the space while we are flushing */ + space->n_pending_flushes++; + + for (fil_node_t* node = UT_LIST_GET_FIRST(space->chain); + node != NULL; + node = UT_LIST_GET_NEXT(chain, node)) { + + int64_t old_mod_counter = node->modification_counter; + + if (old_mod_counter <= node->flush_counter) { + continue; + } + + ut_a(node->is_open()); + + switch (space->purpose) { + case FIL_TYPE_TEMPORARY: + ut_ad(0); // we already checked for this + case FIL_TYPE_TABLESPACE: + case FIL_TYPE_IMPORT: + fil_n_pending_tablespace_flushes++; + break; + case FIL_TYPE_LOG: + fil_n_pending_log_flushes++; + fil_n_log_flushes++; + break; + } +#ifdef _WIN32 + if (node->is_raw_disk) { + + goto skip_flush; + } +#endif /* _WIN32 */ +retry: + if (node->n_pending_flushes > 0) { + /* We want to avoid calling os_file_flush() on + the file twice at the same time, because we do + not know what bugs OS's may contain in file + i/o */ + + int64_t sig_count = os_event_reset(node->sync_event); + + mutex_exit(&fil_system->mutex); + + os_event_wait_low(node->sync_event, sig_count); + + mutex_enter(&fil_system->mutex); + + if (node->flush_counter >= old_mod_counter) { + + goto skip_flush; + } + + goto retry; + } + + ut_a(node->is_open()); + node->n_pending_flushes++; + + mutex_exit(&fil_system->mutex); + + os_file_flush(node->handle); + + mutex_enter(&fil_system->mutex); + + os_event_set(node->sync_event); + + node->n_pending_flushes--; +skip_flush: + if (node->flush_counter < old_mod_counter) { + node->flush_counter = old_mod_counter; + + if (space->is_in_unflushed_spaces + && fil_space_is_flushed(space)) { + + space->is_in_unflushed_spaces = false; + + UT_LIST_REMOVE( + fil_system->unflushed_spaces, + space); + } + } + + switch (space->purpose) { + case FIL_TYPE_TEMPORARY: + break; + case FIL_TYPE_TABLESPACE: + case FIL_TYPE_IMPORT: + fil_n_pending_tablespace_flushes--; + continue; + case FIL_TYPE_LOG: + fil_n_pending_log_flushes--; + continue; + } + + ut_ad(0); + } + + space->n_pending_flushes--; +} + +/** +Fill the pages with NULs +@param[in] node File node +@param[in] page_size physical page size +@param[in] start Offset from the start of the file in bytes +@param[in] len Length in bytes +@param[in] read_only_mode + if true, then read only mode checks are enforced. +@return DB_SUCCESS or error code */ +static +dberr_t +fil_write_zeros( + const fil_node_t* node, + ulint page_size, + os_offset_t start, + ulint len, + bool read_only_mode) +{ + ut_a(len > 0); + + /* Extend at most 1M at a time */ + ulint n_bytes = ut_min(static_cast<ulint>(1024 * 1024), len); + byte* ptr = reinterpret_cast<byte*>(ut_zalloc_nokey(n_bytes + + page_size)); + byte* buf = reinterpret_cast<byte*>(ut_align(ptr, page_size)); + + os_offset_t offset = start; + dberr_t err = DB_SUCCESS; + const os_offset_t end = start + len; + IORequest request(IORequest::WRITE); + + while (offset < end) { + err = os_aio( + request, OS_AIO_SYNC, node->name, + node->handle, buf, offset, n_bytes, read_only_mode, + NULL, NULL); + + if (err != DB_SUCCESS) { + break; + } + + offset += n_bytes; + + n_bytes = ut_min(n_bytes, static_cast<ulint>(end - offset)); + + DBUG_EXECUTE_IF("ib_crash_during_tablespace_extension", + DBUG_SUICIDE();); + } + + ut_free(ptr); + + return(err); +} + + +/** Try to extend a tablespace. +@param[in,out] space tablespace to be extended +@param[in,out] node last file of the tablespace +@param[in] size desired size in number of pages +@param[out] success whether the operation succeeded +@return whether the operation should be retried */ +static UNIV_COLD __attribute__((warn_unused_result, nonnull)) +bool +fil_space_extend_must_retry( + fil_space_t* space, + fil_node_t* node, + ulint size, + bool* success) +{ + ut_ad(mutex_own(&fil_system->mutex)); + ut_ad(UT_LIST_GET_LAST(space->chain) == node); + ut_ad(size >= FIL_IBD_FILE_INITIAL_SIZE); + + *success = space->size >= size; + + if (*success) { + /* Space already big enough */ + return(false); + } + + if (node->being_extended) { + /* Another thread is currently extending the file. Wait + for it to finish. + It'd have been better to use event driven mechanism but + the entire module is peppered with polling stuff. */ + mutex_exit(&fil_system->mutex); + os_thread_sleep(100000); + return(true); + } + + node->being_extended = true; + + if (!fil_node_prepare_for_io(node, fil_system, space)) { + /* The tablespace data file, such as .ibd file, is missing */ + node->being_extended = false; + return(false); + } + + /* At this point it is safe to release fil_system mutex. No + other thread can rename, delete, close or extend the file because + we have set the node->being_extended flag. */ + mutex_exit(&fil_system->mutex); + + ut_ad(size > space->size); + + ulint pages_added = size - space->size; + const page_size_t pageSize(space->flags); + const ulint page_size = pageSize.physical(); + + os_offset_t start = os_file_get_size(node->handle); + ut_a(start != (os_offset_t) -1); + start &= ~(page_size - 1); + const os_offset_t end + = (node->size + pages_added) * page_size; + + *success = end <= start; + + if (!*success) { + DBUG_EXECUTE_IF("ib_crash_during_tablespace_extension", + DBUG_SUICIDE();); + +#ifdef HAVE_POSIX_FALLOCATE + /* On Linux, FusionIO atomic writes cannot extend + files, so we must use posix_fallocate(). */ + int ret = posix_fallocate(node->handle, start, + end - start); + + /* EINVAL means that fallocate() is not supported. + One known case is Linux ext3 file system with O_DIRECT. */ + if (ret == 0) { + } else if (ret != EINVAL) { + ib::error() + << "posix_fallocate(): Failed to preallocate" + " data for file " + << node->name << ", desired size " + << end << " bytes." + " Operating system error number " + << ret << ". Check" + " that the disk is not full or a disk quota" + " exceeded. Some operating system error" + " numbers are described at " REFMAN + "operating-system-error-codes.html"; + } else +#endif + if (DB_SUCCESS != fil_write_zeros( + node, page_size, start, + static_cast<ulint>(end - start), + space->purpose == FIL_TYPE_TEMPORARY + && srv_read_only_mode)) { + ib::warn() + << "Error while writing " << end - start + << " zeroes to " << node->name + << " starting at offset " << start; + } + + /* Check how many pages actually added */ + os_offset_t actual_end = os_file_get_size(node->handle); + ut_a(actual_end != static_cast<os_offset_t>(-1)); + ut_a(actual_end >= start); + + *success = end >= actual_end; + pages_added = static_cast<ulint>( + (std::min(actual_end, end) - start) / page_size); + } + + os_has_said_disk_full = !*success; + + mutex_enter(&fil_system->mutex); + + space->size += pages_added; + + ut_a(node->being_extended); + node->being_extended = false; + node->size += pages_added; + const ulint pages_in_MiB = node->size + & ~((1 << (20 - UNIV_PAGE_SIZE_SHIFT)) - 1); + + fil_node_complete_io(node, fil_system, IORequestWrite); + + /* Keep the last data file size info up to date, rounded to + full megabytes */ + + switch (space->id) { + case TRX_SYS_SPACE: + srv_sys_space.set_last_file_size(pages_in_MiB); + fil_flush_low(space); + return(false); + default: + // TODO: reject CREATE TEMPORARY TABLE...ROW_FORMAT=COMPRESSED + ut_ad(space->purpose == FIL_TYPE_TABLESPACE + || space->purpose == FIL_TYPE_TEMPORARY); + if (space->purpose == FIL_TYPE_TABLESPACE) { + fil_flush_low(space); + } + return(false); + case SRV_TMP_SPACE_ID: + ut_ad(space->purpose == FIL_TYPE_TEMPORARY); + srv_tmp_space.set_last_file_size(pages_in_MiB); + return(false); + } + +} + /*******************************************************************//** Reserves the fil_system mutex and tries to make sure we can open at least one file while holding it. This should be called before calling @@ -978,28 +1211,22 @@ fil_mutex_enter_and_prepare_for_io( /*===============================*/ ulint space_id) /*!< in: space id */ { - fil_space_t* space; - bool success; - bool print_info = false; - ulint count = 0; - ulint count2 = 0; - - for (;;) { + for (ulint count = 0, count2 = 0;;) { mutex_enter(&fil_system->mutex); - if (space_id == 0 || space_id >= SRV_LOG_SPACE_FIRST_ID) { - /* We keep log files and system tablespace files always - open; this is important in preventing deadlocks in this - module, as a page read completion often performs - another read from the insert buffer. The insert buffer - is in tablespace 0, and we cannot end up waiting in - this function. */ - return; + if (space_id >= SRV_LOG_SPACE_FIRST_ID) { + /* We keep log files always open. */ + break; } - space = fil_space_get_by_id(space_id); + fil_space_t* space = fil_space_get_by_id(space_id); - if (space != NULL && space->stop_ios) { + if (space == NULL) { + break; + } + + if (space->stop_ios) { + ut_ad(space->id != 0); /* We are going to do a rename file and want to stop new i/o's for a while. */ @@ -1011,8 +1238,6 @@ fil_mutex_enter_and_prepare_for_io( mutex_exit(&fil_system->mutex); -#ifndef UNIV_HOTBACKUP - /* Wake the i/o-handler threads to make sure pending i/o's are performed */ os_aio_simulated_wake_handler_threads(); @@ -1024,8 +1249,6 @@ fil_mutex_enter_and_prepare_for_io( fil_rename_tablespace() as well. */ os_thread_sleep(20000); -#endif /* UNIV_HOTBACKUP */ - /* Flush tablespaces so that we can close modified files in the LRU list */ fil_flush_file_spaces(FIL_TYPE_TABLESPACE); @@ -1037,63 +1260,107 @@ fil_mutex_enter_and_prepare_for_io( continue; } - if (fil_system->n_open < fil_system->max_n_open) { + fil_node_t* node = UT_LIST_GET_LAST(space->chain); + ut_ad(space->id == 0 + || node == UT_LIST_GET_FIRST(space->chain)); - return; + if (space->id == 0) { + /* We keep the system tablespace files always + open; this is important in preventing + deadlocks in this module, as a page read + completion often performs another read from + the insert buffer. The insert buffer is in + tablespace 0, and we cannot end up waiting in + this function. */ + } else if (!node || node->is_open()) { + /* If the file is already open, no need to do + anything; if the space does not exist, we handle the + situation in the function which called this + function */ + } else { + while (fil_system->n_open >= fil_system->max_n_open) { + /* Too many files are open */ + if (fil_try_to_close_file_in_LRU(count > 1)) { + /* No problem */ + } else if (count >= 2) { + ib::warn() << "innodb_open_files=" + << fil_system->max_n_open + << " is exceeded (" + << fil_system->n_open + << ") files stay open)"; + break; + } else { + mutex_exit(&fil_system->mutex); + os_aio_simulated_wake_handler_threads(); + os_thread_sleep(20000); + /* Flush tablespaces so that we can + close modified files in the LRU list */ + fil_flush_file_spaces(FIL_TYPE_TABLESPACE); + + count++; + continue; + } + } } - /* If the file is already open, no need to do anything; if the - space does not exist, we handle the situation in the function - which called this function. */ - - if (space == NULL || UT_LIST_GET_FIRST(space->chain)->is_open) { - - return; - } + if (ulint size = UNIV_UNLIKELY(space->recv_size)) { + ut_ad(node); + bool success; + if (fil_space_extend_must_retry(space, node, size, + &success)) { + continue; + } - if (count > 1) { - print_info = true; - } + ut_ad(mutex_own(&fil_system->mutex)); + /* Crash recovery requires the file extension + to succeed. */ + ut_a(success); + /* InnoDB data files cannot shrink. */ + ut_a(space->size >= size); - /* Too many files are open, try to close some */ - do { - success = fil_try_to_close_file_in_LRU(print_info); + /* There could be multiple concurrent I/O requests for + this tablespace (multiple threads trying to extend + this tablespace). - } while (success - && fil_system->n_open >= fil_system->max_n_open); + Also, fil_space_set_recv_size() may have been invoked + again during the file extension while fil_system->mutex + was not being held by us. - if (fil_system->n_open < fil_system->max_n_open) { - /* Ok */ - return; - } + Only if space->recv_size matches what we read + originally, reset the field. In this way, a + subsequent I/O request will handle any pending + fil_space_set_recv_size(). */ - if (count >= 2) { - ib::warn() << "Too many (" << fil_system->n_open - << ") files stay open while the maximum" - " allowed value would be " - << fil_system->max_n_open << ". You may need" - " to raise the value of innodb_open_files in" - " my.cnf."; - - return; + if (size == space->recv_size) { + space->recv_size = 0; + } } - mutex_exit(&fil_system->mutex); + break; + } +} -#ifndef UNIV_HOTBACKUP - /* Wake the i/o-handler threads to make sure pending i/o's are - performed */ - os_aio_simulated_wake_handler_threads(); +/** Try to extend a tablespace if it is smaller than the specified size. +@param[in,out] space tablespace +@param[in] size desired size in pages +@return whether the tablespace is at least as big as requested */ +bool +fil_space_extend( + fil_space_t* space, + ulint size) +{ + ut_ad(!srv_read_only_mode || space->purpose == FIL_TYPE_TEMPORARY); - os_thread_sleep(20000); -#endif /* !UNIV_HOTBACKUP */ - /* Flush tablespaces so that we can close modified files in - the LRU list. */ + bool success; - fil_flush_file_spaces(FIL_TYPE_TABLESPACE); + do { + fil_mutex_enter_and_prepare_for_io(space->id); + } while (fil_space_extend_must_retry( + space, UT_LIST_GET_LAST(space->chain), size, + &success)); - count++; - } + mutex_exit(&fil_system->mutex); + return(success); } /** Prepare to free a file node object from a tablespace memory cache. @@ -1110,7 +1377,7 @@ fil_node_close_to_free( ut_a(node->n_pending == 0); ut_a(!node->being_extended); - if (node->is_open) { + if (node->is_open()) { /* We fool the assertion in fil_node_close_file() to think there are no unflushed modifications in the file */ @@ -1184,7 +1451,8 @@ fil_space_free_low( fil_space_t* space) { /* The tablespace must not be in fil_system->named_spaces. */ - ut_ad(srv_fast_shutdown == 2 || space->max_lsn == 0); + ut_ad(srv_fast_shutdown == 2 || !srv_was_started + || space->max_lsn == 0); for (fil_node_t* node = UT_LIST_GET_FIRST(space->chain); node != NULL; ) { @@ -1199,6 +1467,7 @@ fil_space_free_low( ut_ad(space->size == 0); rw_lock_free(&space->latch); + fil_space_destroy_crypt_data(&space->crypt_data); ut_free(space->name); ut_free(space); @@ -1269,13 +1538,15 @@ fil_space_create( ulint id, ulint flags, fil_type_t purpose, - fil_space_crypt_t* crypt_data) /*!< in: crypt data */ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + bool create_table) /*!< in: true if create table */ { fil_space_t* space; ut_ad(fil_system); - ut_ad(fsp_flags_is_valid(flags)); - ut_ad(srv_page_size == UNIV_PAGE_SIZE_ORIG || flags != 0); + ut_ad(fsp_flags_is_valid(flags & ~FSP_FLAGS_MEM_MASK)); + ut_ad(purpose == FIL_TYPE_LOG + || srv_page_size == UNIV_PAGE_SIZE_ORIG || flags != 0); DBUG_EXECUTE_IF("fil_space_create_failure", return(NULL);); @@ -1311,9 +1582,7 @@ fil_space_create( UT_LIST_INIT(space->chain, &fil_node_t::chain); - /* This warning is not applicable while MEB scanning the redo logs */ -#ifndef UNIV_HOTBACKUP - if (fil_type_is_data(purpose) + if ((purpose == FIL_TYPE_TABLESPACE || purpose == FIL_TYPE_IMPORT) && !recv_recovery_on && id > fil_system->max_assigned_id) { @@ -1327,20 +1596,37 @@ fil_space_create( fil_system->max_assigned_id = id; } -#endif /* !UNIV_HOTBACKUP */ + space->purpose = purpose; space->flags = flags; space->magic_n = FIL_SPACE_MAGIC_N; - space->encryption_type = Encryption::NONE; + space->crypt_data = crypt_data; + + /* In create table we write page 0 so we have already + "read" it and for system tablespaces we have read + crypt data at startup. */ + if (create_table || crypt_data != NULL) { + space->page_0_crypt_read = true; + } + + DBUG_LOG("tablespace", + "Tablespace for space " << id << " name " << name + << (create_table ? " created" : " opened")); + if (crypt_data) { + DBUG_LOG("crypt", + "Tablespace " << id << " name " << name + << " encryption " << crypt_data->encryption + << " key id " << crypt_data->key_id + << ":" << fil_crypt_get_mode(crypt_data) + << " " << fil_crypt_get_type(crypt_data)); + } rw_lock_create(fil_space_latch_key, &space->latch, SYNC_FSP); if (space->purpose == FIL_TYPE_TEMPORARY) { -#ifndef UNIV_HOTBACKUP ut_d(space->latch.set_temp_fsp()); -#endif /* !UNIV_HOTBACKUP */ } HASH_INSERT(fil_space_t, hash, fil_system->spaces, id, space); @@ -1355,16 +1641,6 @@ fil_space_create( fil_system->max_assigned_id = id; } - space->crypt_data = crypt_data; - - if (crypt_data) { - space->read_page0 = true; - /* If table could be encrypted print info */ - ib::info() << "Tablespace ID " << id << " name " << space->name - << ":" << fil_crypt_get_mode(crypt_data) - << " " << fil_crypt_get_type(crypt_data); - } - mutex_exit(&fil_system->mutex); return(space); @@ -1524,6 +1800,24 @@ fil_space_get_first_path( return(path); } +/** Set the recovered size of a tablespace in pages. +@param id tablespace ID +@param size recovered size in pages */ +UNIV_INTERN +void +fil_space_set_recv_size(ulint id, ulint size) +{ + mutex_enter(&fil_system->mutex); + ut_ad(size); + ut_ad(id < SRV_LOG_SPACE_FIRST_ID); + + if (fil_space_t* space = fil_space_get_space(id)) { + space->recv_size = size; + } + + mutex_exit(&fil_system->mutex); +} + /*******************************************************************//** Returns the size of the space in pages. The tablespace must be cached in the memory cache. @@ -1579,20 +1873,6 @@ fil_space_get_flags( return(flags); } -/** Check if table is mark for truncate. -@param[in] id space id -@return true if tablespace is marked for truncate. */ -bool -fil_space_is_being_truncated( - ulint id) -{ - bool mark_for_truncate; - mutex_enter(&fil_system->mutex); - mark_for_truncate = fil_space_get_by_id(id)->is_being_truncated; - mutex_exit(&fil_system->mutex); - return(mark_for_truncate); -} - /** Open each fil_node_t of a named fil_space_t if not already open. @param[in] name Tablespace name @return true if all nodes are open */ @@ -1611,7 +1891,7 @@ fil_space_open( node != NULL; node = UT_LIST_GET_NEXT(chain, node)) { - if (!node->is_open + if (!node->is_open() && !fil_node_open_file(node)) { mutex_exit(&fil_system->mutex); return(false); @@ -1645,7 +1925,7 @@ fil_space_close( node != NULL; node = UT_LIST_GET_NEXT(chain, node)) { - if (node->is_open) { + if (node->is_open()) { fil_node_close_file(node); } } @@ -1736,7 +2016,7 @@ fil_open_log_and_system_tablespace_files(void) node != NULL; node = UT_LIST_GET_NEXT(chain, node)) { - if (!node->is_open) { + if (!node->is_open()) { if (!fil_node_open_file(node)) { /* This func is called during server's startup. If some file of log or system @@ -1781,6 +2061,7 @@ fil_close_all_files(void) /* At shutdown, we should not have any files in this list. */ ut_ad(srv_fast_shutdown == 2 + || !srv_was_started || UT_LIST_GET_LEN(fil_system->named_spaces) == 0); mutex_enter(&fil_system->mutex); @@ -1794,7 +2075,7 @@ fil_close_all_files(void) node != NULL; node = UT_LIST_GET_NEXT(chain, node)) { - if (node->is_open) { + if (node->is_open()) { fil_node_close_file(node); } } @@ -1807,6 +2088,7 @@ fil_close_all_files(void) mutex_exit(&fil_system->mutex); ut_ad(srv_fast_shutdown == 2 + || !srv_was_started || UT_LIST_GET_LEN(fil_system->named_spaces) == 0); } @@ -1840,7 +2122,7 @@ fil_close_log_files( node != NULL; node = UT_LIST_GET_NEXT(chain, node)) { - if (node->is_open) { + if (node->is_open()) { fil_node_close_file(node); } } @@ -1909,7 +2191,6 @@ fil_write_flushed_lsn( return(err); } -#ifndef UNIV_HOTBACKUP /** Acquire a tablespace when it could be dropped concurrently. Used by background threads that do not necessarily hold proper locks for concurrency control. @@ -1980,7 +2261,6 @@ fil_space_release( space->n_pending_ops--; mutex_exit(&fil_system->mutex); } -#endif /* !UNIV_HOTBACKUP */ /********************************************************//** Creates the database directory for a table if it does not exist yet. */ @@ -2036,6 +2316,7 @@ fil_op_write_log( ulint len; ut_ad(first_page_no == 0); + ut_ad(fsp_flags_is_valid(flags)); /* fil_name_parse() requires that there be at least one path separator and that the file path end with ".ibd". */ @@ -2091,7 +2372,7 @@ fil_op_write_log( ut_ad(0); } } -#ifndef UNIV_HOTBACKUP + /** Write redo log for renaming a file. @param[in] space_id tablespace id @param[in] first_page_no first page number in the file @@ -2113,7 +2394,7 @@ fil_name_write_rename( MLOG_FILE_RENAME2, space_id, first_page_no, old_name, new_name, 0, mtr); } -#endif /* !UNIV_HOTBACKUP */ + /** Write MLOG_FILE_NAME for a file. @param[in] space_id tablespace id @param[in] first_page_no first page number in the file @@ -2146,7 +2427,6 @@ fil_name_write( fil_name_write(space->id, first_page_no, file->name, mtr); } -#ifndef UNIV_HOTBACKUP /********************************************************//** Recreates table indexes by applying TRUNCATE log record during recovery. @@ -2257,7 +2537,7 @@ fil_recreate_tablespace( /* Align the memory for file i/o */ page = static_cast<byte*>(ut_align(buf, UNIV_PAGE_SIZE)); - flags = fsp_flags_set_page_size(flags, univ_page_size); + flags |= FSP_FLAGS_PAGE_SSIZE(); fsp_header_init_fields(page, space_id, flags); @@ -2333,13 +2613,11 @@ fil_recreate_tablespace( byte* page = buf_block_get_frame(block); - if (!fsp_flags_is_compressed(flags)) { - + if (!FSP_FLAGS_GET_ZIP_SSIZE(flags)) { ut_ad(!page_size.is_compressed()); buf_flush_init_for_writing( - block, page, NULL, recv_lsn, - fsp_is_checksum_disabled(space_id)); + block, page, NULL, recv_lsn, false); err = fil_write(cur_page_id, page_size, 0, page_size.physical(), page); @@ -2387,7 +2665,7 @@ fil_recreate_tablespace( return(err); } -#endif /* UNIV_HOTBACKUP */ + /** Replay a file rename operation if possible. @param[in] space_id tablespace identifier @param[in] first_page_no first page number in the file @@ -2403,9 +2681,6 @@ fil_op_replay_rename( const char* name, const char* new_name) { -#ifdef UNIV_HOTBACKUP - ut_ad(recv_replay_file_ops); -#endif /* UNIV_HOTBACKUP */ ut_ad(first_page_no == 0); /* In order to replay the rename, the following must hold: @@ -2736,7 +3011,6 @@ fil_delete_tablespace( ut_a(space); ut_a(path != 0); -#ifndef UNIV_HOTBACKUP /* IMPORTANT: Because we have set space::stop_new_ops there can't be any new ibuf merges, reads or flushes. We are here because node::n_pending was zero above. However, it is still @@ -2760,15 +3034,9 @@ fil_delete_tablespace( buf_LRU_flush_or_remove_pages(id, buf_remove, 0); -#endif /* !UNIV_HOTBACKUP */ - /* If it is a delete then also delete any generated files, otherwise when we drop the database the remove directory will fail. */ { -#ifdef UNIV_HOTBACKUP - /* When replaying the operation in MySQL Enterprise - Backup, we do not try to write any log record. */ -#else /* UNIV_HOTBACKUP */ /* Before deleting the file, write a log record about it, so that InnoDB crash recovery will expect the file to be gone. */ @@ -2781,7 +3049,6 @@ fil_delete_tablespace( tablespace file, the record must have already been written to the redo log. */ log_write_up_to(mtr.commit_lsn(), true); -#endif /* UNIV_HOTBACKUP */ char* cfg_name = fil_make_filepath(path, NULL, CFG, false); if (cfg_name != NULL) { @@ -2792,13 +3059,7 @@ fil_delete_tablespace( /* Delete the link file pointing to the ibd file we are deleting. */ if (FSP_FLAGS_HAS_DATA_DIR(space->flags)) { - RemoteDatafile::delete_link_file(space->name); - - } else if (FSP_FLAGS_GET_SHARED(space->flags)) { - - RemoteDatafile::delete_link_file(base_name(path)); - } mutex_enter(&fil_system->mutex); @@ -2844,7 +3105,6 @@ fil_delete_tablespace( return(err); } -#ifndef UNIV_HOTBACKUP /** Truncate the tablespace to needed size. @param[in] space_id id of tablespace to truncate @param[in] size_in_pages truncate size. @@ -2876,7 +3136,7 @@ fil_truncate_tablespace( fil_node_t* node = UT_LIST_GET_FIRST(space->chain); - ut_ad(node->is_open); + ut_ad(node->is_open()); space->size = node->size = size_in_pages; @@ -3009,30 +3269,6 @@ fil_space_dec_redo_skipped_count( mutex_exit(&fil_system->mutex); } - -/** -Check whether a single-table tablespace is redo skipped. -@param[in] id space id -@return true if redo skipped */ -bool -fil_space_is_redo_skipped( - ulint id) -{ - fil_space_t* space; - bool is_redo_skipped; - - mutex_enter(&fil_system->mutex); - - space = fil_space_get_by_id(id); - - ut_a(space != NULL); - - is_redo_skipped = space->redo_skipped_count > 0; - - mutex_exit(&fil_system->mutex); - - return(is_redo_skipped); -} #endif /* UNIV_DEBUG */ /*******************************************************************//** @@ -3078,7 +3314,6 @@ fil_discard_tablespace( return(err); } -#endif /* !UNIV_HOTBACKUP */ /*******************************************************************//** Allocates and builds a file name from a path, a table or tablespace name @@ -3315,7 +3550,7 @@ func_exit: } else if (node->modification_counter > node->flush_counter) { /* Flush the space */ sleep = flush = true; - } else if (node->is_open) { + } else if (node->is_open()) { /* Close the file */ fil_node_close_file(node); @@ -3345,7 +3580,7 @@ func_exit: ut_ad(strchr(old_file_name, OS_PATH_SEPARATOR) != NULL); ut_ad(strchr(new_file_name, OS_PATH_SEPARATOR) != NULL); -#ifndef UNIV_HOTBACKUP + if (!recv_recovery_on) { mtr_t mtr; @@ -3355,7 +3590,6 @@ func_exit: mtr.commit(); log_mutex_enter(); } -#endif /* !UNIV_HOTBACKUP */ /* log_sys->mutex is above fil_system->mutex in the latching order */ ut_ad(log_mutex_own()); @@ -3383,11 +3617,9 @@ func_exit: node->name = new_file_name; } -#ifndef UNIV_HOTBACKUP if (!recv_recovery_on) { log_mutex_exit(); } -#endif /* !UNIV_HOTBACKUP */ ut_ad(space->name == old_space_name); if (success) { @@ -3413,10 +3645,9 @@ func_exit: return(success); } -/** Create a new General or Single-Table tablespace +/** Create a tablespace file. @param[in] space_id Tablespace ID @param[in] name Tablespace name in dbname/tablename format. -For general tablespaces, the 'dbname/' part may be missing. @param[in] path Path and filename of the datafile to create. @param[in] flags Tablespace flags @param[in] size Initial size of the tablespace file in @@ -3439,9 +3670,7 @@ fil_ibd_create( byte* buf2; byte* page; bool success; - bool is_temp = FSP_FLAGS_GET_TEMPORARY(flags); - bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags); - bool has_shared_space = FSP_FLAGS_GET_SHARED(flags); + bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags) != 0; fil_space_t* space = NULL; fil_space_crypt_t *crypt_data = NULL; @@ -3449,15 +3678,13 @@ fil_ibd_create( ut_ad(!srv_read_only_mode); ut_a(space_id < SRV_LOG_SPACE_FIRST_ID); ut_a(size >= FIL_IBD_FILE_INITIAL_SIZE); - ut_a(fsp_flags_is_valid(flags)); + ut_a(fsp_flags_is_valid(flags & ~FSP_FLAGS_MEM_MASK)); /* Create the subdirectories in the path, if they are not there already. */ - if (!has_shared_space) { - err = os_file_create_subdirs_if_needed(path); - if (err != DB_SUCCESS) { - return(err); - } + err = os_file_create_subdirs_if_needed(path); + if (err != DB_SUCCESS) { + return(err); } file = os_file_create( @@ -3498,54 +3725,36 @@ fil_ibd_create( return(DB_ERROR); } - bool atomic_write; - -#if !defined(NO_FALLOCATE) && defined(UNIV_LINUX) - if (fil_fusionio_enable_atomic_write(file)) { - - /* This is required by FusionIO HW/Firmware */ - int ret = posix_fallocate(file, 0, size * UNIV_PAGE_SIZE); - - if (ret != 0) { + success= false; +#ifdef HAVE_POSIX_FALLOCATE + /* + Extend the file using posix_fallocate(). This is required by + FusionIO HW/Firmware but should also be the prefered way to extend + a file. + */ + int ret = posix_fallocate(file, 0, size * UNIV_PAGE_SIZE); - ib::error() << - "posix_fallocate(): Failed to preallocate" - " data for file " << path - << ", desired size " - << size * UNIV_PAGE_SIZE - << " Operating system error number " << ret - << ". Check" - " that the disk is not full or a disk quota" - " exceeded. Make sure the file system supports" - " this function. Some operating system error" - " numbers are described at " REFMAN - " operating-system-error-codes.html"; - - success = false; - } else { - success = true; - } - - atomic_write = true; - } else { - atomic_write = false; + if (ret == 0) { + success = true; + } else if (ret != EINVAL) { + ib::error() << + "posix_fallocate(): Failed to preallocate" + " data for file " << path + << ", desired size " + << size * UNIV_PAGE_SIZE + << " Operating system error number " << ret + << ". Check" + " that the disk is not full or a disk quota" + " exceeded. Some operating system error" + " numbers are described at " REFMAN + "operating-system-error-codes.html"; + } +#endif /* HAVE_POSIX_FALLOCATE */ + if (!success) { success = os_file_set_size( path, file, size * UNIV_PAGE_SIZE, srv_read_only_mode); } -#else - atomic_write = false; - - success = os_file_set_size( - path, file, size * UNIV_PAGE_SIZE, srv_read_only_mode); - -#endif /* !NO_FALLOCATE && UNIV_LINUX */ - - if (!success) { - os_file_close(file); - os_file_delete(innodb_data_file_key, path); - return(DB_OUT_OF_FILE_SPACE); - } /* Note: We are actually punching a hole, previous contents will be lost after this call, if it succeeds. In this case the file @@ -3564,6 +3773,14 @@ fil_ibd_create( } } + ulint block_size = os_file_get_block_size(file, path); + + if (!success) { + os_file_close(file); + os_file_delete(innodb_data_file_key, path); + return(DB_OUT_OF_FILE_SPACE); + } + /* printf("Creating tablespace %s id %lu\n", path, space_id); */ /* We have to write the space id to the file immediately and flush the @@ -3580,11 +3797,8 @@ fil_ibd_create( page = static_cast<byte*>(ut_align(buf2, UNIV_PAGE_SIZE)); memset(page, '\0', UNIV_PAGE_SIZE); -#ifndef UNIV_HOTBACKUP - /* Add the UNIV_PAGE_SIZE to the table flags and write them to the - tablespace header. */ - flags = fsp_flags_set_page_size(flags, univ_page_size); -#endif /* !UNIV_HOTBACKUP */ + + flags |= FSP_FLAGS_PAGE_SSIZE(); fsp_header_init_fields(page, space_id, flags); mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id); @@ -3599,9 +3813,6 @@ fil_ibd_create( err = os_file_write( request, path, file, page, 0, page_size.physical()); - - ut_ad(err != DB_IO_NO_PUNCH_HOLE); - } else { page_zip_des_t page_zip; page_zip_set_size(&page_zip, page_size.physical()); @@ -3619,10 +3830,6 @@ fil_ibd_create( err = os_file_write( request, path, file, page_zip.data, 0, page_size.physical()); - - ut_a(err != DB_IO_NO_PUNCH_HOLE); - - punch_hole = false; } ut_free(buf2); @@ -3649,21 +3856,16 @@ fil_ibd_create( return(DB_ERROR); } - /* MEB creates isl files during copy-back, hence they - should not be created during apply log operation. */ -#ifndef UNIV_HOTBACKUP - if (has_data_dir || has_shared_space) { + if (has_data_dir) { /* Make the ISL file if the IBD file is not in the default location. */ - err = RemoteDatafile::create_link_file(name, path, - has_shared_space); + err = RemoteDatafile::create_link_file(name, path); if (err != DB_SUCCESS) { os_file_close(file); os_file_delete(innodb_data_file_key, path); return(err); } } -#endif /* !UNIV_HOTBACKUP */ /* Create crypt data if the tablespace is either encrypted or user has requested it to remain unencrypted. */ @@ -3672,64 +3874,51 @@ fil_ibd_create( crypt_data = fil_space_create_crypt_data(mode, key_id); } - space = fil_space_create(name, space_id, flags, is_temp - ? FIL_TYPE_TEMPORARY : FIL_TYPE_TABLESPACE, - crypt_data); + space = fil_space_create(name, space_id, flags, FIL_TYPE_TABLESPACE, + crypt_data, true); - if (!fil_node_create_low( - path, size, space, false, punch_hole, atomic_write)) { + fil_node_t* node = NULL; + if (space) { + node = fil_node_create_low(path, size, space, false, true); + } + + if (!space || !node) { if (crypt_data) { free(crypt_data); } err = DB_ERROR; - goto error_exit_1; - } - -#ifdef MYSQL_ENCRYPTION - /* For encryption tablespace, initial encryption information. */ - if (FSP_FLAGS_GET_ENCRYPTION(space->flags)) { - err = fil_set_encryption(space->id, - Encryption::AES, - NULL, - NULL); - ut_ad(err == DB_SUCCESS); - } -#endif /* MYSQL_ENCRYPTION */ - -#ifndef UNIV_HOTBACKUP - if (!is_temp) { + } else { mtr_t mtr; const fil_node_t* file = UT_LIST_GET_FIRST(space->chain); - mtr_start(&mtr); + mtr.start(); fil_op_write_log( MLOG_FILE_CREATE2, space_id, 0, file->name, - NULL, space->flags, &mtr); + NULL, space->flags & ~FSP_FLAGS_MEM_MASK, &mtr); fil_name_write(space, 0, file, &mtr); - mtr_commit(&mtr); - } -#endif /* !UNIV_HOTBACKUP */ - err = DB_SUCCESS; + mtr.commit(); + + node->block_size = block_size; + space->punch_hole = punch_hole; - /* Error code is set. Cleanup the various variables used. - These labels reflect the order in which variables are assigned or - actions are done. */ -error_exit_1: - if (err != DB_SUCCESS && (has_data_dir || has_shared_space)) { - RemoteDatafile::delete_link_file(name); + err = DB_SUCCESS; } os_file_close(file); + if (err != DB_SUCCESS) { + if (has_data_dir) { + RemoteDatafile::delete_link_file(name); + } + os_file_delete(innodb_data_file_key, path); } return(err); } -#ifndef UNIV_HOTBACKUP /** Try to open a single-table tablespace and optionally check that the space id in it is correct. If this does not succeed, print an error message to the .err log. This function is used to open a tablespace when we start @@ -3754,7 +3943,7 @@ statement to update the dictionary tables if they are incorrect. @param[in] fix_dict true if the dictionary is available to be fixed @param[in] purpose FIL_TYPE_TABLESPACE or FIL_TYPE_TEMPORARY @param[in] id tablespace ID -@param[in] flags tablespace flags +@param[in] flags expected FSP_SPACE_FLAGS @param[in] space_name tablespace name of the datafile If file-per-table, it is the table name in the databasename/tablename format @param[in] path_in expected filepath, usually read from dictionary @@ -3774,14 +3963,11 @@ fil_ibd_open( bool dict_filepath_same_as_default = false; bool link_file_found = false; bool link_file_is_bad = false; - bool is_shared = FSP_FLAGS_GET_SHARED(flags); - bool is_encrypted = FSP_FLAGS_GET_ENCRYPTION(flags); Datafile df_default; /* default location */ Datafile df_dict; /* dictionary location */ RemoteDatafile df_remote; /* remote location */ ulint tablespaces_found = 0; ulint valid_tablespaces_found = 0; - bool for_import = (purpose == FIL_TYPE_IMPORT); ut_ad(!fix_dict || rw_lock_own(dict_operation_lock, RW_LOCK_X)); @@ -3790,10 +3976,13 @@ fil_ibd_open( ut_ad(!fix_dict || srv_log_file_size != 0); ut_ad(fil_type_is_data(purpose)); - if (!fsp_flags_is_valid(flags)) { + /* Table flags can be ULINT_UNDEFINED if + dict_tf_to_fsp_flags_failure is set. */ + if (flags == ULINT_UNDEFINED) { return(DB_CORRUPTION); } + ut_ad(fsp_flags_is_valid(flags & ~FSP_FLAGS_MEM_MASK)); df_default.init(space_name, flags); df_dict.init(space_name, flags); df_remote.init(space_name, flags); @@ -3801,25 +3990,8 @@ fil_ibd_open( /* Discover the correct file by looking in three possible locations while avoiding unecessary effort. */ - if (is_shared) { - /* Shared tablespaces will have a path_in since the filename - is not generated from the tablespace name. Use the basename - from this path_in with the default datadir as a filepath to - the default location */ - ut_a(path_in); - const char* sep = strrchr(path_in, OS_PATH_SEPARATOR); - const char* basename = (sep == NULL) ? path_in : &sep[1]; - df_default.make_filepath(NULL, basename, IBD); - - /* Always validate shared tablespaces. */ - validate = true; - - /* Set the ISL filepath in the default location. */ - df_remote.set_link_filepath(path_in); - } else { - /* We will always look for an ibd in the default location. */ - df_default.make_filepath(NULL, space_name, IBD); - } + /* We will always look for an ibd in the default location. */ + df_default.make_filepath(NULL, space_name, IBD); /* Look for a filepath embedded in an ISL where the default file would be. */ @@ -3832,6 +4004,7 @@ fil_ibd_open( link_file_found = true; if (table) { table->crypt_data = df_remote.get_crypt_info(); + table->page_0_read = true; } } else if (df_remote.filepath() != NULL) { /* An ISL file was found but contained a bad filepath in it. @@ -3854,6 +4027,7 @@ fil_ibd_open( if (table) { table->crypt_data = df_dict.get_crypt_info(); + table->page_0_read = true; } } } @@ -3868,6 +4042,7 @@ fil_ibd_open( ++tablespaces_found; if (table) { table->crypt_data = df_default.get_crypt_info(); + table->page_0_read = true; } } @@ -3888,61 +4063,36 @@ fil_ibd_open( df_dict.close(); } - bool atomic_write; - -#if !defined(NO_FALLOCATE) && defined(UNIV_LINUX) - if (!srv_use_doublewrite_buf && df_default.is_open()) { - atomic_write = fil_fusionio_enable_atomic_write( - df_default.handle()); - } else { - atomic_write = false; - } -#else - atomic_write = false; -#endif /* !NO_FALLOCATE && UNIV_LINUX */ - /* We have now checked all possible tablespace locations and have a count of how many unique files we found. If things are normal, we only found 1. */ /* For encrypted tablespace, we need to check the encryption in header of first page. */ - if (!validate && tablespaces_found == 1 && !is_encrypted) { + if (!validate && tablespaces_found == 1) { goto skip_validate; } /* Read and validate the first page of these three tablespace locations, if found. */ valid_tablespaces_found += - (df_remote.validate_to_dd(id, flags, for_import) - == DB_SUCCESS) ? 1 : 0; + (df_remote.validate_to_dd(id, flags) == DB_SUCCESS); valid_tablespaces_found += - (df_default.validate_to_dd(id, flags, for_import) - == DB_SUCCESS) ? 1 : 0; + (df_default.validate_to_dd(id, flags) == DB_SUCCESS); valid_tablespaces_found += - (df_dict.validate_to_dd(id, flags, for_import) - == DB_SUCCESS) ? 1 : 0; + (df_dict.validate_to_dd(id, flags) == DB_SUCCESS); /* Make sense of these three possible locations. First, bail out if no tablespace files were found. */ if (valid_tablespaces_found == 0) { - if (!is_encrypted) { - /* The following call prints an error message. - For encrypted tablespace we skip print, since it should - be keyring plugin issues. */ - os_file_get_last_error(true); - ib::error() << "Could not find a valid tablespace file for `" - << space_name << "`. " << TROUBLESHOOT_DATADICT_MSG; - } - + os_file_get_last_error(true); + ib::error() << "Could not find a valid tablespace file for `" + << space_name << "`. " << TROUBLESHOOT_DATADICT_MSG; return(DB_CORRUPTION); } - if (!validate && !is_encrypted) { - return(DB_SUCCESS); - } - if (validate && is_encrypted && fil_space_get(id)) { - return(DB_SUCCESS); + if (!validate) { + goto skip_validate; } /* Do not open any tablespaces if more than one tablespace with @@ -4048,8 +4198,7 @@ fil_ibd_open( RemoteDatafile::delete_link_file(space_name); } - } else if (!is_shared - && (!link_file_found || link_file_is_bad)) { + } else if (!link_file_found || link_file_is_bad) { ut_ad(df_dict.is_open()); /* Fix the link file if we got our filepath from the dictionary but a link file did not @@ -4079,9 +4228,7 @@ fil_ibd_open( path_in is not suppled for file-per-table, we must assume that it matched the ISL. */ if ((path_in != NULL && !dict_filepath_same_as_default) - || (path_in == NULL - && (DICT_TF_HAS_DATA_DIR(flags) - || DICT_TF_HAS_SHARED_SPACE(flags))) + || (path_in == NULL && DICT_TF_HAS_DATA_DIR(flags)) || df_remote.filepath() != NULL) { dict_replace_tablespace_and_filepath( id, space_name, df_default.filepath(), flags); @@ -4094,7 +4241,7 @@ skip_validate: space_name, id, flags, purpose, df_remote.is_open() ? df_remote.get_crypt_info() : df_dict.is_open() ? df_dict.get_crypt_info() : - df_default.get_crypt_info()); + df_default.get_crypt_info(), false); /* We do not measure the size of the file, that is why we pass the 0 below */ @@ -4103,58 +4250,20 @@ skip_validate: df_remote.is_open() ? df_remote.filepath() : df_dict.is_open() ? df_dict.filepath() : df_default.filepath(), 0, space, false, - true, atomic_write) == NULL) { + true) == NULL) { err = DB_ERROR; } -#ifdef MYSQL_ENCRYPTION - /* For encryption tablespace, initialize encryption - information.*/ - if (err == DB_SUCCESS && is_encrypted && !for_import) { - Datafile& df_current = df_remote.is_open() ? - df_remote: df_dict.is_open() ? - df_dict : df_default; - - byte* key = df_current.m_encryption_key; - byte* iv = df_current.m_encryption_iv; - ut_ad(key && iv); - - err = fil_set_encryption(space->id, Encryption::AES, - key, iv); - ut_ad(err == DB_SUCCESS); + if (purpose != FIL_TYPE_IMPORT && !srv_read_only_mode) { + df_remote.close(); + df_dict.close(); + df_default.close(); + fsp_flags_try_adjust(id, flags & ~FSP_FLAGS_MEM_MASK); } -#endif /* MYSQL_ENCRYPTION */ - } return(err); } -#endif /* !UNIV_HOTBACKUP */ - -#ifdef UNIV_HOTBACKUP -/*******************************************************************//** -Allocates a file name for an old version of a single-table tablespace. -The string must be freed by caller with ut_free()! -@return own: file name */ -static -char* -fil_make_ibbackup_old_name( -/*=======================*/ - const char* name) /*!< in: original file name */ -{ - static const char suffix[] = "_ibbackup_old_vers_"; - char* path; - ulint len = strlen(name); - - path = static_cast<char*>(ut_malloc_nokey(len + 15 + sizeof(suffix))); - - memcpy(path, name, len); - memcpy(path + len, suffix, sizeof(suffix) - 1); - ut_sprintf_timestamp_without_extra_chars( - path + len + sizeof(suffix) - 1); - return(path); -} -#endif /* UNIV_HOTBACKUP */ /** Looks for a pre-existing fil_space_t with the given tablespace ID and, if found, returns the name and filepath in newly allocated buffers @@ -4240,7 +4349,7 @@ fil_path_to_space_name( } /** Discover the correct IBD file to open given a remote or missing -filepath from the REDO log. MEB and administrators can move a crashed +filepath from the REDO log. Administrators can move a crashed database to another location on the same machine and try to recover it. Remote IBD files might be moved as well to the new location. The problem with this is that the REDO log contains the old location @@ -4252,29 +4361,18 @@ both locations, we can chose on based on these priorities; @param[in] space_id tablespace ID @param[in] df Datafile object with path from redo @return true if a valid datafile was found, false if not */ +static bool fil_ibd_discover( ulint space_id, Datafile& df) { - Datafile df_def_gen; /* default general datafile */ Datafile df_def_per; /* default file-per-table datafile */ - RemoteDatafile df_rem_gen; /* remote general datafile*/ RemoteDatafile df_rem_per; /* remote file-per-table datafile */ - /* Look for the datafile in the default location. If it is - a general tablespace, it will be in the datadir. */ + /* Look for the datafile in the default location. */ const char* filename = df.filepath(); const char* basename = base_name(filename); - df_def_gen.init(basename, 0); - df_def_gen.make_filepath(NULL, basename, IBD); - if (df_def_gen.open_read_only(false) == DB_SUCCESS - && df_def_gen.validate_for_recovery() == DB_SUCCESS - && df_def_gen.space_id() == space_id) { - df.set_filepath(df_def_gen.filepath()); - df.open_read_only(false); - return(true); - } /* If this datafile is file-per-table it will have a schema dir. */ ulint sep_found = 0; @@ -4295,44 +4393,9 @@ fil_ibd_discover( df.open_read_only(false); return(true); } - } - - /* Did not find a general or file-per-table datafile in the - default location. Look for a remote general tablespace. */ - df_rem_gen.set_name(basename); - if (df_rem_gen.open_link_file() == DB_SUCCESS) { - - /* An ISL file was found with contents. */ - if (df_rem_gen.open_read_only(false) != DB_SUCCESS - || df_rem_gen.validate_for_recovery() != DB_SUCCESS) { - /* Assume that this ISL file is intended to be used. - Do not continue looking for another if this file - cannot be opened or is not a valid IBD file. */ - ib::error() << "ISL file '" - << df_rem_gen.link_filepath() - << "' was found but the linked file '" - << df_rem_gen.filepath() - << "' could not be opened or is not correct."; - return(false); - } + /* Look for a remote file-per-table tablespace. */ - /* Use this file if it has the space_id from the MLOG - record. */ - if (df_rem_gen.space_id() == space_id) { - df.set_filepath(df_rem_gen.filepath()); - df.open_read_only(false); - return(true); - } - - /* Since old MLOG records can use the same basename in - multiple CREATE/DROP sequences, this ISL file could be - pointing to a later version of this basename.ibd file - which has a different space_id. Keep looking. */ - } - - /* Look for a remote file-per-table tablespace. */ - if (sep_found == 2) { df_rem_per.set_name(db); if (df_rem_per.open_link_file() == DB_SUCCESS) { @@ -4413,11 +4476,7 @@ fil_ibd_load( fil_node_t* node = UT_LIST_GET_FIRST(space->chain); if (0 != strcmp(innobase_basename(filename), innobase_basename(node->name))) { -#ifdef UNIV_HOTBACKUP - ib::trace() -#else ib::info() -#endif /* UNIV_HOTBACKUP */ << "Ignoring data file '" << filename << "' with space ID " << space->id << ". Another data file called " << node->name @@ -4428,14 +4487,9 @@ fil_ibd_load( return(FIL_LOAD_OK); } - /* If the filepath in the redo log is a default location in or - under the datadir, then just try to open it there. */ Datafile file; file.set_filepath(filename); - Folder folder(filename, dirname_length(filename)); - if (folder_mysql_datadir >= folder) { - file.open_read_only(false); - } + file.open_read_only(false); if (!file.is_open()) { /* The file has been moved or it is a remote datafile. */ @@ -4453,11 +4507,7 @@ fil_ibd_load( os_offset_t minimum_size; case DB_SUCCESS: if (file.space_id() != space_id) { -#ifdef UNIV_HOTBACKUP - ib::trace() -#else /* !UNIV_HOTBACKUP */ ib::info() -#endif /* UNIV_HOTBACKUP */ << "Ignoring data file '" << file.filepath() << "' with space ID " << file.space_id() @@ -4481,17 +4531,10 @@ fil_ibd_load( " single-table tablespace file '" << file.filepath() << "'"; } else if (size < minimum_size) { -#ifndef UNIV_HOTBACKUP ib::error() << "The size of tablespace file '" << file.filepath() << "' is only " << size << ", should be at least " << minimum_size << "!"; -#else - /* In MEB, we work around this error. */ - file.set_space_id(ULINT_UNDEFINED); - file.set_flags(0); - break; -#endif /* !UNIV_HOTBACKUP */ } else { /* Everything is fine so far. */ break; @@ -4500,12 +4543,6 @@ fil_ibd_load( /* Fall through to error handling */ case DB_TABLESPACE_EXISTS: -#ifdef UNIV_HOTBACKUP - if (file.flags() == ~(ulint)0) { - return FIL_LOAD_OK; - } -#endif /* UNIV_HOTBACKUP */ - return(FIL_LOAD_INVALID); default: @@ -4514,69 +4551,17 @@ fil_ibd_load( ut_ad(space == NULL); -#ifdef UNIV_HOTBACKUP - if (file.space_id() == ULINT_UNDEFINED || file.space_id() == 0) { - char* new_path; - - ib::info() << "Renaming tablespace file '" << file.filepath() - << "' with space ID " << file.space_id() << " to " - << file.name() << "_ibbackup_old_vers_<timestamp>" - " because its size " << size() << " is too small" - " (< 4 pages 16 kB each), or the space id in the" - " file header is not sensible. This can happen in" - " an mysqlbackup run, and is not dangerous."; - file.close(); - - new_path = fil_make_ibbackup_old_name(file.filepath()); - - bool success = os_file_rename( - innodb_data_file_key, file.filepath(), new_path); - - ut_a(success); - - ut_free(new_path); - - return(FIL_LOAD_ID_CHANGED); - } - - /* A backup may contain the same space several times, if the space got - renamed at a sensitive time. Since it is enough to have one version of - the space, we rename the file if a space with the same space id - already exists in the tablespace memory cache. We rather rename the - file than delete it, because if there is a bug, we do not want to - destroy valuable data. */ - - mutex_enter(&fil_system->mutex); - space = fil_space_get_by_id(space_id); - mutex_exit(&fil_system->mutex); - - if (space != NULL) { - ib::info() << "Renaming data file '" << file.filepath() - << "' with space ID " << space_id << " to " - << file.name() - << "_ibbackup_old_vers_<timestamp> because space " - << space->name << " with the same id was scanned" - " earlier. This can happen if you have renamed tables" - " during an mysqlbackup run."; - file.close(); - - char* new_path = fil_make_ibbackup_old_name(file.filepath()); - - bool success = os_file_rename( - innodb_data_file_key, file.filepath(), new_path); - - ut_a(success); - - ut_free(new_path); - return(FIL_LOAD_OK); + /* Adjust the memory-based flags that would normally be set by + dict_tf_to_fsp_flags(). In recovery, we have no data dictionary. */ + ulint flags = file.flags(); + if (FSP_FLAGS_HAS_PAGE_COMPRESSION(flags)) { + flags |= page_zip_level + << FSP_FLAGS_MEM_COMPRESSION_LEVEL; } -#endif /* UNIV_HOTBACKUP */ - bool is_temp = FSP_FLAGS_GET_TEMPORARY(file.flags()); space = fil_space_create( - file.name(), space_id, file.flags(), - is_temp ? FIL_TYPE_TEMPORARY : FIL_TYPE_TABLESPACE, - file.get_crypt_info()); + file.name(), space_id, flags, FIL_TYPE_TABLESPACE, + file.get_crypt_info(), false); if (space == NULL) { return(FIL_LOAD_INVALID); @@ -4589,26 +4574,10 @@ fil_ibd_load( the rounding formula for extents and pages is somewhat complex; we let fil_node_open() do that task. */ - if (!fil_node_create_low(file.filepath(), 0, space, - false, true, false)) { + if (!fil_node_create_low(file.filepath(), 0, space, false, false)) { ut_error; } -#ifdef MYSQL_ENCRYPTION - /* For encryption tablespace, initial encryption information. */ - if (FSP_FLAGS_GET_ENCRYPTION(space->flags) - && file.m_encryption_key != NULL) { - dberr_t err = fil_set_encryption(space->id, - Encryption::AES, - file.m_encryption_key, - file.m_encryption_iv); - if (err != DB_SUCCESS) { - ib::error() << "Can't set encryption information for" - " tablespace " << space->name << "!"; - } - } -#endif /* MYSQL_ENCRYPTION */ - return(FIL_LOAD_OK); } @@ -4660,14 +4629,44 @@ fil_report_missing_tablespace( << " in the InnoDB data dictionary has tablespace id " << space_id << "," " but tablespace with that id or name does not exist. Have" - " you deleted or moved .ibd files? This may also be a table" - " created with CREATE TEMPORARY TABLE whose .ibd and .frm" - " files MySQL automatically removed, but the table still" - " exists in the InnoDB internal data dictionary."; + " you deleted or moved .ibd files?"; } -#ifndef UNIV_HOTBACKUP -/** Returns true if a matching tablespace exists in the InnoDB tablespace +/** Try to adjust FSP_SPACE_FLAGS if they differ from the expectations. +(Typically when upgrading from MariaDB 10.1.0..10.1.20.) +@param[in] space_id tablespace ID +@param[in] flags desired tablespace flags */ +UNIV_INTERN +void +fsp_flags_try_adjust(ulint space_id, ulint flags) +{ + ut_ad(!srv_read_only_mode); + ut_ad(fsp_flags_is_valid(flags)); + + mtr_t mtr; + mtr_start(&mtr); + if (buf_block_t* b = buf_page_get( + page_id_t(space_id, 0), page_size_t(flags), + RW_X_LATCH, &mtr)) { + ulint f = fsp_header_get_flags(b->frame); + /* Suppress the message if only the DATA_DIR flag to differs. */ + if ((f ^ flags) & ~(1U << FSP_FLAGS_POS_RESERVED)) { + ib::warn() + << "adjusting FSP_SPACE_FLAGS of tablespace " + << space_id + << " from " << ib::hex(f) + << " to " << ib::hex(flags); + } + if (f != flags) { + mlog_write_ulint(FSP_HEADER_OFFSET + + FSP_SPACE_FLAGS + b->frame, + flags, MLOG_4BYTES, &mtr); + } + } + mtr_commit(&mtr); +} + +/** Determine if a matching tablespace exists in the InnoDB tablespace memory cache. Note that if we have not done a crash recovery at the database startup, there may be many tablespaces which are not yet in the memory cache. @param[in] id Tablespace ID @@ -4679,6 +4678,7 @@ error log if a matching tablespace is not found from memory. @param[in] heap Heap memory @param[in] table_id table id @param[in] table table +@param[in] table_flags table flags @return true if a matching tablespace exists in the memory cache */ bool fil_space_for_table_exists_in_mem( @@ -4688,12 +4688,13 @@ fil_space_for_table_exists_in_mem( bool adjust_space, mem_heap_t* heap, table_id_t table_id, - dict_table_t* table) + dict_table_t* table, + ulint table_flags) { - fil_space_t* fnamespace = NULL; + fil_space_t* fnamespace; fil_space_t* space; - ut_ad(fil_system); + const ulint expected_flags = dict_tf_to_fsp_flags(table_flags); mutex_enter(&fil_system->mutex); @@ -4701,80 +4702,41 @@ fil_space_for_table_exists_in_mem( space = fil_space_get_by_id(id); - /* If tablespace contains encryption information - copy it also to table. */ - if (space && space->crypt_data && - table && !table->crypt_data) { - table->crypt_data = space->crypt_data; - } - - if (space != NULL - && FSP_FLAGS_GET_SHARED(space->flags) - && adjust_space - && srv_sys_tablespaces_open - && 0 == strncmp(space->name, general_space_name, - strlen(general_space_name))) { - /* This name was assigned during recovery in fil_ibd_load(). - This general tablespace was opened from an MLOG_FILE_NAME log - entry where the tablespace name does not exist. Replace the - temporary name with this name and return this space. */ - HASH_DELETE(fil_space_t, name_hash, fil_system->name_hash, - ut_fold_string(space->name), space); - ut_free(space->name); - space->name = mem_strdup(name); - HASH_INSERT(fil_space_t, name_hash, fil_system->name_hash, - ut_fold_string(space->name), space); - - mutex_exit(&fil_system->mutex); - - return(true); - } - - if (space != NULL) { - if (FSP_FLAGS_GET_SHARED(space->flags) - && !srv_sys_tablespaces_open) { - - /* No need to check the name */ - mutex_exit(&fil_system->mutex); - return(true); - } + /* Look if there is a space with the same name; the name is the + directory path from the datadir to the file */ - /* If this space has the expected name, use it. */ - fnamespace = fil_space_get_by_name(name); - if (space == fnamespace) { - /* Found */ + fnamespace = fil_space_get_by_name(name); + bool valid = space && !((space->flags ^ expected_flags) + & ~FSP_FLAGS_MEM_MASK); - mutex_exit(&fil_system->mutex); - return(true); - } + if (valid && table && !table->crypt_data) { + table->crypt_data = space->crypt_data; } - /* Info from "fnamespace" comes from the ibd file itself, it can - be different from data obtained from System tables since file - operations are not transactional. If adjust_space is set, and the - mismatching space are between a user table and its temp table, we - shall adjust the ibd file name according to system table info */ - if (adjust_space - && space != NULL - && row_is_mysql_tmp_table_name(space->name) - && !row_is_mysql_tmp_table_name(name)) { - + if (!space) { + } else if (!valid || space == fnamespace) { + /* Found with the same file name, or got a flag mismatch. */ + goto func_exit; + } else if (adjust_space + && row_is_mysql_tmp_table_name(space->name) + && !row_is_mysql_tmp_table_name(name)) { + /* Info from fnamespace comes from the ibd file + itself, it can be different from data obtained from + System tables since renaming files is not + transactional. We shall adjust the ibd file name + according to system table info. */ mutex_exit(&fil_system->mutex); DBUG_EXECUTE_IF("ib_crash_before_adjust_fil_space", DBUG_SUICIDE();); - if (fnamespace) { - const char* tmp_name; - - tmp_name = dict_mem_create_temporary_tablename( - heap, name, table_id); + const char* tmp_name = dict_mem_create_temporary_tablename( + heap, name, table_id); - fil_rename_tablespace( - fnamespace->id, - UT_LIST_GET_FIRST(fnamespace->chain)->name, - tmp_name, NULL); - } + fil_rename_tablespace( + fnamespace->id, + UT_LIST_GET_FIRST(fnamespace->chain)->name, + tmp_name, NULL); DBUG_EXECUTE_IF("ib_crash_after_adjust_one_fil_space", DBUG_SUICIDE();); @@ -4789,16 +4751,12 @@ fil_space_for_table_exists_in_mem( mutex_enter(&fil_system->mutex); fnamespace = fil_space_get_by_name(name); ut_ad(space == fnamespace); - mutex_exit(&fil_system->mutex); - - return(true); + goto func_exit; } if (!print_error_if_does_not_exist) { - - mutex_exit(&fil_system->mutex); - - return(false); + valid = false; + goto func_exit; } if (space == NULL) { @@ -4817,10 +4775,8 @@ fil_space_for_table_exists_in_mem( } error_exit: ib::warn() << TROUBLESHOOT_DATADICT_MSG; - - mutex_exit(&fil_system->mutex); - - return(false); + valid = false; + goto func_exit; } if (0 != strcmp(space->name, name)) { @@ -4839,11 +4795,21 @@ error_exit: goto error_exit; } +func_exit: + if (valid) { + /* Adjust the flags that are in FSP_FLAGS_MEM_MASK. + FSP_SPACE_FLAGS will not be written back here. */ + space->flags = expected_flags; + } mutex_exit(&fil_system->mutex); - return(false); + if (valid && !srv_read_only_mode) { + fsp_flags_try_adjust(id, expected_flags & ~FSP_FLAGS_MEM_MASK); + } + + return(valid); } -#endif /* !UNIV_HOTBACKUP */ + /** Return the space ID based on the tablespace name. The tablespace must be found in the tablespace memory cache. This call is made from external to this module, so the mutex is not owned. @@ -4864,323 +4830,6 @@ fil_space_get_id_by_name( return(id); } -/** -Fill the pages with NULs -@param[in] node File node -@param[in] page_size physical page size -@param[in] start Offset from the start of the file in bytes -@param[in] len Length in bytes -@param[in] read_only_mode - if true, then read only mode checks are enforced. -@return DB_SUCCESS or error code */ -static -dberr_t -fil_write_zeros( - const fil_node_t* node, - ulint page_size, - os_offset_t start, - ulint len, - bool read_only_mode) -{ - ut_a(len > 0); - - /* Extend at most 1M at a time */ - ulint n_bytes = ut_min(static_cast<ulint>(1024 * 1024), len); - byte* ptr = reinterpret_cast<byte*>(ut_zalloc_nokey(n_bytes - + page_size)); - byte* buf = reinterpret_cast<byte*>(ut_align(ptr, page_size)); - - os_offset_t offset = start; - dberr_t err = DB_SUCCESS; - const os_offset_t end = start + len; - IORequest request(IORequest::WRITE); - - while (offset < end) { - -#ifdef UNIV_HOTBACKUP - err = os_file_write( - request, node->name, node->handle, buf, offset, - n_bytes); -#else - err = os_aio( - request, OS_AIO_SYNC, node->name, - node->handle, buf, offset, n_bytes, read_only_mode, - NULL, NULL, NULL); -#endif /* UNIV_HOTBACKUP */ - - if (err != DB_SUCCESS) { - break; - } - - offset += n_bytes; - - n_bytes = ut_min(n_bytes, static_cast<ulint>(end - offset)); - - DBUG_EXECUTE_IF("ib_crash_during_tablespace_extension", - DBUG_SUICIDE();); - } - - ut_free(ptr); - - return(err); -} - -/** Try to extend a tablespace if it is smaller than the specified size. -@param[in,out] space tablespace -@param[in] size desired size in pages -@return whether the tablespace is at least as big as requested */ -bool -fil_space_extend( - fil_space_t* space, - ulint size) -{ - /* In read-only mode we allow write to shared temporary tablespace - as intrinsic table created by Optimizer reside in this tablespace. */ - ut_ad(!srv_read_only_mode || fsp_is_system_temporary(space->id)); - -retry: - -#ifdef UNIV_HOTBACKUP - page_size_t page_length(space->flags); - ulint actual_size = space->size; - ib::trace() << "space id : " << space->id << ", space name : " - << space->name << ", space size : " << actual_size << " pages," - << " desired space size : " << size << " pages," - << " page size : " << page_length.physical(); -#endif /* UNIV_HOTBACKUP */ - - bool success = true; - - fil_mutex_enter_and_prepare_for_io(space->id); - - if (space->size >= size) { - /* Space already big enough */ - mutex_exit(&fil_system->mutex); - return(true); - } - - page_size_t pageSize(space->flags); - const ulint page_size = pageSize.physical(); - fil_node_t* node = UT_LIST_GET_LAST(space->chain); - - if (!node->being_extended) { - /* Mark this node as undergoing extension. This flag - is used by other threads to wait for the extension - opereation to finish. */ - node->being_extended = true; - } else { - /* Another thread is currently extending the file. Wait - for it to finish. It'd have been better to use an event - driven mechanism but the entire module is peppered with - polling code. */ - - mutex_exit(&fil_system->mutex); - os_thread_sleep(100000); - goto retry; - } - - if (!fil_node_prepare_for_io(node, fil_system, space)) { - /* The tablespace data file, such as .ibd file, is missing */ - node->being_extended = false; - mutex_exit(&fil_system->mutex); - - return(false); - } - - /* At this point it is safe to release fil_system mutex. No - other thread can rename, delete or close the file because - we have set the node->being_extended flag. */ - mutex_exit(&fil_system->mutex); - - ulint pages_added; - - /* Note: This code is going to be executed independent of FusionIO HW - if the OS supports posix_fallocate() */ - - ut_ad(size > space->size); - - os_offset_t node_start = os_file_get_size(node->handle); - ut_a(node_start != (os_offset_t) -1); - - /* Node first page number */ - ulint node_first_page = space->size - node->size; - - /* Number of physical pages in the node/file */ - ulint n_node_physical_pages - = static_cast<ulint>(node_start) / page_size; - - /* Number of pages to extend in the node/file */ - lint n_node_extend; - - n_node_extend = size - (node_first_page + node->size); - - /* If we already have enough physical pages to satisfy the - extend request on the node then ignore it */ - if (node->size + n_node_extend > n_node_physical_pages) { - - DBUG_EXECUTE_IF("ib_crash_during_tablespace_extension", - DBUG_SUICIDE();); - - os_offset_t len; - dberr_t err = DB_SUCCESS; - - len = ((node->size + n_node_extend) * page_size) - node_start; - ut_ad(len > 0); - const char* name = node->name == NULL ? space->name : node->name; - -#if !defined(NO_FALLOCATE) && defined(UNIV_LINUX) - /* This is required by FusionIO HW/Firmware */ - int ret = posix_fallocate(node->handle, node_start, len); - - /* We already pass the valid offset and len in, if EINVAL - is returned, it could only mean that the file system doesn't - support fallocate(), currently one known case is - ext3 FS with O_DIRECT. We ignore EINVAL here so that the - error message won't flood. */ - if (ret != 0 && ret != EINVAL) { - ib::error() - << "posix_fallocate(): Failed to preallocate" - " data for file " - << name << ", desired size " - << len << " bytes." - " Operating system error number " - << ret << ". Check" - " that the disk is not full or a disk quota" - " exceeded. Make sure the file system supports" - " this function. Some operating system error" - " numbers are described at " REFMAN - " operating-system-error-codes.html"; - - err = DB_IO_ERROR; - } -#endif /* NO_FALLOCATE || !UNIV_LINUX */ - - if (!node->atomic_write || err == DB_IO_ERROR) { - - bool read_only_mode; - - read_only_mode = (space->purpose != FIL_TYPE_TEMPORARY - ? false : srv_read_only_mode); - - err = fil_write_zeros( - node, page_size, node_start, - static_cast<ulint>(len), read_only_mode); - - if (err != DB_SUCCESS) { - - ib::warn() - << "Error while writing " << len - << " zeroes to " << name - << " starting at offset " << node_start; - } - } - - /* Check how many pages actually added */ - os_offset_t end = os_file_get_size(node->handle); - ut_a(end != static_cast<os_offset_t>(-1) && end >= node_start); - - os_has_said_disk_full = !(success = (end == node_start + len)); - - pages_added = static_cast<ulint>(end - node_start) / page_size; - - } else { - success = true; - pages_added = n_node_extend; - os_has_said_disk_full = FALSE; - } - - mutex_enter(&fil_system->mutex); - - ut_a(node->being_extended); - - node->size += pages_added; - space->size += pages_added; - node->being_extended = false; - - fil_node_complete_io(node, fil_system, IORequestWrite); - -#ifndef UNIV_HOTBACKUP - /* Keep the last data file size info up to date, rounded to - full megabytes */ - ulint pages_per_mb = (1024 * 1024) / page_size; - ulint size_in_pages = ((node->size / pages_per_mb) * pages_per_mb); - - if (space->id == srv_sys_space.space_id()) { - srv_sys_space.set_last_file_size(size_in_pages); - } else if (space->id == srv_tmp_space.space_id()) { - srv_tmp_space.set_last_file_size(size_in_pages); - } -#else - ib::trace() << "extended space : " << space->name << " from " - << actual_size << " pages to " << space->size << " pages " - << ", desired space size : " << size << " pages."; -#endif /* !UNIV_HOTBACKUP */ - - mutex_exit(&fil_system->mutex); - - fil_flush(space->id); - - return(success); -} - -#ifdef UNIV_HOTBACKUP -/********************************************************************//** -Extends all tablespaces to the size stored in the space header. During the -mysqlbackup --apply-log phase we extended the spaces on-demand so that log -records could be applied, but that may have left spaces still too small -compared to the size stored in the space header. */ -void -fil_extend_tablespaces_to_stored_len(void) -/*======================================*/ -{ - byte* buf; - ulint actual_size; - ulint size_in_header; - dberr_t error; - bool success; - - buf = (byte*)ut_malloc_nokey(UNIV_PAGE_SIZE); - - mutex_enter(&fil_system->mutex); - - for (fil_space_t* space = UT_LIST_GET_FIRST(fil_system->space_list); - space != NULL; - space = UT_LIST_GET_NEXT(space_list, space)) { - - ut_a(space->purpose == FIL_TYPE_TABLESPACE); - - mutex_exit(&fil_system->mutex); /* no need to protect with a - mutex, because this is a - single-threaded operation */ - error = fil_read( - page_id_t(space->id, 0), - page_size_t(space->flags), - 0, univ_page_size.physical(), buf); - - ut_a(error == DB_SUCCESS); - - size_in_header = fsp_header_get_field(buf, FSP_SIZE); - - success = fil_space_extend(space, size_in_header); - if (!success) { - ib::error() << "Could not extend the tablespace of " - << space->name << " to the size stored in" - " header, " << size_in_header << " pages;" - " size after extension " << actual_size - << " pages. Check that you have free disk" - " space and retry!"; - ut_a(success); - } - - mutex_enter(&fil_system->mutex); - } - - mutex_exit(&fil_system->mutex); - - ut_free(buf); -} -#endif - /*========== RESERVE FREE EXTENTS (for a B-tree split, for example) ===*/ /*******************************************************************//** @@ -5292,7 +4941,7 @@ fil_node_prepare_for_io( << " exceeds the limit " << system->max_n_open; } - if (!node->is_open) { + if (!node->is_open()) { /* File is closed: open it */ ut_a(node->n_pending == 0); @@ -5396,33 +5045,6 @@ fil_report_invalid_page_access( _exit(1); } -#ifdef MYSQL_ENCRYPTION -/** Set encryption information for IORequest. -@param[in,out] req_type IO request -@param[in] page_id page id -@param[in] space table space */ -inline -void -fil_io_set_encryption( - IORequest& req_type, - const page_id_t& page_id, - fil_space_t* space) -{ - /* Don't encrypt the log, page 0 of all tablespaces, all pages - from the system tablespace. */ - if (!req_type.is_log() && page_id.page_no() > 0 - && space->encryption_type != Encryption::NONE) - { - req_type.encryption_key(space->encryption_key, - space->encryption_klen, - space->encryption_iv); - req_type.encryption_algorithm(Encryption::AES); - } else { - req_type.clear_encrypted(); - } -} -#endif /* MYSQL_ENCRYPTION */ - /** Reads or writes data. This operation could be asynchronous (aio). @param[in,out] type IO context @@ -5439,8 +5061,6 @@ fil_io_set_encryption( aligned @param[in] message message for aio handler if non-sync aio used, else ignored -@param[in] write_size actual payload size when written - to avoid extra punch holes in compression @return DB_SUCCESS, DB_TABLESPACE_DELETED or DB_TABLESPACE_TRUNCATED if we are trying to do i/o on a tablespace which does not exist */ dberr_t @@ -5452,8 +5072,7 @@ fil_io( ulint byte_offset, ulint len, void* buf, - void* message, - ulint* write_size) + void* message) { os_offset_t offset; IORequest req_type(type); @@ -5472,8 +5091,6 @@ fil_io( #endif ut_ad(fil_validate_skip()); -#ifndef UNIV_HOTBACKUP - /* ibuf bitmap pages must be read in the sync AIO mode: */ ut_ad(recv_no_ibuf_operations || req_type.is_write() @@ -5504,12 +5121,7 @@ fil_io( } else { mode = OS_AIO_NORMAL; } -#else /* !UNIV_HOTBACKUP */ - ut_a(sync); - ulint mode = OS_AIO_SYNC; -#endif /* !UNIV_HOTBACKUP */ -#ifndef UNIV_HOTBACKUP if (req_type.is_read()) { srv_stats.data_read.add(len); @@ -5521,7 +5133,6 @@ fil_io( srv_stats.data_written.add(len); } -#endif /* !UNIV_HOTBACKUP */ /* Reserve the fil_system mutex and make sure that we can open at least one file while holding it, if the file is not already open */ @@ -5694,76 +5305,15 @@ fil_io( const char* name = node->name == NULL ? space->name : node->name; -#ifdef MYSQL_COMPRESSION - /* Don't compress the log, page 0 of all tablespaces, tables - compresssed with the old scheme and all pages from the system - tablespace. */ - - if (req_type.is_write() - && !req_type.is_log() - && !page_size.is_compressed() - && page_id.page_no() > 0 - && IORequest::is_punch_hole_supported() - && node->punch_hole) { - - ut_ad(!req_type.is_log()); - - req_type.set_punch_hole(); - - req_type.compression_algorithm(space->compression_type); - - } else { - req_type.clear_compressed(); - } -#endif /* MYSQL_COMPRESSION */ + req_type.set_fil_node(node); -#ifdef MYSQL_ENCRYPTION - /* Set encryption information. */ - fil_io_set_encryption(req_type, page_id, space); -#endif /* MYSQL_ENCRYPTION */ - - req_type.block_size(node->block_size); - - dberr_t err; - -#ifdef UNIV_HOTBACKUP - /* In mysqlbackup do normal i/o, not aio */ - if (req_type.is_read()) { - - err = os_file_read(req_type, node->handle, buf, offset, len); - - } else { - - ut_ad(!srv_read_only_mode - || fsp_is_system_temporary(page_id.space())); - - err = os_file_write( - req_type, node->name, node->handle, buf, offset, len); - } -#else /* UNIV_HOTBACKUP */ /* Queue the aio request */ - err = os_aio( + dberr_t err = os_aio( req_type, mode, name, node->handle, buf, offset, len, - fsp_is_system_temporary(page_id.space()) - ? false : srv_read_only_mode, - node, message, write_size); - -#endif /* UNIV_HOTBACKUP */ - - if (err == DB_IO_NO_PUNCH_HOLE) { - - err = DB_SUCCESS; - - if (node->punch_hole) { - - ib::warn() - << "Punch hole failed for '" - << name << "'"; - } - - fil_no_punch_hole(node); - } + space->purpose != FIL_TYPE_TEMPORARY + && srv_read_only_mode, + node, message); /* We an try to recover the page from the double write buffer if the decompression fails or the page is corrupt. */ @@ -5786,7 +5336,6 @@ fil_io( return(err); } -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Waits for an aio operation to complete. This function is used to write the handler for completed requests. The aio array of pending requests is divided @@ -5849,7 +5398,6 @@ fil_aio_wait( ut_ad(0); } -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Flushes to disk possible writes cached by the OS. If the space does not exist @@ -5860,146 +5408,16 @@ fil_flush( ulint space_id) /*!< in: file space id (this can be a group of log files or a tablespace of the database) */ { - fil_node_t* node; - os_file_t file; - mutex_enter(&fil_system->mutex); - fil_space_t* space = fil_space_get_by_id(space_id); - - if (space == NULL - || space->purpose == FIL_TYPE_TEMPORARY - || space->stop_new_ops - || space->is_being_truncated) { - mutex_exit(&fil_system->mutex); - - return; - } - - if (fil_buffering_disabled(space)) { - - /* No need to flush. User has explicitly disabled - buffering. */ - ut_ad(!space->is_in_unflushed_spaces); - ut_ad(fil_space_is_flushed(space)); - ut_ad(space->n_pending_flushes == 0); - -#ifdef UNIV_DEBUG - for (node = UT_LIST_GET_FIRST(space->chain); - node != NULL; - node = UT_LIST_GET_NEXT(chain, node)) { - ut_ad(node->modification_counter - == node->flush_counter); - ut_ad(node->n_pending_flushes == 0); - } -#endif /* UNIV_DEBUG */ - - mutex_exit(&fil_system->mutex); - return; - } - - space->n_pending_flushes++; /*!< prevent dropping of the space while - we are flushing */ - for (node = UT_LIST_GET_FIRST(space->chain); - node != NULL; - node = UT_LIST_GET_NEXT(chain, node)) { - - int64_t old_mod_counter = node->modification_counter; - - if (old_mod_counter <= node->flush_counter) { - continue; - } - - ut_a(node->is_open); - - switch (space->purpose) { - case FIL_TYPE_TEMPORARY: - ut_ad(0); // we already checked for this - case FIL_TYPE_TABLESPACE: - case FIL_TYPE_IMPORT: - fil_n_pending_tablespace_flushes++; - break; - case FIL_TYPE_LOG: - fil_n_pending_log_flushes++; - fil_n_log_flushes++; - break; - } -#ifdef _WIN32 - if (node->is_raw_disk) { - - goto skip_flush; - } -#endif /* _WIN32 */ -retry: - if (node->n_pending_flushes > 0) { - /* We want to avoid calling os_file_flush() on - the file twice at the same time, because we do - not know what bugs OS's may contain in file - i/o */ - -#ifndef UNIV_HOTBACKUP - int64_t sig_count = os_event_reset(node->sync_event); -#endif /* !UNIV_HOTBACKUP */ - - mutex_exit(&fil_system->mutex); - - os_event_wait_low(node->sync_event, sig_count); - - mutex_enter(&fil_system->mutex); - - if (node->flush_counter >= old_mod_counter) { - - goto skip_flush; - } - - goto retry; - } - - ut_a(node->is_open); - file = node->handle; - node->n_pending_flushes++; - - mutex_exit(&fil_system->mutex); - - os_file_flush(file); - - mutex_enter(&fil_system->mutex); - - os_event_set(node->sync_event); - - node->n_pending_flushes--; -skip_flush: - if (node->flush_counter < old_mod_counter) { - node->flush_counter = old_mod_counter; - - if (space->is_in_unflushed_spaces - && fil_space_is_flushed(space)) { - - space->is_in_unflushed_spaces = false; - - UT_LIST_REMOVE( - fil_system->unflushed_spaces, - space); - } - } - - switch (space->purpose) { - case FIL_TYPE_TEMPORARY: - ut_ad(0); // we already checked for this - case FIL_TYPE_TABLESPACE: - case FIL_TYPE_IMPORT: - fil_n_pending_tablespace_flushes--; - continue; - case FIL_TYPE_LOG: - fil_n_pending_log_flushes--; - continue; + if (fil_space_t* space = fil_space_get_by_id(space_id)) { + if (space->purpose != FIL_TYPE_TEMPORARY + && !space->stop_new_ops + && !space->is_being_truncated) { + fil_flush_low(space); } - - ut_ad(0); } - space->n_pending_flushes--; - mutex_exit(&fil_system->mutex); } @@ -6072,8 +5490,8 @@ struct Check { @param[in] elem file node to visit */ void operator()(const fil_node_t* elem) { - ut_a(elem->is_open || !elem->n_pending); - n_open += elem->is_open; + ut_a(elem->is_open() || !elem->n_pending); + n_open += elem->is_open(); size += elem->size; } @@ -6127,7 +5545,7 @@ fil_validate(void) ut_a(fil_node->n_pending == 0); ut_a(!fil_node->being_extended); - ut_a(fil_node->is_open); + ut_a(fil_node->is_open()); ut_a(fil_space_belongs_in_lru(fil_node->space)); } @@ -6182,7 +5600,6 @@ fil_page_set_type( mach_write_to_2(page + FIL_PAGE_TYPE, type); } -#ifndef UNIV_HOTBACKUP /** Reset the page type. Data files created before MySQL 5.1 may contain garbage in FIL_PAGE_TYPE. In MySQL 3.23.53, only undo log pages and index pages were tagged. @@ -6203,7 +5620,6 @@ fil_page_reset_type( << fil_page_get_type(page) << " to " << type << "."; mlog_write_ulint(page + FIL_PAGE_TYPE, type, MLOG_2BYTES, mtr); } -#endif /* !UNIV_HOTBACKUP */ /****************************************************************//** Closes the tablespace memory cache. */ @@ -6224,10 +5640,11 @@ fil_close(void) ut_free(fil_system); fil_system = NULL; + + fil_space_crypt_cleanup(); } } -#ifndef UNIV_HOTBACKUP /********************************************************************//** Initializes a buffer control block when the buf_pool is created. */ static @@ -6260,9 +5677,9 @@ struct fil_iterator_t { for IO */ byte* io_buffer; /*!< Buffer to use for IO */ fil_space_crypt_t *crypt_data; /*!< MariaDB Crypt data (if encrypted) */ - byte* crypt_io_buffer; /*!< MariaDB IO buffer when encrypted */ - byte* encryption_key; /*!< Encryption key */ - byte* encryption_iv; /*!< Encryption iv */ + byte* crypt_io_buffer; /*!< MariaDB IO buffer when + encrypted */ + dict_table_t* table; /*!< Imported table */ }; /********************************************************************//** @@ -6301,11 +5718,13 @@ fil_iterate( for (offset = iter.start; offset < iter.end; offset += n_bytes) { - byte* io_buffer = iter.io_buffer; + byte* io_buffer = iter.io_buffer; + const bool row_compressed + = callback.get_page_size().is_compressed(); block->frame = io_buffer; - if (callback.get_page_size().is_compressed()) { + if (row_compressed) { page_zip_des_init(&block->page.zip); page_zip_set_size(&block->page.zip, iter.page_size); @@ -6338,24 +5757,12 @@ fil_iterate( dberr_t err = DB_SUCCESS; IORequest read_request(read_type); -#ifdef MYSQL_ENCRYPTION - /* For encrypted table, set encryption information. */ - if (iter.encryption_key != NULL && offset != 0) { - read_request.encryption_key(iter.encryption_key, - ENCRYPTION_KEY_LEN, - iter.encryption_iv); - read_request.encryption_algorithm(Encryption::AES); - } -#endif /* MYSQL_ENCRYPTION */ - byte* readptr = io_buffer; byte* writeptr = io_buffer; bool encrypted = false; /* Use additional crypt io buffer if tablespace is encrypted */ - if ((iter.crypt_data != NULL && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) || - (srv_encrypt_tables && - iter.crypt_data && iter.crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { + if (iter.crypt_data != NULL && iter.crypt_data->should_encrypt()) { encrypted = true; readptr = iter.crypt_io_buffer; @@ -6379,9 +5786,11 @@ fil_iterate( bool decrypted = false; for (ulint i = 0; i < n_pages_read; ++i) { - ulint size = iter.page_size; - byte* src = (readptr + (i * size)); - byte* dst = (io_buffer + (i * size)); + ulint size = iter.page_size; + dberr_t err = DB_SUCCESS; + byte* src = readptr + (i * size); + byte* dst = io_buffer + (i * size); + bool frame_changed = false; ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); @@ -6405,9 +5814,12 @@ fil_iterate( if (decrypted) { updated = true; + } else if (!page_compressed + && !row_compressed) { + block->frame = src; + frame_changed = true; } else { - /* TODO: remove unnecessary memcpy's */ - memcpy(dst, src, iter.page_size); + memcpy(dst, src, size); } } @@ -6433,20 +5845,63 @@ fil_iterate( buf_block_set_state(block, BUF_BLOCK_NOT_USED); buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); - src = (io_buffer + (i * size)); + /* If tablespace is encrypted we use additional + temporary scratch area where pages are read + for decrypting readptr == crypt_io_buffer != io_buffer. + + Destination for decryption is a buffer pool block + block->frame == dst == io_buffer that is updated. + Pages that did not require decryption even when + tablespace is marked as encrypted are not copied + instead block->frame is set to src == readptr. + + For encryption we again use temporary scratch area + writeptr != io_buffer == dst + that is then written to the tablespace + + (1) For normal tables io_buffer == dst == writeptr + (2) For only page compressed tables + io_buffer == dst == writeptr + (3) For encrypted (and page compressed) + readptr != io_buffer == dst != writeptr + */ + + ut_ad(!encrypted && !page_compressed ? + src == dst && dst == writeptr + (i * size):1); + ut_ad(page_compressed && !encrypted ? + src == dst && dst == writeptr + (i * size):1); + ut_ad(encrypted ? + src != dst && dst != writeptr + (i * size):1); + + if (encrypted) { + memcpy(writeptr + (i * size), + row_compressed ? block->page.zip.data : + block->frame, size); + } + + if (frame_changed) { + block->frame = dst; + } + + src = io_buffer + (i * size); if (page_compressed) { ulint len = 0; - fil_compress_page(space_id, + + byte * res = fil_compress_page(space_id, src, NULL, size, - fil_space_get_page_compression_level(space_id), + dict_table_page_compression_level(iter.table), fil_space_get_block_size(space_id, offset, size), encrypted, &len, NULL); + if (len != size) { + memset(res+len, 0, size-len); + } + updated = true; } @@ -6454,7 +5909,7 @@ fil_iterate( write it back. Note that we should not encrypt the buffer that is in buffer pool. */ if (decrypted && encrypted) { - unsigned char *dest = (writeptr + (i * size)); + byte *dest = writeptr + (i * size); ulint space = mach_read_from_4( src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET); @@ -6483,16 +5938,6 @@ fil_iterate( IORequest write_request(write_type); -#ifdef MYSQL_ENCRYPTION - /* For encrypted table, set encryption information. */ - if (iter.encryption_key != NULL && offset != 0) { - write_request.encryption_key(iter.encryption_key, - ENCRYPTION_KEY_LEN, - iter.encryption_iv); - write_request.encryption_algorithm(Encryption::AES); - } -#endif /* MYSQL_ENCRYPTION */ - /* A page was updated in the set, write back to disk. Note: We don't have the compression algorithm, we write out the imported file as uncompressed. */ @@ -6503,18 +5948,12 @@ fil_iterate( iter.filepath, iter.file, writeptr, offset, (ulint) n_bytes)) != DB_SUCCESS) { - /* This is not a hard error */ - if (err == DB_IO_NO_PUNCH_HOLE) { - - err = DB_SUCCESS; - write_type &= ~IORequest::PUNCH_HOLE; - - } else { - ib::error() << "os_file_write() failed"; - - return(err); - } + ib::error() << "os_file_write() failed"; + return(err); } + + /* Clean up the temporal buffer. */ + memset(writeptr, 0, n_bytes); } return(DB_SUCCESS); @@ -6633,33 +6072,13 @@ fil_tablespace_iterate( iter.file_size = file_size; iter.n_io_buffers = n_io_buffers; iter.page_size = callback.get_page_size().physical(); - - ulint crypt_data_offset = fsp_header_get_crypt_offset( - callback.get_page_size(), 0); + iter.table = table; /* read (optional) crypt data */ iter.crypt_data = fil_space_read_crypt_data( - 0, page, crypt_data_offset); - -#ifdef MYSQL_ENCRYPTION - /* Set encryption info. */ - iter.encryption_key = table->encryption_key; - iter.encryption_iv = table->encryption_iv; - - /* Check encryption is matched or not. */ - ulint space_flags = callback.get_space_flags(); - if (FSP_FLAGS_GET_ENCRYPTION(space_flags)) { - ut_ad(table->encryption_key != NULL); - - if (!dict_table_is_encrypted(table)) { - ib::error() << "Table is not in an encrypted" - " tablespace, but the data file which" - " trying to import is an encrypted" - " tablespace"; - err = DB_IO_NO_ENCRYPT_TABLESPACE; - } - } -#endif /* MYSQL_ENCRYPTION */ + 0, page, FSP_HEADER_OFFSET + + fsp_header_get_encryption_offset( + callback.get_page_size())); if (err == DB_SUCCESS) { @@ -6680,18 +6099,24 @@ fil_tablespace_iterate( iter.io_buffer = static_cast<byte*>( ut_align(io_buffer, UNIV_PAGE_SIZE)); - /** Add an exta buffer for encryption */ - void* crypt_io_buffer = NULL; - if (iter.crypt_data != NULL) { - crypt_io_buffer = ut_malloc_nokey( - iter.n_io_buffers * UNIV_PAGE_SIZE); + void* crypt_io_buffer; + if (iter.crypt_data) { + crypt_io_buffer = static_cast<byte*>( + ut_malloc_nokey((2 + iter.n_io_buffers) + * UNIV_PAGE_SIZE)); iter.crypt_io_buffer = static_cast<byte*>( - crypt_io_buffer); + ut_align(crypt_io_buffer, + UNIV_PAGE_SIZE)); + } else { + crypt_io_buffer = NULL; } err = fil_iterate(iter, block, callback); ut_free(io_buffer); + ut_free(crypt_io_buffer); + + fil_space_destroy_crypt_data(&iter.crypt_data); } } @@ -6718,16 +6143,6 @@ fil_tablespace_iterate( return(err); } -#endif /* !UNIV_HOTBACKUP */ - -/** Set the tablespace table size. -@param[in] page a page belonging to the tablespace */ -void -PageCallback::set_page_size( - const buf_frame_t* page) UNIV_NOTHROW -{ - m_page_size.copy_from(fsp_header_get_page_size(page)); -} /********************************************************************//** Delete the tablespace file and any related files like .cfg. @@ -6797,7 +6212,6 @@ fil_get_space_names( return(err); } -#ifndef UNIV_HOTBACKUP /** Generate redo log for swapping two .ibd files @param[in] old_table old table @param[in] new_table new table @@ -6814,12 +6228,10 @@ fil_mtr_rename_log( dberr_t err; bool old_is_file_per_table = - !is_system_tablespace(old_table->space) - && !DICT_TF_HAS_SHARED_SPACE(old_table->flags); + !is_system_tablespace(old_table->space); bool new_is_file_per_table = - !is_system_tablespace(new_table->space) - && !DICT_TF_HAS_SHARED_SPACE(new_table->flags); + !is_system_tablespace(new_table->space); /* If neither table is file-per-table, there will be no renaming of files. */ @@ -6896,7 +6308,7 @@ fil_mtr_rename_log( return(DB_SUCCESS); } -#endif /* !UNIV_HOTBACKUP */ + #ifdef UNIV_DEBUG /** Check that a tablespace is valid for mtr_commit(). @param[in] space persistent tablespace that has been changed */ @@ -6984,7 +6396,7 @@ fil_names_dirty_and_write( bogus_name, mtr); }); } -#ifndef UNIV_HOTBACKUP + /** On a log checkpoint, reset fil_names_dirty_and_write() flags and write out MLOG_FILE_NAME and MLOG_CHECKPOINT if needed. @param[in] lsn checkpoint LSN @@ -7064,15 +6476,12 @@ truncate_t::truncate( { dberr_t err = DB_SUCCESS; char* path; - bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags); ut_a(!is_system_tablespace(space_id)); - if (has_data_dir) { + if (FSP_FLAGS_HAS_DATA_DIR(flags)) { ut_ad(dir_path != NULL); - path = fil_make_filepath(dir_path, tablename, IBD, true); - } else { path = fil_make_filepath(NULL, tablename, IBD, false); } @@ -7095,7 +6504,7 @@ truncate_t::truncate( space->size = node->size = FIL_IBD_FILE_INITIAL_SIZE; } - const bool already_open = node->is_open; + const bool already_open = node->is_open(); if (!already_open) { @@ -7116,7 +6525,7 @@ truncate_t::truncate( return(DB_ERROR); } - node->is_open = true; + ut_a(node->is_open()); } os_offset_t trunc_size = trunc_to_default @@ -7146,7 +6555,7 @@ truncate_t::truncate( err = DB_ERROR; } else { - node->is_open = false; + node->handle = OS_FILE_CLOSED; } } @@ -7156,340 +6565,6 @@ truncate_t::truncate( return(err); } -#endif /* !UNIV_HOTBACKUP */ - -/** -Note that the file system where the file resides doesn't support PUNCH HOLE. -Called from AIO handlers when IO returns DB_IO_NO_PUNCH_HOLE -@param[in,out] node Node to set */ -void -fil_no_punch_hole(fil_node_t* node) -{ - node->punch_hole = false; -} - -#ifdef MYSQL_COMPRESSION - -/** Set the compression type for the tablespace of a table -@param[in] table The table that should be compressed -@param[in] algorithm Text representation of the algorithm -@return DB_SUCCESS or error code */ -dberr_t -fil_set_compression( - dict_table_t* table, - const char* algorithm) -{ - ut_ad(table != NULL); - - /* We don't support Page Compression for the system tablespace, - the temporary tablespace, or any general tablespace because - COMPRESSION is set by TABLE DDL, not TABLESPACE DDL. There is - no other technical reason. Also, do not use it for missing - tables or tables with compressed row_format. */ - if (table->ibd_file_missing - || !DICT_TF2_FLAG_IS_SET(table, DICT_TF2_USE_FILE_PER_TABLE) - || DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY) - || page_size_t(table->flags).is_compressed()) { - - return(DB_IO_NO_PUNCH_HOLE_TABLESPACE); - } - - dberr_t err; - Compression compression; - - if (algorithm == NULL || strlen(algorithm) == 0) { - -#ifndef UNIV_DEBUG - compression.m_type = Compression::NONE; -#else - /* This is a Debug tool for setting compression on all - compressible tables not otherwise specified. */ - switch (srv_debug_compress) { - case Compression::LZ4: - case Compression::ZLIB: - case Compression::NONE: - - compression.m_type = - static_cast<Compression::Type>( - srv_debug_compress); - break; - - default: - compression.m_type = Compression::NONE; - } - -#endif /* UNIV_DEBUG */ - - err = DB_SUCCESS; - - } else { - - err = Compression::check(algorithm, &compression); - } - - fil_space_t* space = fil_space_get(table->space); - - if (space == NULL) { - return(DB_NOT_FOUND); - } - - space->compression_type = compression.m_type; - - if (space->compression_type != Compression::NONE) { - - const fil_node_t* node; - - node = UT_LIST_GET_FIRST(space->chain); - - if (!node->punch_hole) { - - return(DB_IO_NO_PUNCH_HOLE_FS); - } - } - - return(err); -} - -/** Get the compression algorithm for a tablespace. -@param[in] space_id Space ID to check -@return the compression algorithm */ -Compression::Type -fil_get_compression( - ulint space_id) -{ - fil_space_t* space = fil_space_get(space_id); - - return(space == NULL ? Compression::NONE : space->compression_type); -} - -/** Set the encryption type for the tablespace -@param[in] space_id Space ID of tablespace for which to set -@param[in] algorithm Encryption algorithm -@param[in] key Encryption key -@param[in] iv Encryption iv -@return DB_SUCCESS or error code */ -dberr_t -fil_set_encryption( - ulint space_id, - Encryption::Type algorithm, - byte* key, - byte* iv) -{ - ut_ad(!is_system_or_undo_tablespace(space_id)); - - if (is_system_tablespace(space_id)) { - return(DB_IO_NO_ENCRYPT_TABLESPACE); - } - - mutex_enter(&fil_system->mutex); - - fil_space_t* space = fil_space_get_by_id(space_id); - - if (space == NULL) { - mutex_exit(&fil_system->mutex); - return(DB_NOT_FOUND); - } - - ut_ad(algorithm != Encryption::NONE); - space->encryption_type = algorithm; - if (key == NULL) { - Encryption::random_value(space->encryption_key); - } else { - memcpy(space->encryption_key, - key, ENCRYPTION_KEY_LEN); - } - - space->encryption_klen = ENCRYPTION_KEY_LEN; - if (iv == NULL) { - Encryption::random_value(space->encryption_iv); - } else { - memcpy(space->encryption_iv, - iv, ENCRYPTION_KEY_LEN); - } - - mutex_exit(&fil_system->mutex); - - return(DB_SUCCESS); -} - -/** Rotate the tablespace keys by new master key. -@return true if the re-encrypt suceeds */ -bool -fil_encryption_rotate() -{ - fil_space_t* space; - mtr_t mtr; - byte encrypt_info[ENCRYPTION_INFO_SIZE_V2]; - - for (space = UT_LIST_GET_FIRST(fil_system->space_list); - space != NULL; ) { - /* Skip unencypted tablespaces. */ - if (is_system_or_undo_tablespace(space->id) - || fsp_is_system_temporary(space->id) - || space->purpose == FIL_TYPE_LOG) { - space = UT_LIST_GET_NEXT(space_list, space); - continue; - } - - if (space->encryption_type != Encryption::NONE) { - mtr_start(&mtr); - mtr.set_named_space(space->id); - - space = mtr_x_lock_space(space->id, &mtr); - - memset(encrypt_info, 0, ENCRYPTION_INFO_SIZE_V2); - - if (!fsp_header_rotate_encryption(space, - encrypt_info, - &mtr)) { - mtr_commit(&mtr); - return(false); - } - - mtr_commit(&mtr); - } - - space = UT_LIST_GET_NEXT(space_list, space); - DBUG_EXECUTE_IF("ib_crash_during_rotation_for_encryption", - DBUG_SUICIDE();); - } - - return(true); -} -#endif /* MYSQL_COMPRESSION */ - -/** Build the basic folder name from the path and length provided -@param[in] path pathname (may also include the file basename) -@param[in] len length of the path, in bytes */ -void -Folder::make_path(const char* path, size_t len) -{ - if (is_absolute_path(path)) { - m_folder = mem_strdupl(path, len); - m_folder_len = len; - } - else { - size_t n = 2 + len + strlen(fil_path_to_mysql_datadir); - m_folder = static_cast<char*>(ut_malloc_nokey(n)); - m_folder_len = 0; - - if (path != fil_path_to_mysql_datadir) { - /* Put the mysqld datadir into m_folder first. */ - ut_ad(fil_path_to_mysql_datadir[0] != '\0'); - m_folder_len = strlen(fil_path_to_mysql_datadir); - memcpy(m_folder, fil_path_to_mysql_datadir, - m_folder_len); - if (m_folder[m_folder_len - 1] != OS_PATH_SEPARATOR) { - m_folder[m_folder_len++] = OS_PATH_SEPARATOR; - } - } - - /* Append the path. */ - memcpy(m_folder + m_folder_len, path, len); - m_folder_len += len; - m_folder[m_folder_len] = '\0'; - } - - os_normalize_path(m_folder); -} - -/** Resolve a relative path in m_folder to an absolute path -in m_abs_path setting m_abs_len. */ -void -Folder::make_abs_path() -{ - my_realpath(m_abs_path, m_folder, MYF(0)); - m_abs_len = strlen(m_abs_path); - - ut_ad(m_abs_len + 1 < sizeof(m_abs_path)); - - /* Folder::related_to() needs a trailing separator. */ - if (m_abs_path[m_abs_len - 1] != OS_PATH_SEPARATOR) { - m_abs_path[m_abs_len] = OS_PATH_SEPARATOR; - m_abs_path[++m_abs_len] = '\0'; - } -} - -/** Constructor -@param[in] path pathname (may also include the file basename) -@param[in] len length of the path, in bytes */ -Folder::Folder(const char* path, size_t len) -{ - make_path(path, len); - make_abs_path(); -} - -/** Assignment operator -@param[in] folder folder string provided */ -class Folder& -Folder::operator=(const char* path) -{ - ut_free(m_folder); - make_path(path, strlen(path)); - make_abs_path(); - - return(*this); -} - -/** Determine if two folders are equal -@param[in] other folder to compare to -@return whether the folders are equal */ -bool Folder::operator==(const Folder& other) const -{ - return(m_abs_len == other.m_abs_len - && !memcmp(m_abs_path, other.m_abs_path, m_abs_len)); -} - -/** Determine if the left folder is the same or an ancestor of -(contains) the right folder. -@param[in] other folder to compare to -@return whether this is the same or an ancestor of the other folder. */ -bool Folder::operator>=(const Folder& other) const -{ - return(m_abs_len <= other.m_abs_len - && (!memcmp(other.m_abs_path, m_abs_path, m_abs_len))); -} - -/** Determine if the left folder is an ancestor of (contains) -the right folder. -@param[in] other folder to compare to -@return whether this is an ancestor of the other folder */ -bool Folder::operator>(const Folder& other) const -{ - return(m_abs_len < other.m_abs_len - && (!memcmp(other.m_abs_path, m_abs_path, m_abs_len))); -} - -/** Determine if the directory referenced by m_folder exists. -@return whether the directory exists */ -bool -Folder::exists() -{ - bool exists; - os_file_type_t type; - -#ifdef _WIN32 - /* Temporarily strip the trailing_separator since it will cause - _stat64() to fail on Windows unless the path is the root of some - drive; like "c:\". _stat64() will fail if it is "c:". */ - size_t len = strlen(m_abs_path); - if (m_abs_path[m_abs_len - 1] == OS_PATH_SEPARATOR - && m_abs_path[m_abs_len - 2] != ':') { - m_abs_path[m_abs_len - 1] = '\0'; - } -#endif /* WIN32 */ - - bool ret = os_file_status(m_abs_path, &exists, &type); - -#ifdef _WIN32 - /* Put the separator back on. */ - if (m_abs_path[m_abs_len - 1] == '\0') { - m_abs_path[m_abs_len - 1] = OS_PATH_SEPARATOR; - } -#endif /* WIN32 */ - - return(ret && exists && type == OS_FILE_TYPE_DIR); -} /* Unit Tests */ #ifdef UNIV_ENABLE_UNIT_TEST_MAKE_FILEPATH @@ -7515,7 +6590,6 @@ test_make_filepath() path = MF("/this/is/a/path/with/a/filename", NULL, IBD, false); DISPLAY; path = MF("/this/is/a/path/with/a/filename", NULL, ISL, false); DISPLAY; path = MF("/this/is/a/path/with/a/filename", NULL, CFG, false); DISPLAY; - path = MF("/this/is/a/path/with/a/filename", NULL, CFP, false); DISPLAY; path = MF("/this/is/a/path/with/a/filename.ibd", NULL, IBD, false); DISPLAY; path = MF("/this/is/a/path/with/a/filename.ibd", NULL, IBD, false); DISPLAY; path = MF("/this/is/a/path/with/a/filename.dat", NULL, IBD, false); DISPLAY; @@ -7525,7 +6599,6 @@ test_make_filepath() path = MF(NULL, "dbname/tablespacename", IBD, false); DISPLAY; path = MF(NULL, "dbname/tablespacename", ISL, false); DISPLAY; path = MF(NULL, "dbname/tablespacename", CFG, false); DISPLAY; - path = MF(NULL, "dbname/tablespacename", CFP, false); DISPLAY; path = MF(NULL, "dbname\\tablespacename", NO_EXT, false); DISPLAY; path = MF(NULL, "dbname\\tablespacename", IBD, false); DISPLAY; path = MF("/this/is/a/path", "dbname/tablespacename", IBD, false); DISPLAY; @@ -7568,11 +6641,56 @@ fil_space_get_crypt_data( space = fil_space_get_by_id(id); + mutex_exit(&fil_system->mutex); + if (space != NULL) { + /* If we have not yet read the page0 + of this tablespace we will do it now. */ + if (!space->crypt_data && !space->page_0_crypt_read) { + ulint space_id = space->id; + fil_node_t* node; + + ut_a(space->crypt_data == NULL); + node = UT_LIST_GET_FIRST(space->chain); + + byte *buf = static_cast<byte*>(ut_malloc(2 * UNIV_PAGE_SIZE, PSI_INSTRUMENT_ME)); + byte *page = static_cast<byte*>(ut_align(buf, UNIV_PAGE_SIZE)); + fil_read(page_id_t(space_id, 0), univ_page_size, 0, univ_page_size.physical(), + page); + ulint offset = FSP_HEADER_OFFSET + + fsp_header_get_encryption_offset( + page_size_t(space->flags)); + space->crypt_data = fil_space_read_crypt_data(space_id, page, offset); + ut_free(buf); + + DBUG_LOG("crypt", + "Read page 0 from" + << " tablespace " << space_id + << " name " << space->name + << " key_id " << (space->crypt_data + ? space->crypt_data->key_id + : 0) + << " encryption " + << (space->crypt_data + ? space->crypt_data->encryption : 0) + << " handle " << node->handle); + + ut_a(space->id == space_id); + + space->page_0_crypt_read = true; + } + crypt_data = space->crypt_data; - } - mutex_exit(&fil_system->mutex); + if (!space->page_0_crypt_read) { + ib::warn() << "Space " << space->id << " name " + << space->name << " contains encryption " + << (space->crypt_data ? space->crypt_data->encryption : 0) + << " information for key_id " + << (space->crypt_data ? space->crypt_data->key_id : 0) + << " but page0 is not read."; + } + } return(crypt_data); } @@ -7777,7 +6895,7 @@ fil_node_t* fil_space_get_node( fil_space_t* space, /*!< in: file spage */ ulint space_id, /*!< in: space id */ - ulint* block_offset, /*!< in/out: offset in number of blocks */ + os_offset_t* block_offset, /*!< in/out: offset in number of blocks */ ulint byte_offset, /*!< in: remainder of offset in bytes; in aio this must be divisible by the OS block size */ @@ -7813,14 +6931,16 @@ fil_space_get_node( /********************************************************************//** Return block size of node in file space +@param[in] space_id space id +@param[in] block_offset page offset +@param[in] len page len @return file block size */ UNIV_INTERN ulint fil_space_get_block_size( -/*=====================*/ - ulint space_id, - ulint block_offset, - ulint len) + ulint space_id, + os_offset_t block_offset, + ulint len) { ulint block_size = 512; ut_ad(!mutex_own(&fil_system->mutex)); @@ -7886,3 +7006,26 @@ fil_system_exit(void) ut_ad(mutex_own(&fil_system->mutex)); mutex_exit(&fil_system->mutex); } + +/** +Get should we punch hole to tablespace. +@param[in] node File node +@return true, if punch hole should be tried, false if not. */ +bool +fil_node_should_punch_hole( + const fil_node_t* node) +{ + return (node->space->punch_hole); +} + +/** +Set punch hole to tablespace to given value. +@param[in] node File node +@param[in] val value to be set. */ +void +fil_space_set_punch_hole( + fil_node_t* node, + bool val) +{ + node->space->punch_hole = val; +} diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index 4479c06f1b2..18f95c94d75 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -48,14 +48,9 @@ Updated 14/02/2015 #include "trx0sys.h" #include "row0mysql.h" #include "ha_prototypes.h" // IB_LOG_ -#ifndef UNIV_HOTBACKUP -# include "buf0lru.h" -# include "ibuf0ibuf.h" -# include "sync0sync.h" -#else /* !UNIV_HOTBACKUP */ -# include "srv0srv.h" -static ulint srv_data_read, srv_data_written; -#endif /* !UNIV_HOTBACKUP */ +#include "buf0lru.h" +#include "ibuf0ibuf.h" +#include "sync0sync.h" #include "zlib.h" #ifdef __linux__ #include <linux/fs.h> @@ -167,8 +162,14 @@ fil_compress_page( switch(comp_method) { #ifdef HAVE_LZ4 case PAGE_LZ4_ALGORITHM: + +#ifdef HAVE_LZ4_COMPRESS_DEFAULT + err = LZ4_compress_default((const char *)buf, + (char *)out_buf+header_len, len, write_size); +#else err = LZ4_compress_limitedOutput((const char *)buf, (char *)out_buf+header_len, len, write_size); +#endif /* HAVE_LZ4_COMPRESS_DEFAULT */ write_size = err; if (err == 0) { @@ -360,6 +361,7 @@ fil_compress_page( if (allocated) { /* TODO: reduce number of memcpy's */ memcpy(buf, out_buf, len); + goto exit_free; } else { return(out_buf); } @@ -385,6 +387,7 @@ err_exit: srv_stats.pages_page_compression_error.inc(); *out_len = len; +exit_free: if (allocated) { ut_free(out_buf); #ifdef HAVE_LZO @@ -417,24 +420,26 @@ fil_decompress_page( { int err = 0; ulint actual_size = 0; - ulint compression_alg = 0; + ib_uint64_t compression_alg = 0; byte *in_buf; ulint ptype; - ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; + ulint header_len; ut_ad(buf); ut_ad(len); ptype = mach_read_from_2(buf+FIL_PAGE_TYPE); - if (ptype == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - header_len += FIL_PAGE_COMPRESSION_METHOD_SIZE; - } - - /* Do not try to uncompressed pages that are not compressed */ - if (ptype != FIL_PAGE_PAGE_COMPRESSED && - ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED && - ptype != FIL_PAGE_COMPRESSED) { + switch (ptype) { + case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED: + header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE + + FIL_PAGE_COMPRESSION_METHOD_SIZE; + break; + case FIL_PAGE_PAGE_COMPRESSED: + header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; + break; + default: + /* The page is not in our format. */ return; } @@ -448,9 +453,7 @@ fil_decompress_page( /* Before actual decompress, make sure that page type is correct */ - if (mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM) != BUF_NO_CHECKSUM_MAGIC || - (ptype != FIL_PAGE_PAGE_COMPRESSED && - ptype != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED)) { + if (mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM) != BUF_NO_CHECKSUM_MAGIC) { ib::error() << "Corruption: We try to uncompress corrupted page:" << " CRC " << mach_read_from_4(buf+FIL_PAGE_SPACE_OR_CHKSUM) @@ -466,7 +469,7 @@ fil_decompress_page( /* Get compression algorithm */ if (ptype == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { - compression_alg = mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE); + compression_alg = static_cast<ib_uint64_t>(mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE)); } else { compression_alg = mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); } @@ -492,10 +495,8 @@ fil_decompress_page( *write_size = actual_size; } -#ifdef UNIV_PAGECOMPRESS_DEBUG - ib::info() << "Preparing for decompress for len " - << actual_size << "."; -#endif /* UNIV_PAGECOMPRESS_DEBUG */ + DBUG_LOG("compress", "Preparing for decompress for len " + << actual_size << "."); switch(compression_alg) { case PAGE_ZLIB_ALGORITHM: diff --git a/storage/innobase/fsp/fsp0file.cc b/storage/innobase/fsp/fsp0file.cc index 501a8e58622..74c153a65d5 100644 --- a/storage/innobase/fsp/fsp0file.cc +++ b/storage/innobase/fsp/fsp0file.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2013, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -33,9 +34,6 @@ Created 2013-7-26 by Kevin Lewis #include "srv0start.h" #include "ut0new.h" #include "fil0crypt.h" -#ifdef UNIV_HOTBACKUP -#include "my_sys.h" -#endif /* UNIV_HOTBACKUP */ /** Initialize the name, size and order of this datafile @param[in] name tablespace name, will be copied @@ -50,8 +48,6 @@ Datafile::init( m_name = mem_strdup(name); m_flags = flags; - m_encryption_key = NULL; - m_encryption_iv = NULL; } /** Release the resources. */ @@ -63,22 +59,11 @@ Datafile::shutdown() ut_free(m_name); m_name = NULL; - free_filepath(); - - if (m_encryption_key != NULL) { - ut_free(m_encryption_key); - m_encryption_key = NULL; - } - - if (m_crypt_info) { - fil_space_destroy_crypt_data(&m_crypt_info); - } - - if (m_encryption_iv != NULL) { - ut_free(m_encryption_iv); - m_encryption_iv = NULL; - } + /* The fil_space_t::crypt_data was freed in + fil_space_free_low(). Invalidate our redundant pointer. */ + m_crypt_info = NULL; + free_filepath(); free_first_page(); } @@ -284,9 +269,8 @@ Datafile::same_as( } /** Allocate and set the datafile or tablespace name in m_name. -If a name is provided, use it; else if the datafile is file-per-table, -extract a file-per-table tablespace name from m_filepath; else it is a -general tablespace, so just call it that for now. The value of m_name +If a name is provided, use it; else extract a file-per-table +tablespace name from m_filepath. The value of m_name will be freed in the destructor. @param[in] name tablespace name if known, NULL if not */ void @@ -296,14 +280,8 @@ Datafile::set_name(const char* name) if (name != NULL) { m_name = mem_strdup(name); - } else if (fsp_is_file_per_table(m_space_id, m_flags)) { - m_name = fil_path_to_space_name(m_filepath); } else { - /* Give this general tablespace a temporary name. */ - m_name = static_cast<char*>( - ut_malloc_nokey(strlen(general_space_name) + 20)); - - sprintf(m_name, "%s_" ULINTPF, general_space_name, m_space_id); + m_name = fil_path_to_space_name(m_filepath); } } @@ -366,16 +344,37 @@ Datafile::read_first_page(bool read_only_mode) } } - if (err == DB_SUCCESS && m_order == 0) { + if (err != DB_SUCCESS) { + return(err); + } + if (m_order == 0) { + m_space_id = fsp_header_get_space_id(m_first_page); m_flags = fsp_header_get_flags(m_first_page); + if (!fsp_flags_is_valid(m_flags)) { + ulint cflags = fsp_flags_convert_from_101(m_flags); + if (cflags == ULINT_UNDEFINED) { + ib::error() + << "Invalid flags " << ib::hex(m_flags) + << " in " << m_filepath; + return(DB_CORRUPTION); + } else { + m_flags = cflags; + } + } + } - m_space_id = fsp_header_get_space_id(m_first_page); + const page_size_t ps(m_flags); + if (ps.physical() > page_size) { + ib::error() << "File " << m_filepath + << " should be longer than " + << page_size << " bytes"; + return(DB_CORRUPTION); } - const page_size_t page_sz = fsp_header_get_page_size(m_first_page); - ulint offset = fsp_header_get_crypt_offset(page_sz, NULL); - m_crypt_info = fil_space_read_crypt_data(m_space_id, m_first_page, offset); + m_crypt_info = fil_space_read_crypt_data( + m_space_id, m_first_page, + FSP_HEADER_OFFSET + fsp_header_get_encryption_offset(ps)); return(err); } @@ -396,14 +395,10 @@ space ID and flags. The file should exist and be successfully opened in order for this function to validate it. @param[in] space_id The expected tablespace ID. @param[in] flags The expected tablespace flags. -@param[in] for_import if it is for importing @retval DB_SUCCESS if tablespace is valid, DB_ERROR if not. m_is_valid is also set true on success, else false. */ dberr_t -Datafile::validate_to_dd( - ulint space_id, - ulint flags, - bool for_import) +Datafile::validate_to_dd(ulint space_id, ulint flags) { dberr_t err; @@ -414,18 +409,17 @@ Datafile::validate_to_dd( /* Validate this single-table-tablespace with the data dictionary, but do not compare the DATA_DIR flag, in case the tablespace was remotely located. */ - err = validate_first_page(0, for_import); + err = validate_first_page(0); if (err != DB_SUCCESS) { return(err); } + flags &= ~FSP_FLAGS_MEM_MASK; + /* Make sure the datafile we found matched the space ID. If the datafile is a file-per-table tablespace then also match the row format and zip page size. */ - if (m_space_id == space_id - && (m_flags & FSP_FLAGS_MASK_SHARED - || (m_flags & ~FSP_FLAGS_MASK_DATA_DIR) - == (flags & ~FSP_FLAGS_MASK_DATA_DIR))) { + if (m_space_id == space_id && m_flags == flags) { /* Datafile matches the tablespace expected. */ return(DB_SUCCESS); } @@ -434,9 +428,10 @@ Datafile::validate_to_dd( m_is_valid = false; ib::error() << "In file '" << m_filepath << "', tablespace id and" - " flags are " << m_space_id << " and " << m_flags << ", but in" - " the InnoDB data dictionary they are " << space_id << " and " - << flags << ". Have you moved InnoDB .ibd files around without" + " flags are " << m_space_id << " and " << ib::hex(m_flags) + << ", but in the InnoDB data dictionary they are " + << space_id << " and " << ib::hex(flags) + << ". Have you moved InnoDB .ibd files around without" " using the commands DISCARD TABLESPACE and IMPORT TABLESPACE?" " " << TROUBLESHOOT_DATADICT_MSG; @@ -458,63 +453,22 @@ Datafile::validate_for_recovery() ut_ad(is_open()); ut_ad(!srv_read_only_mode); - err = validate_first_page(0, false); + err = validate_first_page(0); switch (err) { case DB_SUCCESS: case DB_TABLESPACE_EXISTS: -#ifdef UNIV_HOTBACKUP - err = restore_from_doublewrite(0); - if (err != DB_SUCCESS) { - return(err); - } - /* Free the previously read first page and then re-validate. */ - free_first_page(); - err = validate_first_page(0, false); - if (err == DB_SUCCESS) { - std::string filepath = fil_space_get_first_path( - m_space_id); - if (is_intermediate_file(filepath.c_str())) { - /* Existing intermediate file with same space - id is obsolete.*/ - if (fil_space_free(m_space_id, FALSE)) { - err = DB_SUCCESS; - } - } else { - filepath.assign(m_filepath); - if (is_intermediate_file(filepath.c_str())) { - /* New intermediate file with same space id - shall be ignored.*/ - err = DB_TABLESPACE_EXISTS; - /* Set all bits of 'flags' as a special - indicator for "ignore tablespace". Hopefully - InnoDB will never use all bits or at least all - bits set will not be a meaningful setting - otherwise.*/ - m_flags = ~0; - } - } - } -#endif /* UNIV_HOTBACKUP */ break; default: - /* For encryption tablespace, we skip the retry step, - since it is only because the keyring is not ready. */ - if (FSP_FLAGS_GET_ENCRYPTION(m_flags)) { - return(err); - } /* Re-open the file in read-write mode Attempt to restore page 0 from doublewrite and read the space ID from a survey of the first few pages. */ close(); err = open_read_write(srv_read_only_mode); if (err != DB_SUCCESS) { - ib::error() << "Datafile '" << m_filepath << "' could not" - " be opened in read-write mode so that the" - " doublewrite pages could be restored."; return(err); - }; + } err = find_space_id(); if (err != DB_SUCCESS || m_space_id == 0) { @@ -524,14 +478,13 @@ Datafile::validate_for_recovery() return(err); } - err = restore_from_doublewrite(0); - if (err != DB_SUCCESS) { - return(err); + if (restore_from_doublewrite()) { + return(DB_CORRUPTION); } /* Free the previously read first page and then re-validate. */ free_first_page(); - err = validate_first_page(0, false); + err = validate_first_page(0); } if (err == DB_SUCCESS) { @@ -546,14 +499,11 @@ tablespace is opened. This occurs before the fil_space_t is created so the Space ID found here must not already be open. m_is_valid is set true on success, else false. @param[out] flush_lsn contents of FIL_PAGE_FILE_FLUSH_LSN -@param[in] for_import if it is for importing -(only valid for the first file of the system tablespace) @retval DB_SUCCESS on if the datafile is valid @retval DB_CORRUPTION if the datafile is not readable @retval DB_TABLESPACE_EXISTS if there is a duplicate space_id */ dberr_t -Datafile::validate_first_page(lsn_t* flush_lsn, - bool for_import) +Datafile::validate_first_page(lsn_t* flush_lsn) { char* prev_name; char* prev_filepath; @@ -611,7 +561,9 @@ Datafile::validate_first_page(lsn_t* flush_lsn, free_first_page(); return(DB_ERROR); - + } else if (!fsp_flags_is_valid(m_flags)) { + /* Tablespace flags must be valid. */ + error_txt = "Tablespace flags are invalid"; } else if (page_get_page_no(m_first_page) != 0) { /* First page must be number 0 */ @@ -642,51 +594,6 @@ Datafile::validate_first_page(lsn_t* flush_lsn, } -#ifdef MYSQL_ENCRYPTION - /* For encrypted tablespace, check the encryption info in the - first page can be decrypt by master key, otherwise, this table - can't be open. And for importing, we skip checking it. */ - if (FSP_FLAGS_GET_ENCRYPTION(m_flags) && !for_import) { - m_encryption_key = static_cast<byte*>( - ut_zalloc_nokey(ENCRYPTION_KEY_LEN)); - m_encryption_iv = static_cast<byte*>( - ut_zalloc_nokey(ENCRYPTION_KEY_LEN)); -#ifdef UNIV_ENCRYPT_DEBUG - fprintf(stderr, "Got from file %lu:", m_space_id); -#endif - if (!fsp_header_get_encryption_key(m_flags, - m_encryption_key, - m_encryption_iv, - m_first_page)) { - ib::error() - << "Encryption information in" - << " datafile: " << m_filepath - << " can't be decrypted" - << " , please confirm the keyfile" - << " is match and keyring plugin" - << " is loaded."; - - m_is_valid = false; - free_first_page(); - ut_free(m_encryption_key); - ut_free(m_encryption_iv); - m_encryption_key = NULL; - m_encryption_iv = NULL; - return(DB_CORRUPTION); - } - - if (recv_recovery_is_on() - && memcmp(m_encryption_key, - m_encryption_iv, - ENCRYPTION_KEY_LEN) == 0) { - ut_free(m_encryption_key); - ut_free(m_encryption_iv); - m_encryption_key = NULL; - m_encryption_iv = NULL; - } - } -#endif /* MYSQL_ENCRYPTION */ - if (fil_space_read_name_and_filepath( m_space_id, &prev_name, &prev_filepath)) { @@ -780,30 +687,7 @@ Datafile::find_space_id() err = os_file_read( request, m_handle, page, n_bytes, page_size); - if (err == DB_IO_DECOMPRESS_FAIL) { - - /* If the page was compressed on the fly then - try and decompress the page */ - - n_bytes = os_file_compressed_page_size(page); - - if (n_bytes != ULINT_UNDEFINED) { - - err = os_file_read( - request, - m_handle, page, page_size, - UNIV_PAGE_SIZE_MAX); - - if (err != DB_SUCCESS) { - - ib::info() - << "READ FAIL: " - << "page_no:" << j; - continue; - } - } - - } else if (err != DB_SUCCESS) { + if (err != DB_SUCCESS) { ib::info() << "READ FAIL: page_no:" << j; @@ -894,17 +778,15 @@ Datafile::find_space_id() } -/** Finds a given page of the given space id from the double write buffer -and copies it to the corresponding .ibd file. -@param[in] page_no Page number to restore -@return DB_SUCCESS if page was restored from doublewrite, else DB_ERROR */ -dberr_t -Datafile::restore_from_doublewrite( - ulint restore_page_no) +/** Restore the first page of the tablespace from +the double write buffer. +@return whether the operation failed */ +bool +Datafile::restore_from_doublewrite() { /* Find if double write buffer contains page_no of given space id. */ - const byte* page = recv_sys->dblwr.find_page( - m_space_id, restore_page_no); + const byte* page = recv_sys->dblwr.find_page(m_space_id, 0); + const page_id_t page_id(m_space_id, 0); if (page == NULL) { /* If the first page of the given user tablespace is not there @@ -912,23 +794,34 @@ Datafile::restore_from_doublewrite( now. Hence this is treated as an error. */ ib::error() - << "Corrupted page " - << page_id_t(m_space_id, restore_page_no) + << "Corrupted page " << page_id << " of datafile '" << m_filepath << "' could not be found in the doublewrite buffer."; - return(DB_CORRUPTION); + return(true); } - const ulint flags = mach_read_from_4( + ulint flags = mach_read_from_4( FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + if (!fsp_flags_is_valid(flags)) { + ulint cflags = fsp_flags_convert_from_101(flags); + if (cflags == ULINT_UNDEFINED) { + ib::warn() + << "Ignoring a doublewrite copy of page " + << page_id + << " due to invalid flags " << ib::hex(flags); + return(true); + } + flags = cflags; + /* The flags on the page should be converted later. */ + } + const page_size_t page_size(flags); - ut_a(page_get_page_no(page) == restore_page_no); + ut_a(page_get_page_no(page) == page_id.page_no()); - ib::info() << "Restoring page " - << page_id_t(m_space_id, restore_page_no) + ib::info() << "Restoring page " << page_id << " of datafile '" << m_filepath << "' from the doublewrite buffer. Writing " << page_size.physical() << " bytes into file '" @@ -936,14 +829,10 @@ Datafile::restore_from_doublewrite( IORequest request(IORequest::WRITE); - /* Note: The pages are written out as uncompressed because we don't - have the compression algorithm information at this point. */ - - request.disable_compression(); - return(os_file_write( request, - m_filepath, m_handle, page, 0, page_size.physical())); + m_filepath, m_handle, page, 0, page_size.physical()) + != DB_SUCCESS); } /** Create a link filename based on the contents of m_name, @@ -1026,19 +915,7 @@ the path provided without its suffix, plus DOT_ISL. void RemoteDatafile::set_link_filepath(const char* path) { - if (m_link_filepath != NULL) { - return; - } - - if (path != NULL && FSP_FLAGS_GET_SHARED(flags())) { - /* Make the link_filepath based on the basename. */ - ut_ad(strcmp(&path[strlen(path) - strlen(DOT_IBD)], - DOT_IBD) == 0); - - m_link_filepath = fil_make_filepath(NULL, base_name(path), - ISL, false); - } else { - /* Make the link_filepath based on the m_name. */ + if (m_link_filepath == NULL) { m_link_filepath = fil_make_filepath(NULL, name(), ISL, false); } } @@ -1048,14 +925,11 @@ under the 'datadir' of MySQL. The datadir is the directory of a running mysqld program. We can refer to it by simply using the path ".". @param[in] name tablespace name @param[in] filepath remote filepath of tablespace datafile -@param[in] is_shared true for general tablespace, - false for file-per-table @return DB_SUCCESS or error code */ dberr_t RemoteDatafile::create_link_file( const char* name, - const char* filepath, - bool is_shared) + const char* filepath) { bool success; dberr_t err = DB_SUCCESS; @@ -1065,31 +939,8 @@ RemoteDatafile::create_link_file( ut_ad(!srv_read_only_mode); ut_ad(0 == strcmp(&filepath[strlen(filepath) - 4], DOT_IBD)); - if (is_shared) { - /* The default location for a shared tablespace is the - datadir. We previously made sure that this filepath is - not under the datadir. If it is in the datadir there - is no need for a link file. */ + link_filepath = fil_make_filepath(NULL, name, ISL, false); - size_t len = dirname_length(filepath); - if (len == 0) { - /* File is in the datadir. */ - return(DB_SUCCESS); - } - - Folder folder(filepath, len); - - if (folder_mysql_datadir == folder) { - /* File is in the datadir. */ - return(DB_SUCCESS); - } - - /* Use the file basename to build the ISL filepath. */ - link_filepath = fil_make_filepath(NULL, base_name(filepath), - ISL, false); - } else { - link_filepath = fil_make_filepath(NULL, name, ISL, false); - } if (link_filepath == NULL) { return(DB_ERROR); } @@ -1196,8 +1047,6 @@ RemoteDatafile::delete_link_file( It is always created under the datadir of MySQL. For file-per-table tablespaces, the isl file is expected to be in a 'database' directory and called 'tablename.isl'. -For general tablespaces, there will be no 'database' directory. -The 'basename.isl' will be in the datadir. The caller must free the memory returned if it is not null. @param[in] link_filepath filepath of the ISL file @return Filepath of the IBD file read from the ISL file */ @@ -1205,16 +1054,12 @@ char* RemoteDatafile::read_link_file( const char* link_filepath) { - char* filepath = NULL; - FILE* file = NULL; - - file = fopen(link_filepath, "r+b"); + FILE* file = fopen(link_filepath, "r+b"); if (file == NULL) { return(NULL); } - filepath = static_cast<char*>( - ut_malloc_nokey(OS_FILE_MAX_PATH)); + char* filepath = static_cast<char*>(ut_malloc_nokey(OS_FILE_MAX_PATH)); os_file_read_string(file, filepath, OS_FILE_MAX_PATH); fclose(file); diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 9dc99f3f09d..6886c5164f7 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -31,9 +32,6 @@ Created 11/29/1995 Heikki Tuuri #include "fsp0fsp.ic" #endif -#ifdef UNIV_HOTBACKUP -# include "fut0lst.h" -#else /* UNIV_HOTBACKUP */ #include "buf0buf.h" #include "fil0fil.h" #include "fil0crypt.h" @@ -55,14 +53,19 @@ Created 11/29/1995 Heikki Tuuri // JAN: MySQL 5.7 Encryption // #include <my_aes.h> -/** Returns an extent to the free list of a space. -@param[in] page_id page id in the extent +typedef ulint page_no_t; + +/** Return an extent to the free list of a space. +@param[in,out] space tablespace +@param[in] offset page number in the extent @param[in] page_size page size @param[in,out] mtr mini-transaction */ +MY_ATTRIBUTE((nonnull)) static void fsp_free_extent( - const page_id_t& page_id, + fil_space_t* space, + page_no_t offset, const page_size_t& page_size, mtr_t* mtr); @@ -74,7 +77,7 @@ void fseg_mark_page_used( /*================*/ fseg_inode_t* seg_inode,/*!< in: segment inode */ - ulint page, /*!< in: page offset */ + page_no_t page, /*!< in: page offset */ xdes_t* descr, /*!< in: extent descriptor */ mtr_t* mtr); /*!< in/out: mini-transaction */ @@ -82,15 +85,16 @@ fseg_mark_page_used( We think of the extent lists of the segment catenated in the order FSEG_FULL -> FSEG_NOT_FULL -> FSEG_FREE. @param[in] inode segment inode -@param[in] space_id space id +@param[in] space tablespace @param[in] page_size page size @param[in,out] mtr mini-transaction @return the first extent descriptor, or NULL if none */ +MY_ATTRIBUTE((nonnull, warn_unused_result)) static xdes_t* fseg_get_first_extent( fseg_inode_t* inode, - ulint space_id, + const fil_space_t* space, const page_size_t& page_size, mtr_t* mtr); @@ -150,204 +154,33 @@ fseg_alloc_free_page_low( MY_ATTRIBUTE((warn_unused_result)); /** Gets a pointer to the space header and x-locks its page. -@param[in] id space id +@param[in] space tablespace @param[in] page_size page size @param[in,out] mtr mini-transaction @return pointer to the space header, page x-locked */ UNIV_INLINE fsp_header_t* fsp_get_space_header( - ulint id, + const fil_space_t* space, const page_size_t& page_size, mtr_t* mtr) { buf_block_t* block; fsp_header_t* header; - ut_ad(id != 0 || !page_size.is_compressed()); + ut_ad(space->purpose != FIL_TYPE_LOG); + ut_ad(!FSP_FLAGS_GET_ZIP_SSIZE(space->flags) + == !page_size.is_compressed()); - block = buf_page_get(page_id_t(id, 0), page_size, RW_SX_LATCH, mtr); + block = buf_page_get(page_id_t(space->id, 0), page_size, + RW_SX_LATCH, mtr); header = FSP_HEADER_OFFSET + buf_block_get_frame(block); buf_block_dbg_add_level(block, SYNC_FSP_PAGE); - ut_ad(id == mach_read_from_4(FSP_SPACE_ID + header)); -#ifdef UNIV_DEBUG - const ulint flags = mach_read_from_4(FSP_SPACE_FLAGS + header); - ut_ad(page_size_t(flags).equals_to(page_size)); -#endif /* UNIV_DEBUG */ + ut_ad(space->id == mach_read_from_4(FSP_SPACE_ID + header)); return(header); } -/** Convert a 32 bit integer tablespace flags to the 32 bit table flags. -This can only be done for a tablespace that was built as a file-per-table -tablespace. Note that the fsp_flags cannot show the difference between a -Compact and Redundant table, so an extra Compact boolean must be supplied. - Low order bit - | REDUNDANT | COMPACT | COMPRESSED | DYNAMIC -fil_space_t::flags | 0 | 0 | 1 | 1 -dict_table_t::flags | 0 | 1 | 1 | 1 -@param[in] fsp_flags fil_space_t::flags -@param[in] compact true if not Redundant row format -@return tablespace flags (fil_space_t::flags) */ -ulint -fsp_flags_to_dict_tf( - ulint fsp_flags, - bool compact) -{ - /* If the table in this file-per-table tablespace is Compact - row format, the low order bit will not indicate Compact. */ - bool post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(fsp_flags); - ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(fsp_flags); - bool atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(fsp_flags); - bool data_dir = FSP_FLAGS_HAS_DATA_DIR(fsp_flags); - bool shared_space = FSP_FLAGS_GET_SHARED(fsp_flags); - bool page_compressed = FSP_FLAGS_GET_PAGE_COMPRESSION(fsp_flags); - ulint comp_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(fsp_flags); - bool atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(fsp_flags); - - /* FSP_FLAGS_GET_TEMPORARY(fsp_flags) does not have an equivalent - flag position in the table flags. But it would go into flags2 if - any code is created where that is needed. */ - - ulint flags = dict_tf_init(post_antelope | compact, zip_ssize, - atomic_blobs, data_dir, shared_space, - page_compressed, comp_level, atomic_writes); - - return(flags); -} -#endif /* !UNIV_HOTBACKUP */ - -/** Validate the tablespace flags. -These flags are stored in the tablespace header at offset FSP_SPACE_FLAGS. -They should be 0 for ROW_FORMAT=COMPACT and ROW_FORMAT=REDUNDANT. -The newer row formats, COMPRESSED and DYNAMIC, use a file format > Antelope -so they should have a file format number plus the DICT_TF_COMPACT bit set. -@param[in] flags Tablespace flags -@return true if valid, false if not */ -bool -fsp_flags_is_valid( - ulint flags) -{ - bool post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(flags); - ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); - bool atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(flags); - ulint page_ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); - bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags); - bool is_shared = FSP_FLAGS_GET_SHARED(flags); - bool is_temp = FSP_FLAGS_GET_TEMPORARY(flags); - bool is_encryption = FSP_FLAGS_GET_ENCRYPTION(flags); - ulint unused = FSP_FLAGS_GET_UNUSED(flags); - bool page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(flags); - ulint page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags); - ulint atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(flags); - - const char *file; - ulint line; - -#define GOTO_ERROR file = __FILE__; line = __LINE__; goto err_exit; - - DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", return(false);); - - /* The Antelope row formats REDUNDANT and COMPACT did - not use tablespace flags, so the entire 4-byte field - is zero for Antelope row formats. */ - if (flags == 0) { - return(true); - } - - /* Barracuda row formats COMPRESSED and DYNAMIC use a feature called - ATOMIC_BLOBS which builds on the page structure introduced for the - COMPACT row format by allowing long fields to be broken into prefix - and externally stored parts. So if it is Post_antelope, it uses - Atomic BLOBs. */ - if (post_antelope != atomic_blobs) { - GOTO_ERROR; - return(false); - } - - /* Make sure there are no bits that we do not know about. */ - if (unused != 0) { - GOTO_ERROR; - } - - /* The zip ssize can be zero if it is other than compressed row format, - or it could be from 1 to the max. */ - if (zip_ssize > PAGE_ZIP_SSIZE_MAX) { - GOTO_ERROR; - } - - /* The actual page size must be within 4k and 16K (3 =< ssize =< 5). */ - if (page_ssize != 0 - && (page_ssize < UNIV_PAGE_SSIZE_MIN - || page_ssize > UNIV_PAGE_SSIZE_MAX)) { - GOTO_ERROR; - } - - /* Only single-table tablespaces use the DATA DIRECTORY clause. - It is not compatible with the TABLESPACE clause. Nor is it - compatible with the TEMPORARY clause. */ - if (has_data_dir && (is_shared || is_temp)) { - GOTO_ERROR; - return(false); - } - - /* Only single-table and not temp tablespaces use the encryption - clause. */ - if (is_encryption && (is_shared || is_temp)) { - GOTO_ERROR; - } - - /* Page compression level requires page compression and atomic blobs - to be set */ - if (page_compression_level || page_compression) { - if (!page_compression || !atomic_blobs) { - GOTO_ERROR; - } - } - - if (atomic_writes > ATOMIC_WRITES_OFF) { - GOTO_ERROR; - return (false); - } - -#if UNIV_FORMAT_MAX != UNIV_FORMAT_B -# error UNIV_FORMAT_MAX != UNIV_FORMAT_B, Add more validations. -#endif -#if FSP_FLAGS_POS_UNUSED != 13 - //# error You have added a new FSP_FLAG without adding a validation check. -#endif - - return(true); - -err_exit: - ib::error() << "Tablespace flags: " << flags << " corrupted " - << " in file: " << file << " line: " << line - << " post_antelope: " << post_antelope - << " atomic_blobs: " << atomic_blobs - << " unused: " << unused - << " zip_ssize: " << zip_ssize << " max: " << PAGE_ZIP_SSIZE_MAX - << " page_ssize: " << page_ssize - << " " << UNIV_PAGE_SSIZE_MIN << ":" << UNIV_PAGE_SSIZE_MAX - << " has_data_dir: " << has_data_dir - << " is_shared: " << is_shared - << " is_temp: " << is_temp - << " is_encryption: " << is_encryption - << " page_compressed: " << page_compression - << " page_compression_level: " << page_compression_level - << " atomic_writes: " << atomic_writes; - return (false); -} - -/** Check if tablespace is system temporary. -@param[in] space_id tablespace ID -@return true if tablespace is system temporary. */ -bool -fsp_is_system_temporary( - ulint space_id) -{ - return(space_id == srv_tmp_space.space_id()); -} - /** Check if checksum is disabled for the given space. @param[in] space_id tablespace ID @return true if checksum is disabled for given space. */ @@ -358,22 +191,7 @@ fsp_is_checksum_disabled( return(fsp_is_system_temporary(space_id)); } -/** Check if tablespace is file-per-table. -@param[in] space_id tablespace ID -@param[in] fsp_flags tablespace flags -@return true if tablespace is file-per-table. */ -bool -fsp_is_file_per_table( - ulint space_id, - ulint fsp_flags) -{ - return(!is_system_tablespace(space_id) - && !fsp_is_shared_tablespace(fsp_flags)); -} - -#ifndef UNIV_HOTBACKUP #ifdef UNIV_DEBUG - /** Skip some of the sanity checks that are time consuming even in debug mode and can affect frequent verification runs that are done to ensure stability of the product. @@ -385,7 +203,6 @@ fsp_skip_sanity_check( return(srv_skip_temp_table_checks_debug && fsp_is_system_temporary(space_id)); } - #endif /* UNIV_DEBUG */ /**********************************************************************//** @@ -601,7 +418,7 @@ xdes_init( /** Get pointer to a the extent descriptor of a page. @param[in,out] sp_header tablespace header page, x-latched -@param[in] space tablespace identifier +@param[in] space tablespace @param[in] offset page offset @param[in,out] mtr mini-transaction @param[in] init_space whether the tablespace is being initialized @@ -612,43 +429,36 @@ exist in the space or if the offset exceeds free limit */ UNIV_INLINE MY_ATTRIBUTE((warn_unused_result)) xdes_t* xdes_get_descriptor_with_space_hdr( - fsp_header_t* sp_header, - ulint space, - ulint offset, - mtr_t* mtr, - bool init_space = false, - buf_block_t** desc_block = NULL) + fsp_header_t* sp_header, + const fil_space_t* space, + page_no_t offset, + mtr_t* mtr, + bool init_space = false, + buf_block_t** desc_block = NULL) { ulint limit; ulint size; ulint descr_page_no; - ulint flags; page_t* descr_page; -#ifdef UNIV_DEBUG - const fil_space_t* fspace = fil_space_get(space); - ut_ad(fspace != NULL); -#endif /* UNIV_DEBUG */ - ut_ad(mtr_memo_contains(mtr, &fspace->latch, MTR_MEMO_X_LOCK)); + ut_ad(mtr_memo_contains(mtr, &space->latch, MTR_MEMO_X_LOCK)); ut_ad(mtr_memo_contains_page(mtr, sp_header, MTR_MEMO_PAGE_SX_FIX)); ut_ad(page_offset(sp_header) == FSP_HEADER_OFFSET); /* Read free limit and space size */ limit = mach_read_from_4(sp_header + FSP_FREE_LIMIT); size = mach_read_from_4(sp_header + FSP_SIZE); - flags = mach_read_from_4(sp_header + FSP_SPACE_FLAGS); - ut_ad(limit == fspace->free_limit - || (fspace->free_limit == 0 + ut_ad(limit == space->free_limit + || (space->free_limit == 0 && (init_space - || fspace->purpose == FIL_TYPE_TEMPORARY + || space->purpose == FIL_TYPE_TEMPORARY || (srv_startup_is_before_trx_rollback_phase - && fspace->id <= srv_undo_tablespaces)))); - ut_ad(size == fspace->size_in_header); - ut_ad(flags == fspace->flags); + && space->id <= srv_undo_tablespaces)))); + ut_ad(size == space->size_in_header); if ((offset >= size) || (offset >= limit)) { return(NULL); } - const page_size_t page_size(flags); + const page_size_t page_size(space->flags); descr_page_no = xdes_calc_descriptor_page(page_size, offset); @@ -661,7 +471,7 @@ xdes_get_descriptor_with_space_hdr( block = NULL; } else { block = buf_page_get( - page_id_t(space, descr_page_no), page_size, + page_id_t(space->id, descr_page_no), page_size, RW_SX_LATCH, mtr); buf_block_dbg_add_level(block, SYNC_FSP_PAGE); @@ -677,63 +487,60 @@ xdes_get_descriptor_with_space_hdr( + XDES_SIZE * xdes_calc_descriptor_index(page_size, offset)); } -/** Gets pointer to a the extent descriptor of a page. -The page where the extent descriptor resides is x-locked. If the page offset -is equal to the free limit of the space, adds new extents from above the free -limit to the space free list, if not free limit == space size. This adding -is necessary to make the descriptor defined, as they are uninitialized -above the free limit. -@param[in] space_id space id +/** Get the extent descriptor of a page. +The page where the extent descriptor resides is x-locked. If the page +offset is equal to the free limit of the space, we will add new +extents from above the free limit to the space free list, if not free +limit == space size. This adding is necessary to make the descriptor +defined, as they are uninitialized above the free limit. +@param[in] space tablespace @param[in] offset page offset; if equal to the free limit, we try to add new extents to the space free list @param[in] page_size page size @param[in,out] mtr mini-transaction -@return pointer to the extent descriptor, NULL if the page does not -exist in the space or if the offset exceeds the free limit */ +@return the extent descriptor */ +static xdes_t* xdes_get_descriptor( - ulint space_id, - ulint offset, + const fil_space_t* space, + page_no_t offset, const page_size_t& page_size, mtr_t* mtr) { buf_block_t* block; fsp_header_t* sp_header; - block = buf_page_get(page_id_t(space_id, 0), page_size, + block = buf_page_get(page_id_t(space->id, 0), page_size, RW_SX_LATCH, mtr); buf_block_dbg_add_level(block, SYNC_FSP_PAGE); sp_header = FSP_HEADER_OFFSET + buf_block_get_frame(block); - return(xdes_get_descriptor_with_space_hdr(sp_header, space_id, offset, - mtr)); + return(xdes_get_descriptor_with_space_hdr( + sp_header, space, offset, mtr)); } -/********************************************************************//** -Gets pointer to a the extent descriptor if the file address -of the descriptor list node is known. The page where the +/** Get a pointer to the extent descriptor. The page where the extent descriptor resides is x-locked. +@param[in] space tablespace +@param[in] page_size page size +@param[in] lst_node file address of the list node + contained in the descriptor +@param[in,out] mtr mini-transaction @return pointer to the extent descriptor */ +MY_ATTRIBUTE((nonnull, warn_unused_result)) UNIV_INLINE xdes_t* xdes_lst_get_descriptor( -/*====================*/ - ulint space, /*!< in: space id */ + const fil_space_t* space, const page_size_t& page_size, - fil_addr_t lst_node,/*!< in: file address of the list node - contained in the descriptor */ - mtr_t* mtr) /*!< in/out: mini-transaction */ + fil_addr_t lst_node, + mtr_t* mtr) { - xdes_t* descr; - - ut_ad(mtr); - ut_ad(mtr_memo_contains(mtr, fil_space_get_latch(space, NULL), - MTR_MEMO_X_LOCK)); - descr = fut_get_ptr(space, page_size, lst_node, RW_SX_LATCH, mtr) - - XDES_FLST_NODE; - - return(descr); + ut_ad(mtr_memo_contains(mtr, &space->latch, MTR_MEMO_X_LOCK)); + ut_ad(page_size.equals_to(page_size_t(space->flags))); + return(fut_get_ptr(space->id, page_size, lst_node, RW_SX_LATCH, mtr) + - XDES_FLST_NODE); } /********************************************************************//** @@ -751,7 +558,6 @@ xdes_get_offset( + ((page_offset(descr) - XDES_ARR_OFFSET) / XDES_SIZE) * FSP_EXTENT_SIZE); } -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Inits a file page whose prior contents should be ignored. */ @@ -781,8 +587,7 @@ fsp_init_file_page_low( } } -#ifndef UNIV_HOTBACKUP -# ifdef UNIV_DEBUG +#ifdef UNIV_DEBUG /** Assert that the mini-transaction is compatible with updating an allocation bitmap page. @param[in] id tablespace identifier @@ -790,59 +595,54 @@ updating an allocation bitmap page. static void fsp_space_modify_check( - ulint id, - const mtr_t* mtr) + const fil_space_t* space, + const mtr_t* mtr) { switch (mtr->get_log_mode()) { case MTR_LOG_SHORT_INSERTS: case MTR_LOG_NONE: /* These modes are only allowed within a non-bitmap page when there is a higher-level redo log record written. */ + ut_ad(space->purpose == FIL_TYPE_TABLESPACE + || space->purpose == FIL_TYPE_TEMPORARY); break; case MTR_LOG_NO_REDO: -#ifdef UNIV_DEBUG - { - const fil_type_t type = fil_space_get_type(id); - ut_a(id == srv_tmp_space.space_id() - || srv_is_tablespace_truncated(id) - || fil_space_is_being_truncated(id) - || fil_space_get_flags(id) == ULINT_UNDEFINED - || type == FIL_TYPE_TEMPORARY - || type == FIL_TYPE_IMPORT - || fil_space_is_redo_skipped(id)); - } -#endif /* UNIV_DEBUG */ + ut_ad(space->purpose == FIL_TYPE_TEMPORARY + || space->purpose == FIL_TYPE_IMPORT + || space->redo_skipped_count + || space->is_being_truncated + || srv_is_tablespace_truncated(space->id)); return; case MTR_LOG_ALL: - /* We must not write redo log for the shared temporary - tablespace. */ - ut_ad(id != srv_tmp_space.space_id()); - /* If we write redo log, the tablespace must exist. */ - ut_ad(fil_space_get_type(id) == FIL_TYPE_TABLESPACE); - ut_ad(mtr->is_named_space(id)); + /* We may only write redo log for a persistent tablespace. */ + ut_ad(space->purpose == FIL_TYPE_TABLESPACE); + ut_ad(mtr->is_named_space(space->id)); return; } ut_ad(0); } -# endif /* UNIV_DEBUG */ +#endif /* UNIV_DEBUG */ /** Initialize a file page. +@param[in] space tablespace @param[in,out] block file page @param[in,out] mtr mini-transaction */ +MY_ATTRIBUTE((nonnull)) static void fsp_init_file_page( - buf_block_t* block, - mtr_t* mtr) + const fil_space_t* space MY_ATTRIBUTE((unused)), + buf_block_t* block, + mtr_t* mtr) { + ut_d(fsp_space_modify_check(space, mtr)); + ut_ad(space->id == block->page.id.space()); fsp_init_file_page_low(block); - ut_d(fsp_space_modify_check(block->page.id.space(), mtr)); mlog_write_initial_log_record(buf_block_get_frame(block), MLOG_INIT_FILE_PAGE2, mtr); } -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses a redo log record of a file page init. @@ -895,6 +695,7 @@ fsp_header_init_fields( ulint space_id, /*!< in: space id */ ulint flags) /*!< in: tablespace flags (FSP_SPACE_FLAGS) */ { + flags &= ~FSP_FLAGS_MEM_MASK; ut_a(fsp_flags_is_valid(flags)); mach_write_to_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page, @@ -903,203 +704,6 @@ fsp_header_init_fields( flags); } -#ifndef UNIV_HOTBACKUP -/** Get the offset of encrytion information in page 0. -@param[in] page_size page size. -@return offset on success, otherwise 0. */ -static -ulint -fsp_header_get_encryption_offset( - const page_size_t& page_size) -{ - ulint offset; -#ifdef UNIV_DEBUG - ulint left_size; -#endif - - offset = XDES_ARR_OFFSET + XDES_SIZE * xdes_arr_size(page_size); -#ifdef UNIV_DEBUG - left_size = page_size.physical() - FSP_HEADER_OFFSET - offset - - FIL_PAGE_DATA_END; - - ut_ad(left_size >= ENCRYPTION_INFO_SIZE_V2); -#endif - - return offset; -} - -#if 0 /* MySQL 5.7 Encryption */ -/** Fill the encryption info. -@param[in] space tablespace -@param[in,out] encrypt_info buffer for encrypt key. -@return true if success. */ -bool -fsp_header_fill_encryption_info( - fil_space_t* space, - byte* encrypt_info) -{ - byte* ptr; - lint elen; - ulint master_key_id; - byte* master_key; - byte key_info[ENCRYPTION_KEY_LEN * 2]; - ulint crc; - Encryption::Version version; -#ifdef UNIV_ENCRYPT_DEBUG - const byte* data; - ulint i; -#endif - - /* Get master key from key ring */ - Encryption::get_master_key(&master_key_id, &master_key, &version); - if (master_key == NULL) { - return(false); - } - - memset(encrypt_info, 0, ENCRYPTION_INFO_SIZE_V2); - memset(key_info, 0, ENCRYPTION_KEY_LEN * 2); - - /* Use the new master key to encrypt the tablespace - key. */ - ut_ad(encrypt_info != NULL); - ptr = encrypt_info; - - /* Write magic header. */ - if (version == Encryption::ENCRYPTION_VERSION_1) { - memcpy(ptr, ENCRYPTION_KEY_MAGIC_V1, ENCRYPTION_MAGIC_SIZE); - } else { - memcpy(ptr, ENCRYPTION_KEY_MAGIC_V2, ENCRYPTION_MAGIC_SIZE); - } - ptr += ENCRYPTION_MAGIC_SIZE; - - /* Write master key id. */ - mach_write_to_4(ptr, master_key_id); - ptr += sizeof(ulint); - - /* Write server uuid. */ - if (version == Encryption::ENCRYPTION_VERSION_2) { - memcpy(ptr, Encryption::uuid, ENCRYPTION_SERVER_UUID_LEN); - ptr += ENCRYPTION_SERVER_UUID_LEN; - } - - /* Write tablespace key to temp space. */ - memcpy(key_info, - space->encryption_key, - ENCRYPTION_KEY_LEN); - - /* Write tablespace iv to temp space. */ - memcpy(key_info + ENCRYPTION_KEY_LEN, - space->encryption_iv, - ENCRYPTION_KEY_LEN); - -#ifdef UNIV_ENCRYPT_DEBUG - fprintf(stderr, "Set %lu:%lu ",space->id, - Encryption::master_key_id); - for (data = (const byte*) master_key, i = 0; - i < ENCRYPTION_KEY_LEN; i++) - fprintf(stderr, "%02lx", (ulong)*data++); - fprintf(stderr, " "); - for (data = (const byte*) space->encryption_key, - i = 0; i < ENCRYPTION_KEY_LEN; i++) - fprintf(stderr, "%02lx", (ulong)*data++); - fprintf(stderr, " "); - for (data = (const byte*) space->encryption_iv, - i = 0; i < ENCRYPTION_KEY_LEN; i++) - fprintf(stderr, "%02lx", (ulong)*data++); - fprintf(stderr, "\n"); -#endif - /* Encrypt tablespace key and iv. */ - elen = my_aes_encrypt( - key_info, - ENCRYPTION_KEY_LEN * 2, - ptr, - master_key, - ENCRYPTION_KEY_LEN, - my_aes_256_ecb, - NULL, false); - - if (elen == MY_AES_BAD_DATA) { - my_free(master_key); - return(false); - } - - ptr += ENCRYPTION_KEY_LEN * 2; - - /* Write checksum bytes. */ - crc = ut_crc32(key_info, ENCRYPTION_KEY_LEN * 2); - mach_write_to_4(ptr, crc); - - my_free(master_key); - return(true); -} -#endif /* ! */ - -/** Rotate the encryption info in the space header. -@param[in] space tablespace -@param[in] encrypt_info buffer for re-encrypt key. -@param[in,out] mtr mini-transaction -@return true if success. */ -bool -fsp_header_rotate_encryption( - fil_space_t* space, - byte* encrypt_info, - mtr_t* mtr) -{ - buf_block_t* block; - ulint offset; - - ut_ad(mtr); - - const page_size_t page_size(space->flags); - -#if MYSQL_ENCRYPTION - page_t* page; - ulint master_key_id; - ut_ad(space->encryption_type != Encryption::NONE); - /* Fill encryption info. */ - if (!fsp_header_fill_encryption_info(space, - encrypt_info)) { - return(false); - } -#endif - - /* Save the encryption info to the page 0. */ - block = buf_page_get(page_id_t(space->id, 0), - page_size, - RW_SX_LATCH, mtr); - buf_block_dbg_add_level(block, SYNC_FSP_PAGE); - ut_ad(space->id == page_get_space_id(buf_block_get_frame(block))); - - offset = fsp_header_get_encryption_offset(page_size); - ut_ad(offset != 0 && offset < UNIV_PAGE_SIZE); - - -#if MYSQL_ENCRYPTION - page = buf_block_get_frame(block); - /* If is in recovering, skip all master key id is rotated - tablespaces. */ - master_key_id = mach_read_from_4( - page + offset + ENCRYPTION_MAGIC_SIZE); - if (recv_recovery_is_on() - && master_key_id == Encryption::master_key_id) { - ut_ad(memcmp(page + offset, - ENCRYPTION_KEY_MAGIC_V1, - ENCRYPTION_MAGIC_SIZE) == 0 - || memcmp(page + offset, - ENCRYPTION_KEY_MAGIC_V2, - ENCRYPTION_MAGIC_SIZE) == 0); - return(true); - } - - mlog_write_string(page + offset, - encrypt_info, - ENCRYPTION_INFO_SIZE_V2, - mtr); -#endif /* MYSQL_ENCRYPTION */ - - return(true); -} - /** Initializes the space header of a new created space and creates also the insert buffer tree root if space == 0. @param[in] space_id space id @@ -1133,7 +737,7 @@ fsp_header_init( /* The prior contents of the file page should be ignored */ - fsp_init_file_page(block, mtr); + fsp_init_file_page(space, block, mtr); page = buf_block_get_frame(block); mlog_write_ulint(page + FIL_PAGE_TYPE, FIL_PAGE_TYPE_FSP_HDR, @@ -1146,7 +750,8 @@ fsp_header_init( mlog_write_ulint(header + FSP_SIZE, size, MLOG_4BYTES, mtr); mlog_write_ulint(header + FSP_FREE_LIMIT, 0, MLOG_4BYTES, mtr); - mlog_write_ulint(header + FSP_SPACE_FLAGS, space->flags, + mlog_write_ulint(header + FSP_SPACE_FLAGS, + space->flags & ~FSP_FLAGS_MEM_MASK, MLOG_4BYTES, mtr); mlog_write_ulint(header + FSP_FRAG_N_USED, 0, MLOG_4BYTES, mtr); @@ -1161,31 +766,6 @@ fsp_header_init( fsp_fill_free_list(!is_system_tablespace(space_id), space, header, mtr); -#if 0 /* MySQL 5.7 Encryption */ - /* For encryption tablespace, we need to save the encryption - info to the page 0. */ - if (FSP_FLAGS_GET_ENCRYPTION(space->flags)) { - ulint offset = fsp_header_get_encryption_offset(page_size); - byte encryption_info[ENCRYPTION_INFO_SIZE_V2]; - - if (offset == 0) - return(false); - - if (!fsp_header_fill_encryption_info(space, - encryption_info)) { - space->encryption_type = Encryption::NONE; - memset(space->encryption_key, 0, ENCRYPTION_KEY_LEN); - memset(space->encryption_iv, 0, ENCRYPTION_KEY_LEN); - return(false); - } - - mlog_write_string(page + offset, - encryption_info, - ENCRYPTION_INFO_SIZE_V2, - mtr); - } -#endif /* ! */ - if (space_id == srv_sys_space.space_id()) { if (btr_create(DICT_CLUSTERED | DICT_UNIVERSAL | DICT_IBUF, 0, univ_page_size, DICT_IBUF_ID_MIN + space_id, @@ -1194,13 +774,14 @@ fsp_header_init( } } - ulint maxsize = 0; - ulint offset = fsp_header_get_crypt_offset(page_size, &maxsize); - fil_space_write_crypt_data(space_id, page, offset, maxsize, mtr); + ulint offset = FSP_HEADER_OFFSET + + fsp_header_get_encryption_offset(page_size); + fil_space_write_crypt_data(space_id, page, offset, + page_size.logical() + - offset - FIL_PAGE_DATA_END, mtr); return(true); } -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Reads the space id from the first page of a tablespace. @@ -1229,175 +810,6 @@ fsp_header_get_space_id( return(id); } -/** Reads the page size from the first page of a tablespace. -@param[in] page first page of a tablespace -@return page size */ -page_size_t -fsp_header_get_page_size( - const page_t* page) -{ - return(page_size_t(fsp_header_get_flags(page))); -} - -#if 0 /* MySQL 5.7 Encryption */ -/** Decoding the encryption info -from the first page of a tablespace. -@param[in/out] key key -@param[in/out] iv iv -@param[in] encryption_info encrytion info. -@return true if success */ -bool -fsp_header_decode_encryption_info( - byte* key, - byte* iv, - byte* encryption_info) -{ - byte* ptr; - ulint master_key_id; - byte* master_key = NULL; - lint elen; - byte key_info[ENCRYPTION_KEY_LEN * 2]; - ulint crc1; - ulint crc2; - char srv_uuid[ENCRYPTION_SERVER_UUID_LEN + 1]; - Encryption::Version version; -#ifdef UNIV_ENCRYPT_DEBUG - const byte* data; - ulint i; -#endif - - ptr = encryption_info; - - /* For compatibility with 5.7.11, we need to handle the - encryption information which created in this old version. */ - if (memcmp(ptr, ENCRYPTION_KEY_MAGIC_V1, - ENCRYPTION_MAGIC_SIZE) == 0) { - version = Encryption::ENCRYPTION_VERSION_1; - } else { - version = Encryption::ENCRYPTION_VERSION_2; - } - /* Check magic. */ - if (version == Encryption::ENCRYPTION_VERSION_2 - && memcmp(ptr, ENCRYPTION_KEY_MAGIC_V2, ENCRYPTION_MAGIC_SIZE) != 0) { - /* We ignore report error for recovery, - since the encryption info maybe hasn't writen - into datafile when the table is newly created. */ - if (!recv_recovery_is_on()) { - return(false); - } else { - return(true); - } - } - ptr += ENCRYPTION_MAGIC_SIZE; - - /* Get master key id. */ - master_key_id = mach_read_from_4(ptr); - ptr += sizeof(ulint); - - /* Get server uuid. */ - if (version == Encryption::ENCRYPTION_VERSION_2) { - memset(srv_uuid, 0, ENCRYPTION_SERVER_UUID_LEN + 1); - memcpy(srv_uuid, ptr, ENCRYPTION_SERVER_UUID_LEN); - ptr += ENCRYPTION_SERVER_UUID_LEN; - } - - /* Get master key by key id. */ - memset(key_info, 0, ENCRYPTION_KEY_LEN * 2); - if (version == Encryption::ENCRYPTION_VERSION_1) { - Encryption::get_master_key(master_key_id, NULL, &master_key); - } else { - Encryption::get_master_key(master_key_id, srv_uuid, &master_key); - } - if (master_key == NULL) { - return(false); - } - -#ifdef UNIV_ENCRYPT_DEBUG - fprintf(stderr, "%lu ", master_key_id); - for (data = (const byte*) master_key, i = 0; - i < ENCRYPTION_KEY_LEN; i++) - fprintf(stderr, "%02lx", (ulong)*data++); -#endif - - /* Decrypt tablespace key and iv. */ - elen = my_aes_decrypt( - ptr, - ENCRYPTION_KEY_LEN * 2, - key_info, - master_key, - ENCRYPTION_KEY_LEN, - my_aes_256_ecb, NULL, false); - - if (elen == MY_AES_BAD_DATA) { - my_free(master_key); - return(NULL); - } - - /* Check checksum bytes. */ - ptr += ENCRYPTION_KEY_LEN * 2; - - crc1 = mach_read_from_4(ptr); - crc2 = ut_crc32(key_info, ENCRYPTION_KEY_LEN * 2); - if (crc1 != crc2) { - ib::error() << "Failed to decrpt encryption information," - << " please check key file is not changed!"; - return(false); - } - - /* Get tablespace key */ - memcpy(key, key_info, ENCRYPTION_KEY_LEN); - - /* Get tablespace iv */ - memcpy(iv, key_info + ENCRYPTION_KEY_LEN, - ENCRYPTION_KEY_LEN); - -#ifdef UNIV_ENCRYPT_DEBUG - fprintf(stderr, " "); - for (data = (const byte*) key, - i = 0; i < ENCRYPTION_KEY_LEN; i++) - fprintf(stderr, "%02lx", (ulong)*data++); - fprintf(stderr, " "); - for (data = (const byte*) iv, - i = 0; i < ENCRYPTION_KEY_LEN; i++) - fprintf(stderr, "%02lx", (ulong)*data++); - fprintf(stderr, "\n"); -#endif - - my_free(master_key); - - if (Encryption::master_key_id < master_key_id) { - Encryption::master_key_id = master_key_id; - memcpy(Encryption::uuid, srv_uuid, ENCRYPTION_SERVER_UUID_LEN); - } - - return(true); -} - -/** Reads the encryption key from the first page of a tablespace. -@param[in] fsp_flags tablespace flags -@param[in/out] key tablespace key -@param[in/out] iv tablespace iv -@param[in] page first page of a tablespace -@return true if success */ -bool -fsp_header_get_encryption_key( - ulint fsp_flags, - byte* key, - byte* iv, - page_t* page) -{ - ulint offset; - const page_size_t page_size(fsp_flags); - offset = fsp_header_get_encryption_offset(page_size); - if (offset == 0) { - return(false); - } - - return(fsp_header_decode_encryption_info(key, iv, page + offset)); -} -#endif /* ! */ - -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Increases the space size field of a space. */ void @@ -1413,10 +825,10 @@ fsp_header_inc_size( ut_ad(mtr); fil_space_t* space = mtr_x_lock_space(space_id, mtr); - ut_d(fsp_space_modify_check(space_id, mtr)); + ut_d(fsp_space_modify_check(space, mtr)); header = fsp_get_space_header( - space_id, page_size_t(space->flags), mtr); + space, page_size_t(space->flags), mtr); size = mach_read_from_4(header + FSP_SIZE); ut_ad(size == space->size_in_header); @@ -1443,12 +855,9 @@ fsp_header_get_tablespace_size(void) mtr_start(&mtr); -#ifdef UNIV_DEBUG - fil_space_t* space = -#endif /* UNIV_DEBUG */ - mtr_x_lock_space(TRX_SYS_SPACE, &mtr); + fil_space_t* space = mtr_x_lock_space(TRX_SYS_SPACE, &mtr); - header = fsp_get_space_header(TRX_SYS_SPACE, univ_page_size, &mtr); + header = fsp_get_space_header(space, univ_page_size, &mtr); size = mach_read_from_4(header + FSP_SIZE); ut_ad(space->size_in_header == size); @@ -1477,7 +886,7 @@ fsp_try_extend_data_file_with_pages( ulint size; ut_a(!is_system_tablespace(space->id)); - ut_d(fsp_space_modify_check(space->id, mtr)); + ut_d(fsp_space_modify_check(space, mtr)); size = mach_read_from_4(header + FSP_SIZE); ut_ad(size == space->size_in_header); @@ -1511,7 +920,7 @@ fsp_try_extend_data_file( "ran out of space. Please add another file or use" " 'autoextend' for the last file in setting"; - ut_d(fsp_space_modify_check(space->id, mtr)); + ut_d(fsp_space_modify_check(space, mtr)); if (space->id == srv_sys_space.space_id() && !srv_sys_space.can_auto_extend_last_file()) { @@ -1521,8 +930,8 @@ fsp_try_extend_data_file( to reset the flag to false as dealing with this error requires server restart. */ if (!srv_sys_space.get_tablespace_full_status()) { - ib::error() << "Tablespace " << srv_sys_space.name() - << " " << OUT_OF_SPACE_MSG + ib::error() << "The InnoDB system tablespace " + << OUT_OF_SPACE_MSG << " innodb_data_file_path."; srv_sys_space.set_tablespace_full_status(true); } @@ -1535,8 +944,8 @@ fsp_try_extend_data_file( to reset the flag to false as dealing with this error requires server restart. */ if (!srv_tmp_space.get_tablespace_full_status()) { - ib::error() << "Tablespace " << srv_tmp_space.name() - << " " << OUT_OF_SPACE_MSG + ib::error() << "The InnoDB temporary tablespace " + << OUT_OF_SPACE_MSG << " innodb_temp_data_file_path."; srv_tmp_space.set_tablespace_full_status(true); } @@ -1549,15 +958,14 @@ fsp_try_extend_data_file( const page_size_t page_size( mach_read_from_4(header + FSP_SPACE_FLAGS)); - if (space->id == srv_sys_space.space_id()) { - + switch (space->id) { + case TRX_SYS_SPACE: size_increase = srv_sys_space.get_increment(); - - } else if (space->id == srv_tmp_space.space_id()) { - + break; + case SRV_TMP_SPACE_ID: size_increase = srv_tmp_space.get_increment(); - - } else { + break; + default: ulint extent_pages = fsp_get_extent_size_in_pages(page_size); if (size < extent_pages) { @@ -1597,7 +1005,7 @@ fsp_try_extend_data_file( } /** Calculate the number of pages to extend a datafile. -We extend single-table and general tablespaces first one extent at a time, +We extend single-table tablespaces first one extent at a time, but 4 at a time for bigger tablespaces. It is not enough to extend always by one extent, because we need to add at least one extent to FSP_FREE. A single extent descriptor page will track many extents. And the extent @@ -1656,7 +1064,6 @@ fsp_fill_free_list( { ulint limit; ulint size; - ulint flags; xdes_t* descr; ulint count = 0; ulint frag_n_used; @@ -1665,25 +1072,29 @@ fsp_fill_free_list( ut_ad(header != NULL); ut_ad(mtr != NULL); ut_ad(page_offset(header) == FSP_HEADER_OFFSET); - ut_d(fsp_space_modify_check(space->id, mtr)); + ut_d(fsp_space_modify_check(space, mtr)); /* Check if we can fill free list from above the free list limit */ size = mach_read_from_4(header + FSP_SIZE); limit = mach_read_from_4(header + FSP_FREE_LIMIT); - flags = mach_read_from_4(header + FSP_SPACE_FLAGS); ut_ad(size == space->size_in_header); ut_ad(limit == space->free_limit); - ut_ad(flags == space->flags); - const page_size_t page_size(flags); + const page_size_t page_size(space->flags); if (size < limit + FSP_EXTENT_SIZE * FSP_FREE_ADD) { - if ((!init_space && !is_system_tablespace(space->id)) - || (space->id == srv_sys_space.space_id() - && srv_sys_space.can_auto_extend_last_file()) - || (space->id == srv_tmp_space.space_id() - && srv_tmp_space.can_auto_extend_last_file())) { + bool skip_resize = init_space; + switch (space->id) { + case TRX_SYS_SPACE: + skip_resize = !srv_sys_space.can_auto_extend_last_file(); + break; + case SRV_TMP_SPACE_ID: + skip_resize = srv_tmp_space.can_auto_extend_last_file(); + break; + } + + if (!skip_resize) { ulint n_pages = 0; fsp_try_extend_data_file(space, header, mtr, &n_pages); size = space->size_in_header; @@ -1721,7 +1132,7 @@ fsp_fill_free_list( buf_block_dbg_add_level(block, SYNC_FSP_PAGE); - fsp_init_file_page(block, mtr); + fsp_init_file_page(space, block, mtr); mlog_write_ulint(buf_block_get_frame(block) + FIL_PAGE_TYPE, FIL_PAGE_TYPE_XDES, @@ -1733,7 +1144,7 @@ fsp_fill_free_list( order, and we must be able to release its latch. Note: Insert-Buffering is disabled for tables that reside in the temp-tablespace. */ - if (space->id != srv_tmp_space.space_id()) { + if (space->purpose != FIL_TYPE_TEMPORARY) { mtr_t ibuf_mtr; mtr_start(&ibuf_mtr); @@ -1741,9 +1152,7 @@ fsp_fill_free_list( /* Avoid logging while truncate table fix-up is active. */ - if (space->purpose == FIL_TYPE_TEMPORARY - || srv_is_tablespace_truncated( - space->id)) { + if (srv_is_tablespace_truncated(space->id)) { mtr_set_log_mode( &ibuf_mtr, MTR_LOG_NO_REDO); } @@ -1761,7 +1170,7 @@ fsp_fill_free_list( buf_block_dbg_add_level(block, SYNC_FSP_PAGE); - fsp_init_file_page(block, &ibuf_mtr); + fsp_init_file_page(space, block, &ibuf_mtr); ibuf_bitmap_page_init(block, &ibuf_mtr); @@ -1771,7 +1180,7 @@ fsp_fill_free_list( buf_block_t* desc_block = NULL; descr = xdes_get_descriptor_with_space_hdr( - header, space->id, i, mtr, init_space, &desc_block); + header, space, i, mtr, init_space, &desc_block); if (desc_block != NULL) { fil_block_check_type( desc_block, FIL_PAGE_TYPE_XDES, mtr); @@ -1808,7 +1217,7 @@ fsp_fill_free_list( } /** Allocates a new free extent. -@param[in] space_id tablespace identifier +@param[in,out] space tablespace @param[in] page_size page size @param[in] hint hint of which extent would be desirable: any page offset in the extent goes; the hint must not be > FSP_FREE_LIMIT @@ -1817,7 +1226,7 @@ page offset in the extent goes; the hint must not be > FSP_FREE_LIMIT static xdes_t* fsp_alloc_free_extent( - ulint space_id, + fil_space_t* space, const page_size_t& page_size, ulint hint, mtr_t* mtr) @@ -1827,13 +1236,10 @@ fsp_alloc_free_extent( xdes_t* descr; buf_block_t* desc_block = NULL; - header = fsp_get_space_header(space_id, page_size, mtr); + header = fsp_get_space_header(space, page_size, mtr); descr = xdes_get_descriptor_with_space_hdr( - header, space_id, hint, mtr, false, &desc_block); - - fil_space_t* space = fil_space_get(space_id); - ut_a(space != NULL); + header, space, hint, mtr, false, &desc_block); if (desc_block != NULL) { fil_block_check_type(desc_block, FIL_PAGE_TYPE_XDES, mtr); @@ -1857,7 +1263,7 @@ fsp_alloc_free_extent( } descr = xdes_lst_get_descriptor( - space_id, page_size, first, mtr); + space, page_size, first, mtr); } flst_remove(header + FSP_FREE, descr + XDES_FLST_NODE, mtr); @@ -1906,7 +1312,8 @@ fsp_alloc_from_free_frag( NOTE: If init_mtr != mtr, the block will only be initialized if it was not previously x-latched. It is assumed that the block has been x-latched only by mtr, and freed in mtr in that case. -@param[in] page_id page id of the allocated page +@param[in,out] space tablespace +@param[in] offset page number of the allocated page @param[in] page_size page size of the allocated page @param[in] rw_latch RW_SX_LATCH, RW_X_LATCH @param[in,out] mtr mini-transaction of the allocation @@ -1916,19 +1323,21 @@ or rw_lock_x_lock_count(&block->lock) == 1 */ static buf_block_t* fsp_page_create( - const page_id_t& page_id, + fil_space_t* space, + page_no_t offset, const page_size_t& page_size, rw_lock_type_t rw_latch, mtr_t* mtr, mtr_t* init_mtr) { - buf_block_t* block = buf_page_create(page_id, page_size, init_mtr); + ut_ad(page_size.equals_to(page_size_t(space->flags))); - ut_ad(mtr_memo_contains(mtr, block, MTR_MEMO_PAGE_X_FIX) - == rw_lock_own(&block->lock, RW_LOCK_X)); + buf_block_t* block = buf_page_create(page_id_t(space->id, offset), + page_size, init_mtr); - ut_ad(mtr_memo_contains(mtr, block, MTR_MEMO_PAGE_SX_FIX) - == rw_lock_own(&block->lock, RW_LOCK_SX)); + ut_d(bool latched = mtr_memo_contains_flagged(mtr, block, + MTR_MEMO_PAGE_X_FIX + | MTR_MEMO_PAGE_SX_FIX)); ut_ad(rw_latch == RW_X_LATCH || rw_latch == RW_SX_LATCH); @@ -1954,12 +1363,8 @@ fsp_page_create( /* Initialize the page, unless it was already SX-latched in mtr. (In this case, we would want to allocate another page that has not been freed in mtr.) */ - ut_ad(init_mtr == mtr - || !mtr_memo_contains_flagged(mtr, block, - MTR_MEMO_PAGE_X_FIX - | MTR_MEMO_PAGE_SX_FIX)); - - fsp_init_file_page(block, init_mtr); + ut_ad(init_mtr == mtr || !latched); + fsp_init_file_page(space, block, init_mtr); } return(block); @@ -1967,7 +1372,7 @@ fsp_page_create( /** Allocates a single free page from a space. The page is marked as used. -@param[in] space space id +@param[in,out] space tablespace @param[in] page_size page size @param[in] hint hint of which page would be desirable @param[in] rw_latch RW_SX_LATCH, RW_X_LATCH @@ -1981,7 +1386,7 @@ initialized (may be the same as mtr) static MY_ATTRIBUTE((warn_unused_result)) buf_block_t* fsp_alloc_free_page( - ulint space, + fil_space_t* space, const page_size_t& page_size, ulint hint, rw_lock_type_t rw_latch, @@ -1992,8 +1397,7 @@ fsp_alloc_free_page( fil_addr_t first; xdes_t* descr; ulint free; - ulint page_no; - ulint space_size; + const ulint space_id = space->id; ut_ad(mtr); ut_ad(init_mtr); @@ -2052,18 +1456,18 @@ fsp_alloc_free_page( ut_error; } - page_no = xdes_get_offset(descr) + free; + page_no_t page_no = xdes_get_offset(descr) + free; - space_size = mach_read_from_4(header + FSP_SIZE); - ut_ad(space_size == fil_space_get(space)->size_in_header - || (space == TRX_SYS_SPACE + page_no_t space_size = mach_read_from_4(header + FSP_SIZE); + ut_ad(space_size == space->size_in_header + || (space_id == TRX_SYS_SPACE && srv_startup_is_before_trx_rollback_phase)); if (space_size <= page_no) { /* It must be that we are extending a single-table tablespace whose size is still < 64 pages */ - ut_a(!is_system_tablespace(space)); + ut_a(!is_system_tablespace(space_id)); if (page_no >= FSP_EXTENT_SIZE) { ib::error() << "Trying to extend a single-table" " tablespace " << space << " , by single" @@ -2072,9 +1476,7 @@ fsp_alloc_free_page( return(NULL); } - fil_space_t* fspace = fil_space_get(space); - - if (!fsp_try_extend_data_file_with_pages(fspace, page_no, + if (!fsp_try_extend_data_file_with_pages(space, page_no, header, mtr)) { /* No disk space left */ return(NULL); @@ -2082,19 +1484,21 @@ fsp_alloc_free_page( } fsp_alloc_from_free_frag(header, descr, free, mtr); - return(fsp_page_create(page_id_t(space, page_no), page_size, - rw_latch, mtr, init_mtr)); + return(fsp_page_create(space, page_no, page_size, rw_latch, + mtr, init_mtr)); } /** Frees a single page of a space. The page is marked as free and clean. +@param[in,out] space tablespace @param[in] page_id page id @param[in] page_size page size @param[in,out] mtr mini-transaction */ static void fsp_free_page( - const page_id_t& page_id, + fil_space_t* space, + ulint offset, const page_size_t& page_size, mtr_t* mtr) { @@ -2104,24 +1508,22 @@ fsp_free_page( ulint frag_n_used; ut_ad(mtr); - ut_d(fsp_space_modify_check(page_id.space(), mtr)); + ut_d(fsp_space_modify_check(space, mtr)); /* fprintf(stderr, "Freeing page %lu in space %lu\n", page, space); */ - header = fsp_get_space_header( - page_id.space(), page_size, mtr); + header = fsp_get_space_header(space, page_size, mtr); descr = xdes_get_descriptor_with_space_hdr( - header, page_id.space(), page_id.page_no(), mtr); + header, space, offset, mtr); state = xdes_get_state(descr, mtr); - if (state != XDES_FREE_FRAG && state != XDES_FULL_FRAG) { + if (UNIV_UNLIKELY(state != XDES_FREE_FRAG + && state != XDES_FULL_FRAG)) { ib::error() << "File space extent descriptor of page " - << page_id << " has state " << state; - fputs("InnoDB: Dump of descriptor: ", stderr); - ut_print_buf(stderr, ((byte*) descr) - 50, 200); - putc('\n', stderr); + << page_id_t(space->id, offset) + << " has state " << state; /* Crash in debug version, so that we get a core dump of this corruption. */ ut_ad(0); @@ -2137,12 +1539,11 @@ fsp_free_page( } if (xdes_mtr_get_bit(descr, XDES_FREE_BIT, - page_id.page_no() % FSP_EXTENT_SIZE, mtr)) { + offset % FSP_EXTENT_SIZE, mtr)) { ib::error() << "File space extent descriptor of page " - << page_id << " says it is free. Dump of descriptor: "; - ut_print_buf(stderr, ((byte*) descr) - 50, 200); - putc('\n', stderr); + << page_id_t(space->id, offset) + << " says it is free."; /* Crash in debug version, so that we get a core dump of this corruption. */ ut_ad(0); @@ -2153,7 +1554,7 @@ fsp_free_page( return; } - const ulint bit = page_id.page_no() % FSP_EXTENT_SIZE; + const ulint bit = offset % FSP_EXTENT_SIZE; xdes_set_bit(descr, XDES_FREE_BIT, bit, TRUE, mtr); xdes_set_bit(descr, XDES_CLEAN_BIT, bit, TRUE, mtr); @@ -2180,37 +1581,39 @@ fsp_free_page( /* The extent has become free: move it to another list */ flst_remove(header + FSP_FREE_FRAG, descr + XDES_FLST_NODE, mtr); - fsp_free_extent(page_id, page_size, mtr); + fsp_free_extent(space, offset, page_size, mtr); } } -/** Returns an extent to the free list of a space. -@param[in] page_id page id in the extent +/** Return an extent to the free list of a space. +@param[in,out] space tablespace +@param[in] offset page number in the extent @param[in] page_size page size @param[in,out] mtr mini-transaction */ static void fsp_free_extent( - const page_id_t& page_id, + fil_space_t* space, + page_no_t offset, const page_size_t& page_size, mtr_t* mtr) { fsp_header_t* header; xdes_t* descr; - ut_ad(mtr); + ut_ad(mtr_memo_contains(mtr, &space->latch, MTR_MEMO_X_LOCK)); - header = fsp_get_space_header(page_id.space(), page_size, mtr); + header = fsp_get_space_header(space, page_size, mtr); descr = xdes_get_descriptor_with_space_hdr( - header, page_id.space(), page_id.page_no(), mtr); + header, space, offset, mtr); ut_a(xdes_get_state(descr, mtr) != XDES_FREE); xdes_init(descr, mtr); flst_add_last(header + FSP_FREE, descr + XDES_FLST_NODE, mtr); - fil_space_get(page_id.space())->free_len++; + space->free_len++; } /** Returns the nth inode slot on an inode page. @@ -2298,33 +1701,34 @@ fsp_seg_inode_page_find_free( return(ULINT_UNDEFINED); } -/**********************************************************************//** -Allocates a new file segment inode page. -@return TRUE if could be allocated */ +/** Allocate a file segment inode page. +@param[in,out] space tablespace +@param[in,out] space_header tablespace header +@param[in,out] mtr mini-transaction +@return whether the allocation succeeded */ +MY_ATTRIBUTE((nonnull, warn_unused_result)) static -ibool +bool fsp_alloc_seg_inode_page( -/*=====================*/ - fsp_header_t* space_header, /*!< in: space header */ - mtr_t* mtr) /*!< in/out: mini-transaction */ + fil_space_t* space, + fsp_header_t* space_header, + mtr_t* mtr) { fseg_inode_t* inode; buf_block_t* block; page_t* page; - ulint space; ut_ad(page_offset(space_header) == FSP_HEADER_OFFSET); + ut_ad(page_get_space_id(page_align(space_header)) == space->id); - space = page_get_space_id(page_align(space_header)); - - const page_size_t page_size(mach_read_from_4(FSP_SPACE_FLAGS - + space_header)); + const page_size_t page_size(space->flags); - block = fsp_alloc_free_page(space, page_size, 0, RW_SX_LATCH, mtr, mtr); + block = fsp_alloc_free_page( + space, page_size, 0, RW_SX_LATCH, mtr, mtr); if (block == NULL) { - return(FALSE); + return(false); } buf_block_dbg_add_level(block, SYNC_FSP_PAGE); @@ -2347,18 +1751,22 @@ fsp_alloc_seg_inode_page( space_header + FSP_SEG_INODES_FREE, page + FSEG_INODE_PAGE_NODE, mtr); - return(TRUE); + return(true); } -/**********************************************************************//** -Allocates a new file segment inode. -@return segment inode, or NULL if not enough space */ +/** Allocate a file segment inode. +@param[in,out] space tablespace +@param[in,out] space_header tablespace header +@param[in,out] mtr mini-transaction +@return segment inode +@retval NULL if not enough space */ +MY_ATTRIBUTE((nonnull, warn_unused_result)) static fseg_inode_t* fsp_alloc_seg_inode( -/*================*/ - fsp_header_t* space_header, /*!< in: space header */ - mtr_t* mtr) /*!< in/out: mini-transaction */ + fil_space_t* space, + fsp_header_t* space_header, + mtr_t* mtr) { buf_block_t* block; page_t* page; @@ -2369,14 +1777,12 @@ fsp_alloc_seg_inode( /* Allocate a new segment inode page if needed. */ if (flst_get_len(space_header + FSP_SEG_INODES_FREE) == 0 - && !fsp_alloc_seg_inode_page(space_header, mtr)) { + && !fsp_alloc_seg_inode_page(space, space_header, mtr)) { return(NULL); } - const page_size_t page_size( - mach_read_from_4(FSP_SPACE_FLAGS + space_header)); - + const page_size_t page_size(space->flags); const page_id_t page_id( - page_get_space_id(page_align(space_header)), + space->id, flst_get_first(space_header + FSP_SEG_INODES_FREE, mtr).page); block = buf_page_get(page_id, page_size, RW_SX_LATCH, mtr); @@ -2409,14 +1815,14 @@ fsp_alloc_seg_inode( } /** Frees a file segment inode. -@param[in] space space id +@param[in,out] space tablespace @param[in] page_size page size @param[in,out] inode segment inode @param[in,out] mtr mini-transaction */ static void fsp_free_seg_inode( - ulint space, + fil_space_t* space, const page_size_t& page_size, fseg_inode_t* inode, mtr_t* mtr) @@ -2455,8 +1861,7 @@ fsp_free_seg_inode( flst_remove(space_header + FSP_SEG_INODES_FREE, page + FSEG_INODE_PAGE_NODE, mtr); - fsp_free_page(page_id_t(space, page_get_page_no(page)), - page_size, mtr); + fsp_free_page(space, page_get_page_no(page), page_size, mtr); } } @@ -2673,10 +2078,10 @@ fseg_create_general( ut_ad(mtr); ut_ad(byte_offset + FSEG_HEADER_SIZE <= UNIV_PAGE_SIZE - FIL_PAGE_DATA_END); - ut_d(fsp_space_modify_check(space_id, mtr)); fil_space_t* space = mtr_x_lock_space(space_id, mtr); const page_size_t page_size(space->flags); + ut_d(fsp_space_modify_check(space, mtr)); if (page != 0) { block = buf_page_get(page_id_t(space_id, page), page_size, @@ -2707,9 +2112,9 @@ fseg_create_general( DBUG_RETURN(NULL); } - space_header = fsp_get_space_header(space_id, page_size, mtr); + space_header = fsp_get_space_header(space, page_size, mtr); - inode = fsp_alloc_seg_inode(space_header, mtr); + inode = fsp_alloc_seg_inode(space, space_header, mtr); if (inode == NULL) { @@ -2751,7 +2156,7 @@ fseg_create_general( if (block == NULL) { - fsp_free_seg_inode(space_id, page_size, inode, mtr); + fsp_free_seg_inode(space, page_size, inode, mtr); goto funct_exit; } @@ -2863,7 +2268,7 @@ This happens if the segment is big enough to allow extents in the free list, the free list is empty, and the extents can be allocated consecutively from the hint onward. @param[in] inode segment inode -@param[in] space space id +@param[in] space tablespace @param[in] page_size page size @param[in] hint hint which extent would be good as the first extent @@ -2872,7 +2277,7 @@ static void fseg_fill_free_list( fseg_inode_t* inode, - ulint space, + fil_space_t* space, const page_size_t& page_size, ulint hint, mtr_t* mtr) @@ -2932,7 +2337,7 @@ the segment, then tries to allocate from the space free list. NOTE that the extent returned still resides in the segment free list, it is not yet taken off it! @param[in] inode segment inode -@param[in] space space id +@param[in,out] space tablespace @param[in] page_size page size @param[in,out] mtr mini-transaction @retval NULL if no page could be allocated @@ -2943,7 +2348,7 @@ static xdes_t* fseg_alloc_free_extent( fseg_inode_t* inode, - ulint space, + fil_space_t* space, const page_size_t& page_size, mtr_t* mtr) { @@ -3043,21 +2448,21 @@ fseg_alloc_free_page_low( seg_id = mach_read_from_8(seg_inode + FSEG_ID); ut_ad(seg_id); - ut_d(fsp_space_modify_check(space_id, mtr)); + ut_d(fsp_space_modify_check(space, mtr)); ut_ad(fil_page_get_type(page_align(seg_inode)) == FIL_PAGE_INODE); reserved = fseg_n_reserved_pages_low(seg_inode, &used, mtr); - space_header = fsp_get_space_header(space_id, page_size, mtr); + space_header = fsp_get_space_header(space, page_size, mtr); - descr = xdes_get_descriptor_with_space_hdr(space_header, space_id, + descr = xdes_get_descriptor_with_space_hdr(space_header, space, hint, mtr); if (descr == NULL) { /* Hint outside space or too high above free limit: reset hint */ /* The file space header page is always allocated. */ hint = 0; - descr = xdes_get_descriptor(space_id, hint, page_size, mtr); + descr = xdes_get_descriptor(space, hint, page_size, mtr); } /* In the big if-else below we look for ret_page and ret_descr */ @@ -3084,8 +2489,7 @@ take_hinted_page: ========================================================= the hinted page ===============*/ - ret_descr = fsp_alloc_free_extent( - space_id, page_size, hint, mtr); + ret_descr = fsp_alloc_free_extent(space, page_size, hint, mtr); ut_a(ret_descr == descr); @@ -3095,7 +2499,7 @@ take_hinted_page: ret_descr + XDES_FLST_NODE, mtr); /* Try to fill the segment free list */ - fseg_fill_free_list(seg_inode, space_id, page_size, + fseg_fill_free_list(seg_inode, space, page_size, hint + FSP_EXTENT_SIZE, mtr); goto take_hinted_page; /*-----------------------------------------------------------*/ @@ -3104,7 +2508,7 @@ take_hinted_page: && (used >= FSEG_FRAG_LIMIT) && (!!(ret_descr = fseg_alloc_free_extent( - seg_inode, space_id, page_size, mtr)))) { + seg_inode, space, page_size, mtr)))) { /* 3. We take any free extent (which was already assigned above =============================================================== @@ -3150,7 +2554,7 @@ take_hinted_page: return(NULL); } - ret_descr = xdes_lst_get_descriptor(space_id, page_size, + ret_descr = xdes_lst_get_descriptor(space, page_size, first, mtr); ret_page = xdes_get_offset(ret_descr) + xdes_find_bit(ret_descr, XDES_FREE_BIT, TRUE, @@ -3161,7 +2565,7 @@ take_hinted_page: /* 6. We allocate an individual page from the space ===================================================*/ buf_block_t* block = fsp_alloc_free_page( - space_id, page_size, hint, rw_latch, mtr, init_mtr); + space, page_size, hint, rw_latch, mtr, init_mtr); ut_ad(!has_done_reservation || block != NULL); @@ -3184,7 +2588,7 @@ take_hinted_page: /* 7. We allocate a new extent and take its first page ======================================================*/ ret_descr = fseg_alloc_free_extent(seg_inode, - space_id, page_size, mtr); + space, page_size, mtr); if (ret_descr == NULL) { ret_page = FIL_NULL; @@ -3232,7 +2636,7 @@ got_hinted_page: The extent is still in the appropriate list (FSEG_NOT_FULL or FSEG_FREE), and the page is not yet marked as used. */ - ut_ad(xdes_get_descriptor(space_id, ret_page, page_size, mtr) + ut_ad(xdes_get_descriptor(space, ret_page, page_size, mtr) == ret_descr); ut_ad(xdes_mtr_get_bit( @@ -3242,10 +2646,8 @@ got_hinted_page: fseg_mark_page_used(seg_inode, ret_page, ret_descr, mtr); } - ut_ad(space->flags - == mach_read_from_4(FSP_SPACE_FLAGS + space_header)); - return(fsp_page_create(page_id_t(space_id, ret_page), page_size, - rw_latch, mtr, init_mtr)); + return(fsp_page_create(space, ret_page, page_size, rw_latch, + mtr, init_mtr)); } /**********************************************************************//** @@ -3355,7 +2757,7 @@ fsp_reserve_free_pages( ut_a(size < FSP_EXTENT_SIZE); descr = xdes_get_descriptor_with_space_hdr( - space_header, space->id, 0, mtr); + space_header, space, 0, mtr); n_used = xdes_get_n_used(descr, mtr); ut_a(n_used <= size); @@ -3429,7 +2831,7 @@ fsp_reserve_free_extents( fil_space_t* space = mtr_x_lock_space(space_id, mtr); const page_size_t page_size(space->flags); - space_header = fsp_get_space_header(space_id, page_size, mtr); + space_header = fsp_get_space_header(space, page_size, mtr); try_again: size = mach_read_from_4(space_header + FSP_SIZE); ut_ad(size == space->size_in_header); @@ -3645,7 +3047,8 @@ fseg_mark_page_used( /** Frees a single page of a segment. @param[in] seg_inode segment inode -@param[in] page_id page id +@param[in,out] space tablespace +@param[in] offset page number @param[in] page_size page size @param[in] ahi whether we may need to drop the adaptive hash index @@ -3654,7 +3057,8 @@ static void fseg_free_page_low( fseg_inode_t* seg_inode, - const page_id_t& page_id, + fil_space_t* space, + page_no_t offset, const page_size_t& page_size, bool ahi, mtr_t* mtr) @@ -3671,30 +3075,27 @@ fseg_free_page_low( ut_ad(mach_read_from_4(seg_inode + FSEG_MAGIC_N) == FSEG_MAGIC_N_VALUE); ut_ad(!((page_offset(seg_inode) - FSEG_ARR_OFFSET) % FSEG_INODE_SIZE)); - ut_d(fsp_space_modify_check(page_id.space(), mtr)); + ut_d(fsp_space_modify_check(space, mtr)); /* Drop search system page hash index if the page is found in the pool and is hashed */ if (ahi) { - btr_search_drop_page_hash_when_freed(page_id, page_size); + btr_search_drop_page_hash_when_freed( + page_id_t(space->id, offset), page_size); } - descr = xdes_get_descriptor(page_id.space(), page_id.page_no(), - page_size, mtr); + descr = xdes_get_descriptor(space, offset, page_size, mtr); if (xdes_mtr_get_bit(descr, XDES_FREE_BIT, - page_id.page_no() % FSP_EXTENT_SIZE, mtr)) { - fputs("InnoDB: Dump of the tablespace extent descriptor: ", - stderr); - ut_print_buf(stderr, descr, 40); - ib::error() << "InnoDB is trying to free page " << page_id + offset % FSP_EXTENT_SIZE, mtr)) { + ib::fatal() << "InnoDB is trying to free page " + << page_id_t(space->id, offset) << " though it is already marked as free in the" " tablespace! The tablespace free space info is" " corrupt. You may need to dump your tables and" - " recreate the whole database!"; -crash: - ib::fatal() << FORCE_RECOVERY_MSG; + " recreate the whole database!" + << FORCE_RECOVERY_MSG; } state = xdes_get_state(descr, mtr); @@ -3704,7 +3105,7 @@ crash: for (i = 0;; i++) { if (fseg_get_nth_frag_page_no(seg_inode, i, mtr) - == page_id.page_no()) { + == offset) { fseg_set_nth_frag_page_no(seg_inode, i, FIL_NULL, mtr); @@ -3712,7 +3113,7 @@ crash: } } - fsp_free_page(page_id, page_size, mtr); + fsp_free_page(space, offset, page_size, mtr); return; } @@ -3730,10 +3131,11 @@ crash: ut_print_buf(stderr, seg_inode, 40); putc('\n', stderr); - ib::error() << "InnoDB is trying to free page " << page_id + ib::fatal() << "InnoDB is trying to free page " + << page_id_t(space->id, offset) << ", which does not belong to segment " << descr_id - << " but belongs to segment " << seg_id << "."; - goto crash; + << " but belongs to segment " << seg_id << "." + << FORCE_RECOVERY_MSG; } not_full_n_used = mtr_read_ulint(seg_inode + FSEG_NOT_FULL_N_USED, @@ -3753,7 +3155,7 @@ crash: not_full_n_used - 1, MLOG_4BYTES, mtr); } - const ulint bit = page_id.page_no() % FSP_EXTENT_SIZE; + const ulint bit = offset % FSP_EXTENT_SIZE; xdes_set_bit(descr, XDES_FREE_BIT, bit, TRUE, mtr); xdes_set_bit(descr, XDES_CLEAN_BIT, bit, TRUE, mtr); @@ -3762,7 +3164,7 @@ crash: /* The extent has become free: free it to space */ flst_remove(seg_inode + FSEG_NOT_FULL, descr + XDES_FLST_NODE, mtr); - fsp_free_extent(page_id, page_size, mtr); + fsp_free_extent(space, offset, page_size, mtr); } } @@ -3778,20 +3180,24 @@ fseg_free_page( the adaptive hash index */ mtr_t* mtr) /*!< in/out: mini-transaction */ { + DBUG_ENTER("fseg_free_page"); fseg_inode_t* seg_inode; buf_block_t* iblock; - const fil_space_t* space = mtr_x_lock_space(space_id, mtr); + fil_space_t* space = mtr_x_lock_space(space_id, mtr); const page_size_t page_size(space->flags); + DBUG_LOG("fseg_free_page", "space_id: " << space_id + << ", page_no: " << page); + seg_inode = fseg_inode_get(seg_header, space_id, page_size, mtr, &iblock); fil_block_check_type(iblock, FIL_PAGE_INODE, mtr); - const page_id_t page_id(space_id, page); + fseg_free_page_low(seg_inode, space, page, page_size, ahi, mtr); - fseg_free_page_low(seg_inode, page_id, page_size, ahi, mtr); + ut_d(buf_page_set_file_page_was_freed(page_id_t(space_id, page))); - ut_d(buf_page_set_file_page_was_freed(page_id)); + DBUG_VOID_RETURN; } /**********************************************************************//** @@ -3820,7 +3226,7 @@ fseg_page_is_free( == FSEG_MAGIC_N_VALUE); ut_ad(!((page_offset(seg_inode) - FSEG_ARR_OFFSET) % FSEG_INODE_SIZE)); - descr = xdes_get_descriptor(space_id, page, page_size, &mtr); + descr = xdes_get_descriptor(space, page, page_size, &mtr); ut_a(descr); is_free = xdes_mtr_get_bit( @@ -3831,19 +3237,24 @@ fseg_page_is_free( return(is_free); } -/**********************************************************************//** -Frees an extent of a segment to the space free list. */ -static MY_ATTRIBUTE((nonnull)) +/** Free an extent of a segment to the space free list. +@param[in,out] seg_inode segment inode +@param[in,out] space tablespace +@param[in] page_size page size +@param[in] page page number in the extent +@param[in] ahi whether we may need to drop + the adaptive hash index +@param[in,out] mtr mini-transaction */ +MY_ATTRIBUTE((nonnull)) +static void fseg_free_extent( -/*=============*/ - fseg_inode_t* seg_inode, /*!< in: segment inode */ - ulint space, /*!< in: space id */ + fseg_inode_t* seg_inode, + fil_space_t* space, const page_size_t& page_size, - ulint page, /*!< in: a page in the extent */ - bool ahi, /*!< in: whether we may need to drop - the adaptive hash index */ - mtr_t* mtr) /*!< in/out: mini-transaction */ + ulint page, + bool ahi, + mtr_t* mtr) { ulint first_page_in_extent; xdes_t* descr; @@ -3851,7 +3262,6 @@ fseg_free_extent( ulint descr_n_used; ulint i; - ut_ad(seg_inode != NULL); ut_ad(mtr != NULL); descr = xdes_get_descriptor(space, page, page_size, mtr); @@ -3873,7 +3283,7 @@ fseg_free_extent( is hashed */ btr_search_drop_page_hash_when_freed( - page_id_t(space, + page_id_t(space->id, first_page_in_extent + i), page_size); } @@ -3900,13 +3310,13 @@ fseg_free_extent( MLOG_4BYTES, mtr); } - fsp_free_extent(page_id_t(space, page), page_size, mtr); + fsp_free_extent(space, page, page_size, mtr); #ifdef UNIV_DEBUG for (i = 0; i < FSP_EXTENT_SIZE; i++) { buf_page_set_file_page_was_freed( - page_id_t(space, first_page_in_extent + i)); + page_id_t(space->id, first_page_in_extent + i)); } #endif /* UNIV_DEBUG */ } @@ -3940,10 +3350,10 @@ fseg_free_step( space_id = page_get_space_id(page_align(header)); header_page = page_get_page_no(page_align(header)); - const fil_space_t* space = mtr_x_lock_space(space_id, mtr); + fil_space_t* space = mtr_x_lock_space(space_id, mtr); const page_size_t page_size(space->flags); - descr = xdes_get_descriptor(space_id, header_page, page_size, mtr); + descr = xdes_get_descriptor(space, header_page, page_size, mtr); /* Check that the header resides on a page which has not been freed yet */ @@ -3961,13 +3371,13 @@ fseg_free_step( } fil_block_check_type(iblock, FIL_PAGE_INODE, mtr); - descr = fseg_get_first_extent(inode, space_id, page_size, mtr); + descr = fseg_get_first_extent(inode, space, page_size, mtr); if (descr != NULL) { /* Free the extent held by the segment */ page = xdes_get_offset(descr); - fseg_free_extent(inode, space_id, page_size, page, ahi, mtr); + fseg_free_extent(inode, space, page_size, page, ahi, mtr); DBUG_RETURN(FALSE); } @@ -3977,21 +3387,21 @@ fseg_free_step( if (n == ULINT_UNDEFINED) { /* Freeing completed: free the segment inode */ - fsp_free_seg_inode(space_id, page_size, inode, mtr); + fsp_free_seg_inode(space, page_size, inode, mtr); DBUG_RETURN(TRUE); } fseg_free_page_low( - inode, - page_id_t(space_id, fseg_get_nth_frag_page_no(inode, n, mtr)), + inode, space, + fseg_get_nth_frag_page_no(inode, n, mtr), page_size, ahi, mtr); n = fseg_find_last_used_frag_page_slot(inode, mtr); if (n == ULINT_UNDEFINED) { /* Freeing completed: free the segment inode */ - fsp_free_seg_inode(space_id, page_size, inode, mtr); + fsp_free_seg_inode(space, page_size, inode, mtr); DBUG_RETURN(TRUE); } @@ -4022,20 +3432,20 @@ fseg_free_step_not_header( space_id = page_get_space_id(page_align(header)); ut_ad(mtr->is_named_space(space_id)); - const fil_space_t* space = mtr_x_lock_space(space_id, mtr); + fil_space_t* space = mtr_x_lock_space(space_id, mtr); const page_size_t page_size(space->flags); buf_block_t* iblock; inode = fseg_inode_get(header, space_id, page_size, mtr, &iblock); fil_block_check_type(iblock, FIL_PAGE_INODE, mtr); - descr = fseg_get_first_extent(inode, space_id, page_size, mtr); + descr = fseg_get_first_extent(inode, space, page_size, mtr); if (descr != NULL) { /* Free the extent held by the segment */ page = xdes_get_offset(descr); - fseg_free_extent(inode, space_id, page_size, page, ahi, mtr); + fseg_free_extent(inode, space, page_size, page, ahi, mtr); return(FALSE); } @@ -4055,8 +3465,7 @@ fseg_free_step_not_header( return(TRUE); } - fseg_free_page_low(inode, page_id_t(space_id, page_no), page_size, ahi, - mtr); + fseg_free_page_low(inode, space, page_no, page_size, ahi, mtr); return(FALSE); } @@ -4065,28 +3474,24 @@ fseg_free_step_not_header( We think of the extent lists of the segment catenated in the order FSEG_FULL -> FSEG_NOT_FULL -> FSEG_FREE. @param[in] inode segment inode -@param[in] space_id space id +@param[in] space tablespace @param[in] page_size page size @param[in,out] mtr mini-transaction @return the first extent descriptor, or NULL if none */ +MY_ATTRIBUTE((nonnull, warn_unused_result)) static xdes_t* fseg_get_first_extent( fseg_inode_t* inode, - ulint space_id, + const fil_space_t* space, const page_size_t& page_size, mtr_t* mtr) { fil_addr_t first; - xdes_t* descr; - ut_ad(inode && mtr); - - ut_ad(space_id == page_get_space_id(page_align(inode))); + ut_ad(space->id == page_get_space_id(page_align(inode))); ut_ad(mach_read_from_4(inode + FSEG_MAGIC_N) == FSEG_MAGIC_N_VALUE); - first = fil_addr_null; - if (flst_get_len(inode + FSEG_FULL) > 0) { first = flst_get_first(inode + FSEG_FULL, mtr); @@ -4098,15 +3503,14 @@ fseg_get_first_extent( } else if (flst_get_len(inode + FSEG_FREE) > 0) { first = flst_get_first(inode + FSEG_FREE, mtr); - } - - if (first.page == FIL_NULL) { - + } else { return(NULL); } - descr = xdes_lst_get_descriptor(space_id, page_size, first, mtr); - return(descr); + ut_ad(first.page != FIL_NULL); + + return(first.page == FIL_NULL ? NULL + : xdes_lst_get_descriptor(space, page_size, first, mtr)); } #ifdef UNIV_DEBUG @@ -4123,7 +3527,6 @@ fseg_validate_low( ulint space_id; ib_id_t seg_id; mtr_t mtr; - xdes_t* descr; fil_addr_t node_addr; ulint n_used = 0; ulint n_used2 = 0; @@ -4147,11 +3550,8 @@ fseg_validate_low( mtr_start(&mtr); const fil_space_t* space = mtr_x_lock_space( space_id, &mtr); - - const page_size_t page_size(space->flags); - - descr = xdes_lst_get_descriptor(space_id, page_size, - node_addr, &mtr); + const xdes_t* descr = xdes_lst_get_descriptor( + space, page_size_t(space->flags), node_addr, &mtr); ut_a(xdes_get_n_used(descr, &mtr) == 0); ut_a(xdes_get_state(descr, &mtr) == XDES_FSEG); @@ -4169,10 +3569,8 @@ fseg_validate_low( mtr_start(&mtr); const fil_space_t* space = mtr_x_lock_space( space_id, &mtr); - const page_size_t page_size(space->flags); - - descr = xdes_lst_get_descriptor(space_id, page_size, - node_addr, &mtr); + const xdes_t* descr = xdes_lst_get_descriptor( + space, page_size_t(space->flags), node_addr, &mtr); ut_a(xdes_get_n_used(descr, &mtr) > 0); ut_a(xdes_get_n_used(descr, &mtr) < FSP_EXTENT_SIZE); @@ -4193,10 +3591,8 @@ fseg_validate_low( mtr_start(&mtr); const fil_space_t* space = mtr_x_lock_space( space_id, &mtr); - const page_size_t page_size(space->flags); - - descr = xdes_lst_get_descriptor(space_id, page_size, - node_addr, &mtr); + const xdes_t* descr = xdes_lst_get_descriptor( + space, page_size_t(space->flags), node_addr, &mtr); ut_a(xdes_get_n_used(descr, &mtr) == FSP_EXTENT_SIZE); ut_a(xdes_get_state(descr, &mtr) == XDES_FSEG); @@ -4305,7 +3701,6 @@ fseg_print( fseg_print_low(inode, mtr); } #endif /* UNIV_BTR_PRINT */ -#endif /* !UNIV_HOTBACKUP */ #ifdef UNIV_DEBUG /** Print the file segment header to the given output stream. @@ -4344,44 +3739,12 @@ fsp_page_is_free_func( { ut_ad(mtr); - const fil_space_t* space = mtr_x_lock_space(space_id, mtr); + fil_space_t* space = mtr_x_lock_space(space_id, mtr); const page_size_t page_size(space->flags); - xdes_t* descr = xdes_get_descriptor(space_id, page_no, page_size, mtr); + xdes_t* descr = xdes_get_descriptor(space, page_no, page_size, mtr); ut_a(descr); return xdes_mtr_get_bit( descr, XDES_FREE_BIT, page_no % FSP_EXTENT_SIZE, mtr); } - -/**********************************************************************//** -Compute offset after xdes where crypt data can be stored -@return offset */ -ulint -fsp_header_get_crypt_offset( -/*========================*/ - const page_size_t& page_size,/*!< in: page size */ - ulint* max_size) /*!< out: free space available for crypt data */ -{ - ulint pageno = 0; - /* compute first page_no that will have xdes stored on page != 0*/ - - for (ulint i = 0; - (pageno = xdes_calc_descriptor_page(page_size, i)) == 0; ) - i++; - - /* use pageno prior to this...i.e last page on page 0 */ - ut_ad(pageno > 0); - pageno--; - - ulint iv_offset = XDES_ARR_OFFSET + - XDES_SIZE * (1 + xdes_calc_descriptor_index(page_size, pageno)); - - if (max_size != NULL) { - /* return how much free space there is available on page */ - *max_size = (page_size.logical() ? page_size.logical() : UNIV_PAGE_SIZE) - - (FSP_HEADER_OFFSET + iv_offset + FIL_PAGE_DATA_END); - } - - return FSP_HEADER_OFFSET + iv_offset; -} diff --git a/storage/innobase/fsp/fsp0space.cc b/storage/innobase/fsp/fsp0space.cc index f66f7b8fc78..76269a749f9 100644 --- a/storage/innobase/fsp/fsp0space.cc +++ b/storage/innobase/fsp/fsp0space.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2013, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -18,7 +19,7 @@ this program; if not, write to the Free Software Foundation, Inc., /**************************************************//** @file fsp/fsp0space.cc -General shared tablespace implementation. +Shared tablespace implementation. Created 2012-11-16 by Sunny Bains as srv/srv0space.cc *******************************************************/ @@ -27,14 +28,10 @@ Created 2012-11-16 by Sunny Bains as srv/srv0space.cc #include "fsp0space.h" #include "fsp0sysspace.h" -#ifndef UNIV_HOTBACKUP #include "fsp0fsp.h" #include "os0file.h" -#endif /* !UNIV_HOTBACKUP */ - #include "my_sys.h" - /** Check if two tablespaces have common data file names. @param other_space Tablespace to check against this. @return true if they have the same data filenames and paths */ @@ -122,19 +119,6 @@ Tablespace::open_or_create(bool is_temp) break; } - bool atomic_write; - -#if !defined(NO_FALLOCATE) && defined(UNIV_LINUX) - if (!srv_use_doublewrite_buf) { - atomic_write = fil_fusionio_enable_atomic_write( - it->m_handle); - } else { - atomic_write = false; - } -#else - atomic_write = false; -#endif /* !NO_FALLOCATE && UNIV_LINUX */ - /* We can close the handle now and open the tablespace the proper way. */ it->close(); @@ -142,15 +126,14 @@ Tablespace::open_or_create(bool is_temp) if (it == begin) { /* First data file. */ - ulint flags; - - flags = fsp_flags_set_page_size(0, univ_page_size); - /* Create the tablespace entry for the multi-file tablespace in the tablespace manager. */ space = fil_space_create( - m_name, m_space_id, flags, is_temp - ? FIL_TYPE_TEMPORARY : FIL_TYPE_TABLESPACE, it->m_crypt_info); + m_name, m_space_id, FSP_FLAGS_PAGE_SSIZE(), + is_temp + ? FIL_TYPE_TEMPORARY : FIL_TYPE_TABLESPACE, + it->m_crypt_info, + false); } ut_a(fil_validate()); @@ -158,7 +141,7 @@ Tablespace::open_or_create(bool is_temp) /* Create the tablespace node entry for this data file. */ if (!fil_node_create( it->m_filepath, it->m_size, space, false, - atomic_write)) { + TRUE)) { err = DB_ERROR; break; @@ -185,7 +168,6 @@ Tablespace::find(const char* filename) return(false); } - /** Delete all the data files. */ void Tablespace::delete_files() @@ -207,17 +189,6 @@ Tablespace::delete_files() } } -/** Check if undo tablespace. -@return true if undo tablespace */ -bool -Tablespace::is_undo_tablespace( - ulint id) -{ - return(id <= srv_undo_tablespaces_open - && id != srv_sys_space.space_id() - && id != srv_tmp_space.space_id()); -} - /** Use the ADD DATAFILE path to create a Datafile object and add it to the front of m_files. Parse the datafile path into a path and a filename with extension 'ibd'. diff --git a/storage/innobase/fsp/fsp0sysspace.cc b/storage/innobase/fsp/fsp0sysspace.cc index 66b5da15e8b..6f7d09b6faa 100644 --- a/storage/innobase/fsp/fsp0sysspace.cc +++ b/storage/innobase/fsp/fsp0sysspace.cc @@ -29,7 +29,6 @@ Refactored 2013-7-26 by Kevin Lewis #include "fsp0sysspace.h" #include "srv0start.h" #include "trx0sys.h" -#ifndef UNIV_HOTBACKUP #include "dict0load.h" #include "mem0mem.h" #include "os0file.h" @@ -40,9 +39,6 @@ Refactored 2013-7-26 by Kevin Lewis If server passes the option for create/open DB to SE, we should remove such direct reference to server header and global variable */ #include "mysqld.h" -#else -my_bool opt_initialize = 0; -#endif /* !UNIV_HOTBACKUP */ /** The control info of the system tablespace. */ SysTablespace srv_sys_space; @@ -364,7 +360,8 @@ SysTablespace::check_size( also the data file could contain an incomplete extent. So we need to round the size downward to a megabyte.*/ - ulint rounded_size_pages = get_pages_from_size(size); + const ulint rounded_size_pages = static_cast<ulint>( + size >> UNIV_PAGE_SIZE_SHIFT); /* If last file */ if (&file == &m_files.back() && m_auto_extend_last_file) { @@ -375,7 +372,7 @@ SysTablespace::check_size( ib::error() << "The Auto-extending " << name() << " data file '" << file.filepath() << "' is" " of a different size " << rounded_size_pages - << " pages (rounded down to MB) than specified" + << " pages than specified" " in the .cnf file: initial " << file.m_size << " pages, max " << m_last_file_size_max << " (relevant if non-zero) pages!"; @@ -388,7 +385,7 @@ SysTablespace::check_size( if (rounded_size_pages != file.m_size) { ib::error() << "The " << name() << " data file '" << file.filepath() << "' is of a different size " - << rounded_size_pages << " pages (rounded down to MB)" + << rounded_size_pages << " pages" " than the " << file.m_size << " pages specified in" " the .cnf file!"; return(DB_ERROR); @@ -537,7 +534,6 @@ SysTablespace::open_file( return(err); } -#ifndef UNIV_HOTBACKUP /** Check the tablespace header for this tablespace. @param[out] flushed_lsn the value of FIL_PAGE_FILE_FLUSH_LSN @return DB_SUCCESS or error code */ @@ -581,11 +577,11 @@ SysTablespace::read_lsn_and_check_flags(lsn_t* flushed_lsn) first datafile. */ for (int retry = 0; retry < 2; ++retry) { - err = it->validate_first_page(flushed_lsn, false); + err = it->validate_first_page(flushed_lsn); if (err != DB_SUCCESS && (retry == 1 - || it->restore_from_doublewrite(0) != DB_SUCCESS)) { + || it->restore_from_doublewrite())) { it->close(); @@ -612,7 +608,7 @@ SysTablespace::read_lsn_and_check_flags(lsn_t* flushed_lsn) return(DB_SUCCESS); } -#endif /* !UNIV_HOTBACKUP */ + /** Check if a file can be opened in the correct mode. @param[in] file data file object @param[out] reason exact reason if file_status check failed. @@ -759,7 +755,7 @@ SysTablespace::file_found( /* Need to create the system tablespace for new raw device. */ return(file.m_type == SRV_NEW_RAW); } -#ifndef UNIV_HOTBACKUP + /** Check the data file specification. @param[out] create_new_db true if a new database is to be created @param[in] min_expected_size Minimum expected tablespace size in bytes @@ -779,7 +775,8 @@ SysTablespace::check_file_spec( return(DB_ERROR); } - if (get_sum_of_sizes() < min_expected_size / UNIV_PAGE_SIZE) { + if (!m_auto_extend_last_file + && get_sum_of_sizes() < min_expected_size / UNIV_PAGE_SIZE) { ib::error() << "Tablespace size must be at least " << min_expected_size / (1024 * 1024) << " MB"; @@ -909,28 +906,6 @@ SysTablespace::open_or_create( return(err); } -#if !defined(NO_FALLOCATE) && defined(UNIV_LINUX) - /* Note: This should really be per node and not per - tablespace because a tablespace can contain multiple - files (nodes). The implication is that all files of - the tablespace should be on the same medium. */ - - if (fil_fusionio_enable_atomic_write(it->m_handle)) { - - if (srv_use_doublewrite_buf) { - ib::info() << "FusionIO atomic IO enabled," - " disabling the double write buffer"; - - srv_use_doublewrite_buf = false; - } - - it->m_atomic_write = true; - } else { - it->m_atomic_write = false; - } -#else - it->m_atomic_write = false; -#endif /* !NO_FALLOCATE && UNIV_LINUX*/ } if (!create_new_db && flush_lsn) { @@ -966,7 +941,8 @@ SysTablespace::open_or_create( space = fil_space_create( name(), space_id(), flags(), is_temp - ? FIL_TYPE_TEMPORARY : FIL_TYPE_TABLESPACE, m_crypt_info); + ? FIL_TYPE_TEMPORARY : FIL_TYPE_TABLESPACE, m_crypt_info, + false); } ut_a(fil_validate()); @@ -981,7 +957,7 @@ SysTablespace::open_or_create( if (!fil_node_create( it->m_filepath, it->m_size, space, it->m_type != SRV_NOT_RAW, - it->m_atomic_write, max_size)) { + TRUE, max_size)) { err = DB_ERROR; break; @@ -990,7 +966,7 @@ SysTablespace::open_or_create( return(err); } -#endif /* UNIV_HOTBACKUP */ + /** Normalize the file size, convert from megabytes to number of pages. */ void SysTablespace::normalize() diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 3451607a37e..f05ad4421f6 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1779,13 +1780,6 @@ fts_create_in_mem_aux_table( aux_table_name, table->space, n_cols, 0, table->flags, fts_get_table_flags2_for_aux_tables(table->flags2)); - if (DICT_TF_HAS_SHARED_SPACE(table->flags)) { - ut_ad(table->space == fil_space_get_id_by_name( - table->tablespace())); - new_table->tablespace = mem_heap_strdup( - new_table->heap, table->tablespace); - } - if (DICT_TF_HAS_DATA_DIR(table->flags)) { ut_ad(table->data_dir_path != NULL); new_table->data_dir_path = mem_heap_strdup( @@ -1837,7 +1831,7 @@ fts_create_one_common_table( FTS_CONFIG_TABLE_VALUE_COL_LEN); } - error = row_create_table_for_mysql(new_table, NULL, trx, false, + error = row_create_table_for_mysql(new_table, trx, false, FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY); if (error == DB_SUCCESS) { @@ -1857,7 +1851,7 @@ fts_create_one_common_table( TRX_DICT_OP_TABLE. */ trx_dict_op_t op = trx_get_dict_operation(trx); - error = row_create_index_for_mysql(index, trx, NULL, NULL); + error = row_create_index_for_mysql(index, trx, NULL); trx->dict_operation = op; } @@ -1974,7 +1968,7 @@ fts_create_common_tables( op = trx_get_dict_operation(trx); - error = row_create_index_for_mysql(index, trx, NULL, NULL); + error = row_create_index_for_mysql(index, trx, NULL); trx->dict_operation = op; @@ -1992,18 +1986,20 @@ func_exit: return(error); } -/** Creates one FTS auxiliary index table for an FTS index. + +/** Create one FTS auxiliary index table for an FTS index. @param[in,out] trx transaction @param[in] index the index instance @param[in] fts_table fts_table structure -@param[in] heap memory heap +@param[in,out] heap memory heap +@see row_merge_create_fts_sort_index() @return DB_SUCCESS or error code */ static dict_table_t* fts_create_one_index_table( trx_t* trx, const dict_index_t* index, - fts_table_t* fts_table, + const fts_table_t* fts_table, mem_heap_t* heap) { dict_field_t* field; @@ -2027,7 +2023,8 @@ fts_create_one_index_table( charset == &my_charset_latin1 ? DATA_VARCHAR : DATA_VARMYSQL, field->col->prtype, - FTS_INDEX_WORD_LEN); + FTS_MAX_WORD_LEN_IN_CHAR + * DATA_MBMAXLEN(field->col->mbminmaxlen)); dict_mem_table_add_col(new_table, heap, "first_doc_id", DATA_INT, DATA_NOT_NULL | DATA_UNSIGNED, @@ -2051,7 +2048,7 @@ fts_create_one_index_table( (DATA_MTYPE_MAX << 16) | DATA_UNSIGNED | DATA_NOT_NULL, FTS_INDEX_ILIST_LEN); - error = row_create_table_for_mysql(new_table, NULL, trx, false, + error = row_create_table_for_mysql(new_table, trx, false, FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY); if (error == DB_SUCCESS) { @@ -2063,7 +2060,7 @@ fts_create_one_index_table( trx_dict_op_t op = trx_get_dict_operation(trx); - error = row_create_index_for_mysql(index, trx, NULL, NULL); + error = row_create_index_for_mysql(index, trx, NULL); trx->dict_operation = op; } diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index 5989aff83f4..45c71f6f1cd 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -285,7 +286,7 @@ fts_zip_initialize( zip->last_big_block = 0; zip->word.f_len = 0; - memset(zip->word.f_str, 0, FTS_MAX_WORD_LEN); + *zip->word.f_str = 0; ib_vector_reset(zip->blocks); diff --git a/storage/innobase/fut/fut0lst.cc b/storage/innobase/fut/fut0lst.cc index 8b39a4de1fb..2300d189407 100644 --- a/storage/innobase/fut/fut0lst.cc +++ b/storage/innobase/fut/fut0lst.cc @@ -385,115 +385,6 @@ flst_remove( } /********************************************************************//** -Cuts off the tail of the list, including the node given. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -void -flst_cut_end( -/*=========*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node to remove */ - ulint n_nodes,/*!< in: number of nodes to remove, - must be >= 1 */ - mtr_t* mtr) /*!< in: mini-transaction handle */ -{ - ulint space; - flst_node_t* node1; - fil_addr_t node1_addr; - fil_addr_t node2_addr; - ulint len; - - ut_ad(mtr && node2 && base); - ut_ad(mtr_memo_contains_page_flagged(mtr, base, - MTR_MEMO_PAGE_X_FIX - | MTR_MEMO_PAGE_SX_FIX)); - ut_ad(mtr_memo_contains_page_flagged(mtr, node2, - MTR_MEMO_PAGE_X_FIX - | MTR_MEMO_PAGE_SX_FIX)); - ut_ad(n_nodes > 0); - - buf_ptr_get_fsp_addr(node2, &space, &node2_addr); - - node1_addr = flst_get_prev_addr(node2, mtr); - - if (!fil_addr_is_null(node1_addr)) { - - /* Update next field of node1 */ - - if (node1_addr.page == node2_addr.page) { - - node1 = page_align(node2) + node1_addr.boffset; - } else { - bool found; - const page_size_t& page_size - = fil_space_get_page_size(space, &found); - - ut_ad(found); - - node1 = fut_get_ptr(space, page_size, - node1_addr, RW_SX_LATCH, mtr); - } - - flst_write_addr(node1 + FLST_NEXT, fil_addr_null, mtr); - } else { - /* node2 was first in list: update the field in base */ - flst_write_addr(base + FLST_FIRST, fil_addr_null, mtr); - } - - flst_write_addr(base + FLST_LAST, node1_addr, mtr); - - /* Update len of base node */ - len = flst_get_len(base); - ut_ad(len >= n_nodes); - - mlog_write_ulint(base + FLST_LEN, len - n_nodes, MLOG_4BYTES, mtr); -} - -/********************************************************************//** -Cuts off the tail of the list, not including the given node. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -void -flst_truncate_end( -/*==============*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node not to remove */ - ulint n_nodes,/*!< in: number of nodes to remove */ - mtr_t* mtr) /*!< in: mini-transaction handle */ -{ - fil_addr_t node2_addr; - ulint len; - ulint space; - - ut_ad(mtr && node2 && base); - ut_ad(mtr_memo_contains_page_flagged(mtr, base, - MTR_MEMO_PAGE_X_FIX - | MTR_MEMO_PAGE_SX_FIX)); - ut_ad(mtr_memo_contains_page_flagged(mtr, node2, - MTR_MEMO_PAGE_X_FIX - | MTR_MEMO_PAGE_SX_FIX)); - if (n_nodes == 0) { - - ut_ad(fil_addr_is_null(flst_get_next_addr(node2, mtr))); - - return; - } - - buf_ptr_get_fsp_addr(node2, &space, &node2_addr); - - /* Update next field of node2 */ - flst_write_addr(node2 + FLST_NEXT, fil_addr_null, mtr); - - flst_write_addr(base + FLST_LAST, node2_addr, mtr); - - /* Update len of base node */ - len = flst_get_len(base); - ut_ad(len >= n_nodes); - - mlog_write_ulint(base + FLST_LEN, len - n_nodes, MLOG_4BYTES, mtr); -} - -/********************************************************************//** Validates a file-based list. @return TRUE if ok */ ibool diff --git a/storage/innobase/gis/gis0geo.cc b/storage/innobase/gis/gis0geo.cc index b90b47dc08c..436249c0026 100644 --- a/storage/innobase/gis/gis0geo.cc +++ b/storage/innobase/gis/gis0geo.cc @@ -364,8 +364,9 @@ mbr_join_square( b += 2; } while (a != end); - /* Check for infinity or NaN, so we don't get NaN in calculations */ - if (my_isinf(square) || my_isnan(square)) { + /* Check if finite (not infinity or NaN), + so we don't get NaN in calculations */ + if (!isfinite(square)) { return DBL_MAX; } diff --git a/storage/innobase/gis/gis0rtree.cc b/storage/innobase/gis/gis0rtree.cc index ee89e0a913f..559ac2a6aa5 100644 --- a/storage/innobase/gis/gis0rtree.cc +++ b/storage/innobase/gis/gis0rtree.cc @@ -28,8 +28,6 @@ Created 2013/03/27 Allen Lai and Jimmy Yang #include "page0cur.h" #include "page0zip.h" #include "gis0rtree.h" - -#ifndef UNIV_HOTBACKUP #include "btr0cur.h" #include "btr0sea.h" #include "btr0pcur.h" @@ -40,8 +38,6 @@ Created 2013/03/27 Allen Lai and Jimmy Yang #include "srv0mon.h" #include "gis0geo.h" -#endif /* UNIV_HOTBACKUP */ - /*************************************************************//** Initial split nodes info for R-tree split. @return initialized split nodes array */ @@ -1988,7 +1984,7 @@ rtr_estimate_n_rows_in_range( mtr_commit(&mtr); mem_heap_free(heap); - if (my_isinf(area) || my_isnan(area)) { + if (!isfinite(area)) { return(HA_POS_ERROR); } diff --git a/storage/innobase/gis/gis0sea.cc b/storage/innobase/gis/gis0sea.cc index 2c0c5b453a3..0bc96199486 100644 --- a/storage/innobase/gis/gis0sea.cc +++ b/storage/innobase/gis/gis0sea.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -28,8 +29,6 @@ Created 2014/01/16 Jimmy Yang #include "page0cur.h" #include "page0zip.h" #include "gis0rtree.h" - -#ifndef UNIV_HOTBACKUP #include "btr0cur.h" #include "btr0sea.h" #include "btr0pcur.h" @@ -40,8 +39,6 @@ Created 2014/01/16 Jimmy Yang #include "srv0mon.h" #include "gis0geo.h" -#endif /* UNIV_HOTBACKUP */ - /*************************************************************//** Pop out used parent path entry, until we find the parent with matching page number */ @@ -133,12 +130,10 @@ rtr_pcur_getnext_from_path( || latch_mode & BTR_MODIFY_LEAF); mtr_s_lock(dict_index_get_lock(index), mtr); } else { - ut_ad(mtr_memo_contains(mtr, dict_index_get_lock(index), - MTR_MEMO_SX_LOCK) - || mtr_memo_contains(mtr, dict_index_get_lock(index), - MTR_MEMO_S_LOCK) - || mtr_memo_contains(mtr, dict_index_get_lock(index), - MTR_MEMO_X_LOCK)); + ut_ad(mtr_memo_contains_flagged(mtr, &index->lock, + MTR_MEMO_SX_LOCK + | MTR_MEMO_S_LOCK + | MTR_MEMO_X_LOCK)); } const page_size_t& page_size = dict_table_page_size(index->table); @@ -605,10 +600,9 @@ rtr_pcur_open_low( } if (latch_mode & BTR_MODIFY_TREE) { - ut_ad(mtr_memo_contains(mtr, dict_index_get_lock(index), - MTR_MEMO_X_LOCK) - || mtr_memo_contains(mtr, dict_index_get_lock(index), - MTR_MEMO_SX_LOCK)); + ut_ad(mtr_memo_contains_flagged(mtr, &index->lock, + MTR_MEMO_X_LOCK + | MTR_MEMO_SX_LOCK)); tree_latched = true; } @@ -1550,7 +1544,6 @@ rtr_copy_buf( (another) thread while it was copied. */ memcpy(&matches->block.page, &block->page, sizeof(buf_page_t)); matches->block.frame = block->frame; -#ifndef UNIV_HOTBACKUP matches->block.unzip_LRU = block->unzip_LRU; ut_d(matches->block.in_unzip_LRU_list = block->in_unzip_LRU_list); @@ -1569,12 +1562,9 @@ rtr_copy_buf( matches->block.curr_n_fields = block->curr_n_fields; matches->block.curr_left_side = block->curr_left_side; matches->block.index = block->index; - matches->block.made_dirty_with_no_latch - = block->made_dirty_with_no_latch; ut_d(matches->block.debug_latch = block->debug_latch); -#endif /* !UNIV_HOTBACKUP */ } /****************************************************************//** diff --git a/storage/innobase/ha/ha0ha.cc b/storage/innobase/ha/ha0ha.cc index f57a6d383d9..cf92f22ac15 100644 --- a/storage/innobase/ha/ha0ha.cc +++ b/storage/innobase/ha/ha0ha.cc @@ -28,11 +28,10 @@ Created 8/22/1994 Heikki Tuuri #include "ha0ha.ic" #endif -#ifndef UNIV_HOTBACKUP #ifdef UNIV_DEBUG # include "buf0buf.h" #endif /* UNIV_DEBUG */ -# include "btr0sea.h" +#include "btr0sea.h" #include "page0page.h" /*************************************************************//** @@ -537,4 +536,3 @@ builds, see http://bugs.mysql.com/36941 */ (ulong) n_bufs); } } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/ha/hash0hash.cc b/storage/innobase/ha/hash0hash.cc index 234fd7ac032..7c5798ae254 100644 --- a/storage/innobase/ha/hash0hash.cc +++ b/storage/innobase/ha/hash0hash.cc @@ -32,8 +32,6 @@ Created 5/20/1997 Heikki Tuuri #include "mem0mem.h" #include "sync0sync.h" -#ifndef UNIV_HOTBACKUP - /************************************************************//** Reserves the mutex for a fold value in a hash table. */ void @@ -249,8 +247,6 @@ hash_unlock_x_all_but( } } -#endif /* !UNIV_HOTBACKUP */ - /*************************************************************//** Creates a hash table with >= n array cells. The actual number of cells is chosen to be a prime number slightly bigger than n. @@ -277,14 +273,12 @@ hash_create( table->type = HASH_TABLE_SYNC_NONE; table->array = array; table->n_cells = prime; -#ifndef UNIV_HOTBACKUP -# if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG +#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG table->adaptive = FALSE; -# endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ +#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ table->n_sync_obj = 0; table->sync_obj.mutexes = NULL; table->heaps = NULL; -#endif /* !UNIV_HOTBACKUP */ table->heap = NULL; ut_d(table->magic_n = HASH_TABLE_MAGIC_N); @@ -307,7 +301,6 @@ hash_table_free( ut_free(table); } -#ifndef UNIV_HOTBACKUP /*************************************************************//** Creates a sync object array to protect a hash table. ::sync_obj can be mutexes or rw_locks depening on the type of @@ -362,4 +355,3 @@ hash_create_sync_obj( table->n_sync_obj = n_sync_obj; } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 87941833182..ef599bc5d04 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1,10 +1,10 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -65,8 +65,6 @@ this program; if not, write to the Free Software Foundation, Inc., #include <key.h> /* Include necessary InnoDB headers */ -#include "api0api.h" -#include "api0misc.h" #include "btr0btr.h" #include "btr0cur.h" #include "btr0bulk.h" @@ -91,7 +89,7 @@ this program; if not, write to the Free Software Foundation, Inc., #include "fts0types.h" #include "ibuf0ibuf.h" #include "lock0lock.h" -#include "log0log.h" +#include "log0crypt.h" #include "mem0mem.h" #include "mtr0mtr.h" #include "os0file.h" @@ -126,6 +124,14 @@ this program; if not, write to the Free Software Foundation, Inc., #define thd_get_trx_isolation(X) ((enum_tx_isolation)thd_tx_isolation(X)) extern "C" void thd_mark_transaction_to_rollback(MYSQL_THD thd, bool all); +unsigned long long thd_get_query_id(const MYSQL_THD thd); +TABLE *find_fk_open_table(THD *thd, const char *db, size_t db_len, + const char *table, size_t table_len); +MYSQL_THD create_thd(); +void destroy_thd(MYSQL_THD thd); +void reset_thd(MYSQL_THD thd); +TABLE *open_purge_table(THD *thd, const char *db, size_t dblen, + const char *tb, size_t tblen); #ifdef MYSQL_DYNAMIC_PLUGIN #define tc_size 400 @@ -182,7 +188,6 @@ static ulong commit_threads = 0; static mysql_cond_t commit_cond; static mysql_mutex_t commit_cond_m; static mysql_mutex_t pending_checkpoint_mutex; -static bool innodb_inited = 0; #define INSIDE_HA_INNOBASE_CC @@ -239,10 +244,9 @@ static char* innobase_server_stopword_table = NULL; /* Below we have boolean-valued start-up parameters, and their default values */ -static ulong innobase_fast_shutdown = 1; static my_bool innobase_file_format_check = TRUE; -static my_bool innobase_use_atomic_writes = FALSE; -static my_bool innobase_use_fallocate = TRUE; +static my_bool innobase_use_atomic_writes = TRUE; +static my_bool innobase_use_fallocate; static my_bool innobase_use_doublewrite = TRUE; static my_bool innobase_use_checksums = TRUE; static my_bool innobase_locks_unsafe_for_binlog = FALSE; @@ -254,7 +258,6 @@ static my_bool innodb_optimize_fulltext_only = FALSE; static char* innodb_version_str = (char*) INNODB_VERSION_STR; -extern uint srv_n_fil_crypt_threads; extern uint srv_fil_crypt_rotate_key_age; extern uint srv_n_fil_crypt_iops; @@ -281,6 +284,80 @@ void set_my_errno(int err) errno = err; } + +/** Checks whether the file name belongs to a partition of a table. +@param[in] file_name file name +@return pointer to the end of the table name part of the file name, or NULL */ +static +char* +is_partition( +/*=========*/ + char* file_name) +{ + /* We look for pattern #P# to see if the table is partitioned + MariaDB table. */ +#ifdef _WIN32 + return strstr(file_name, "#p#"); +#else + return strstr(file_name, "#P#"); +#endif /* _WIN32 */ +} + +/** Signal to shut down InnoDB (NULL if shutdown was signaled, or if +running in innodb_read_only mode, srv_read_only_mode) */ +volatile st_my_thread_var *srv_running; +/** Service thread that waits for the server shutdown and stops purge threads. +Purge workers have THDs that are needed to calculate virtual columns. +This THDs must be destroyed rather early in the server shutdown sequence. +This service thread creates a THD and idly waits for it to get a signal to +die. Then it notifies all purge workers to shutdown. +*/ +static pthread_t thd_destructor_thread; + +pthread_handler_t +thd_destructor_proxy(void *) +{ + mysql_mutex_t thd_destructor_mutex; + mysql_cond_t thd_destructor_cond; + + my_thread_init(); + mysql_mutex_init(PSI_NOT_INSTRUMENTED, &thd_destructor_mutex, 0); + mysql_cond_init(PSI_NOT_INSTRUMENTED, &thd_destructor_cond, 0); + + st_my_thread_var *myvar= _my_thread_var(); + THD *thd= create_thd(); + thd_proc_info(thd, "InnoDB background thread"); + + myvar->current_mutex = &thd_destructor_mutex; + myvar->current_cond = &thd_destructor_cond; + + mysql_mutex_lock(&thd_destructor_mutex); + srv_running = myvar; + /* wait until the server wakes the THD to abort and die */ + while (!srv_running->abort) + mysql_cond_wait(&thd_destructor_cond, &thd_destructor_mutex); + mysql_mutex_unlock(&thd_destructor_mutex); + srv_running = NULL; + + if (srv_fast_shutdown == 0) { + while (trx_sys_any_active_transactions()) { + os_thread_sleep(1000); + } + + /* Some background threads might generate undo pages that will + need to be purged, so they have to be shut down before purge + threads if slow shutdown is requested. */ + srv_shutdown_bg_undo_sources(); + } + srv_purge_wakeup(); + + destroy_thd(thd); + mysql_cond_destroy(&thd_destructor_cond); + mysql_mutex_destroy(&thd_destructor_mutex); + my_thread_end(); + return 0; +} + /** Return the InnoDB ROW_FORMAT enum value @param[in] row_format row_format from "innodb_default_row_format" @return InnoDB ROW_FORMAT value from rec_format_t enum. */ @@ -304,25 +381,6 @@ get_row_format( static ulong innodb_default_row_format = DEFAULT_ROW_FORMAT_DYNAMIC; -#ifdef UNIV_DEBUG -/** Values for --innodb-debug-compress names. */ -static const char* innodb_debug_compress_names[] = { - "none", - "zlib", - "lz4", - "lz4hc", - NullS -}; - -/** Enumeration of --innodb-debug-compress */ -static TYPELIB innodb_debug_compress_typelib = { - array_elements(innodb_debug_compress_names) - 1, - "innodb_debug_compress_typelib", - innodb_debug_compress_names, - NULL -}; -#endif /* UNIV_DEBUG */ - /** Possible values for system variable "innodb_stats_method". The values are defined the same as its corresponding MyISAM system variable "myisam_stats_method"(see "myisam_stats_method_names"), for better usability */ @@ -415,20 +473,6 @@ static const char* innobase_change_buffering_values[IBUF_USE_COUNT] = { "all" /* IBUF_USE_ALL */ }; - -/* This tablespace name is reserved by InnoDB in order to explicitly -create a file_per_table tablespace for the table. */ -const char reserved_file_per_table_space_name[] = "innodb_file_per_table"; - -/* This tablespace name is reserved by InnoDB for the system tablespace -which uses space_id 0 and stores extra types of system pages like UNDO -and doublewrite. */ -const char reserved_system_space_name[] = "innodb_system"; - -/* This tablespace name is reserved by InnoDB for the predefined temporary -tablespace. */ -const char reserved_temporary_space_name[] = "innodb_temporary"; - /* Call back function array defined by MySQL and used to retrieve FTS results. */ const struct _ft_vft ft_vft_result = {NULL, @@ -453,9 +497,11 @@ static mysql_pfs_key_t innobase_share_mutex_key; static mysql_pfs_key_t commit_cond_mutex_key; static mysql_pfs_key_t commit_cond_key; static mysql_pfs_key_t pending_checkpoint_mutex_key; +static mysql_pfs_key_t thd_destructor_thread_key; static PSI_mutex_info all_pthread_mutexes[] = { PSI_KEY(commit_cond_mutex), + PSI_KEY(pending_checkpoint_mutex), PSI_KEY(innobase_share_mutex) }; @@ -528,8 +574,7 @@ static PSI_mutex_info all_innodb_mutexes[] = { PSI_KEY(rtr_path_mutex), PSI_KEY(rtr_ssn_mutex), PSI_KEY(trx_sys_mutex), - PSI_KEY(zip_pad_mutex), - PSI_KEY(master_key_id_mutex), + PSI_KEY(zip_pad_mutex) }; # endif /* UNIV_PFS_MUTEX */ @@ -580,6 +625,7 @@ static PSI_thread_info all_innodb_threads[] = { PSI_KEY(srv_purge_thread), PSI_KEY(srv_worker_thread), PSI_KEY(trx_rollback_clean_thread), + PSI_KEY(thd_destructor_thread), }; # endif /* UNIV_PFS_THREAD */ @@ -594,74 +640,6 @@ static PSI_file_info all_innodb_files[] = { # endif /* UNIV_PFS_IO */ #endif /* HAVE_PSI_INTERFACE */ -/** Set up InnoDB API callback function array */ -ib_cb_t innodb_api_cb[] = { - (ib_cb_t) ib_cursor_open_table, - (ib_cb_t) ib_cursor_read_row, - (ib_cb_t) ib_cursor_insert_row, - (ib_cb_t) ib_cursor_delete_row, - (ib_cb_t) ib_cursor_update_row, - (ib_cb_t) ib_cursor_moveto, - (ib_cb_t) ib_cursor_first, - (ib_cb_t) ib_cursor_next, - (ib_cb_t) ib_cursor_set_match_mode, - (ib_cb_t) ib_sec_search_tuple_create, - (ib_cb_t) ib_clust_read_tuple_create, - (ib_cb_t) ib_tuple_delete, - (ib_cb_t) ib_tuple_read_u8, - (ib_cb_t) ib_tuple_read_u16, - (ib_cb_t) ib_tuple_read_u32, - (ib_cb_t) ib_tuple_read_u64, - (ib_cb_t) ib_tuple_read_i8, - (ib_cb_t) ib_tuple_read_i16, - (ib_cb_t) ib_tuple_read_i32, - (ib_cb_t) ib_tuple_read_i64, - (ib_cb_t) ib_tuple_get_n_cols, - (ib_cb_t) ib_col_set_value, - (ib_cb_t) ib_col_get_value, - (ib_cb_t) ib_col_get_meta, - (ib_cb_t) ib_trx_begin, - (ib_cb_t) ib_trx_commit, - (ib_cb_t) ib_trx_rollback, - (ib_cb_t) ib_trx_start, - (ib_cb_t) ib_trx_release, - (ib_cb_t) ib_cursor_lock, - (ib_cb_t) ib_cursor_close, - (ib_cb_t) ib_cursor_new_trx, - (ib_cb_t) ib_cursor_reset, - (ib_cb_t) ib_col_get_name, - (ib_cb_t) ib_table_truncate, - (ib_cb_t) ib_cursor_open_index_using_name, - (ib_cb_t) ib_cfg_get_cfg, - (ib_cb_t) ib_cursor_set_memcached_sync, - (ib_cb_t) ib_cursor_set_cluster_access, - (ib_cb_t) ib_cursor_commit_trx, - (ib_cb_t) ib_cfg_trx_level, - (ib_cb_t) ib_tuple_get_n_user_cols, - (ib_cb_t) ib_cursor_set_lock_mode, - (ib_cb_t) ib_get_idx_field_name, - (ib_cb_t) ib_trx_get_start_time, - (ib_cb_t) ib_cfg_bk_commit_interval, - (ib_cb_t) ib_ut_strerr, - (ib_cb_t) ib_cursor_stmt_begin, - (ib_cb_t) ib_trx_read_only, - (ib_cb_t) ib_is_virtual_table -}; - -/******************************************************************//** -Function used to loop a thread (for debugging/instrumentation -purpose). */ -void -srv_debug_loop(void) -/*================*/ -{ - ibool set = TRUE; - - while (set) { - os_thread_yield(); - } -} - /******************************************************************//** Debug function used to read a MBR data */ @@ -710,8 +688,6 @@ ha_create_table_option innodb_table_option_list[]= /* With this option user can set zip compression level for page compression for this table*/ HA_TOPTION_NUMBER("PAGE_COMPRESSION_LEVEL", page_compression_level, 0, 1, 9, 1), - /* With this option user can enable atomic writes feature for this table */ - HA_TOPTION_ENUM("ATOMIC_WRITES", atomic_writes, "DEFAULT,ON,OFF", 0), /* With this option the user can enable encryption for the table */ HA_TOPTION_ENUM("ENCRYPTED", encryption, "DEFAULT,YES,NO", 0), /* With this option the user defines the key identifier using for the encryption */ @@ -851,13 +827,17 @@ innobase_map_isolation_level( /** Gets field offset for a field in a table. @param[in] table MySQL table object -@param[in] field MySQL field object +@param[in] field MySQL field object (from table->field array) @return offset */ static inline uint get_field_offset( const TABLE* table, - const Field* field); + const Field* field) +{ + return field->offset(table->record[0]); +} + /*************************************************************//** Check for a valid value of innobase_compression_algorithm. @@ -878,6 +858,7 @@ static ibool innodb_have_lz4=IF_LZ4(1, 0); static ibool innodb_have_lzma=IF_LZMA(1, 0); static ibool innodb_have_bzip2=IF_BZIP2(1, 0); static ibool innodb_have_snappy=IF_SNAPPY(1, 0); +static ibool innodb_have_punch_hole=IF_PUNCH_HOLE(1, 0); static int @@ -1037,6 +1018,8 @@ static SHOW_VAR innodb_status_variables[]= { (char*) &export_vars.innodb_pages_created, SHOW_LONG}, {"pages_read", (char*) &export_vars.innodb_pages_read, SHOW_LONG}, + {"pages0_read", + (char*) &export_vars.innodb_page0_read, SHOW_LONG}, {"pages_written", (char*) &export_vars.innodb_pages_written, SHOW_LONG}, {"row_lock_current_waits", @@ -1083,20 +1066,6 @@ static SHOW_VAR innodb_status_variables[]= { /* Status variables for page compression */ {"page_compression_saved", (char*) &export_vars.innodb_page_compression_saved, SHOW_LONGLONG}, - {"page_compression_trim_sect512", - (char*) &export_vars.innodb_page_compression_trim_sect512, SHOW_LONGLONG}, - {"page_compression_trim_sect1024", - (char*) &export_vars.innodb_page_compression_trim_sect1024, SHOW_LONGLONG}, - {"page_compression_trim_sect2048", - (char*) &export_vars.innodb_page_compression_trim_sect2048, SHOW_LONGLONG}, - {"page_compression_trim_sect4096", - (char*) &export_vars.innodb_page_compression_trim_sect4096, SHOW_LONGLONG}, - {"page_compression_trim_sect8192", - (char*) &export_vars.innodb_page_compression_trim_sect8192, SHOW_LONGLONG}, - {"page_compression_trim_sect16384", - (char*) &export_vars.innodb_page_compression_trim_sect16384, SHOW_LONGLONG}, - {"page_compression_trim_sect32768", - (char*) &export_vars.innodb_page_compression_trim_sect32768, SHOW_LONGLONG}, {"num_index_pages_written", (char*) &export_vars.innodb_index_pages_written, SHOW_LONGLONG}, {"num_non_index_pages_written", @@ -1105,8 +1074,6 @@ static SHOW_VAR innodb_status_variables[]= { (char*) &export_vars.innodb_pages_page_compressed, SHOW_LONGLONG}, {"num_page_compressed_trim_op", (char*) &export_vars.innodb_page_compressed_trim_op, SHOW_LONGLONG}, - {"num_page_compressed_trim_op_saved", - (char*) &export_vars.innodb_page_compressed_trim_op_saved, SHOW_LONGLONG}, {"num_pages_page_decompressed", (char*) &export_vars.innodb_pages_page_decompressed, SHOW_LONGLONG}, {"num_pages_page_compression_error", @@ -1125,6 +1092,8 @@ static SHOW_VAR innodb_status_variables[]= { (char*) &innodb_have_bzip2, SHOW_BOOL}, {"have_snappy", (char*) &innodb_have_snappy, SHOW_BOOL}, + {"have_punch_hole", + (char*) &innodb_have_punch_hole, SHOW_BOOL}, /* Defragmentation */ {"defragment_compression_failures", @@ -1185,6 +1154,11 @@ static SHOW_VAR innodb_status_variables[]= { {"scrub_background_page_split_failures_unknown", (char*) &export_vars.innodb_scrub_page_split_failures_unknown, SHOW_LONG}, + {"scrub_log", + (char*) &export_vars.innodb_scrub_log, + SHOW_LONGLONG}, + {"encryption_num_key_requests", + (char*) &export_vars.innodb_encryption_key_requests, SHOW_LONGLONG}, {NullS, NullS, SHOW_LONG} }; @@ -1408,20 +1382,6 @@ innobase_rollback_by_xid( XID* xid); /*!< in: X/Open XA transaction identification */ -#ifdef MYSQL_TABLESPACES -/** This API handles CREATE, ALTER & DROP commands for InnoDB tablespaces. -@param[in] hton Handlerton of InnoDB -@param[in] thd Connection -@param[in] alter_info Describies the command and how to do it. -@return MySQL error code*/ -static -int -innobase_alter_tablespace( - handlerton* hton, - THD* thd, - st_alter_tablespace* alter_info); -#endif /* MYSQL_TABLESPACES */ - /** Remove all tables in the named database inside InnoDB. @param[in] hton handlerton from InnoDB @param[in] path Database path; Inside InnoDB the name of the last @@ -1693,6 +1653,56 @@ thd_trx_is_read_only( return(thd != 0 && thd_tx_is_read_only(thd)); } +static MYSQL_THDVAR_BOOL(background_thread, + PLUGIN_VAR_NOCMDOPT | PLUGIN_VAR_NOSYSVAR, + "Internal (not user visible) flag to mark " + "background purge threads", NULL, NULL, 0); + +/** Create a MYSQL_THD for background purge threads and mark it as such. +@returns new MYSQL_THD */ +MYSQL_THD +innobase_create_background_thd() +/*============================*/ +{ + MYSQL_THD thd= create_thd(); + thd_proc_info(thd, "InnoDB background thread"); + THDVAR(thd, background_thread) = true; + return thd; +} + + +/** Destroy a background purge thread THD. +@param[in] thd MYSQL_THD to destroy */ +void +innobase_destroy_background_thd( +/*============================*/ + MYSQL_THD thd) +{ + /* need to close the connection explicitly, the server won't do it + if innodb is in the PLUGIN_IS_DYING state */ + innobase_close_connection(innodb_hton_ptr, thd); + thd_set_ha_data(thd, innodb_hton_ptr, NULL); + destroy_thd(thd); +} + +/** Close opened tables, free memory, delete items for a MYSQL_THD. +@param[in] thd MYSQL_THD to reset */ +void +innobase_reset_background_thd(MYSQL_THD thd) +{ + if (!thd) { + thd = current_thd; + } + + ut_ad(thd); + ut_ad(THDVAR(thd, background_thread)); + + /* background purge thread */ + reset_thd(thd); + thd_proc_info(thd, ""); +} + + #if 0 /** Check if the transaction can be rolled back @@ -1790,13 +1800,6 @@ innobase_srv_conc_enter_innodb( } #endif /* WITH_WSREP */ - /* We rely on server to do external_lock(F_UNLCK) to reset the - srv_conc.n_active counter. Since there are no locks on instrinsic - tables, we should skip this for intrinsic temporary tables. */ - if (dict_table_is_intrinsic(prebuilt->table)) { - return; - } - trx_t* trx = prebuilt->trx; if (srv_thread_concurrency) { if (trx->n_tickets_to_enter_innodb > 0) { @@ -1835,13 +1838,6 @@ innobase_srv_conc_exit_innodb( } #endif /* WITH_WSREP */ - /* We rely on server to do external_lock(F_UNLCK) to reset the - srv_conc.n_active counter. Since there are no locks on instrinsic - tables, we should skip this for intrinsic temporary tab */ - if (dict_table_is_intrinsic(prebuilt->table)) { - return; - } - trx_t* trx = prebuilt->trx; #ifdef UNIV_DEBUG btrsea_sync_check check(trx->has_search_latch); @@ -1963,29 +1959,6 @@ thd_innodb_tmpdir( return(tmp_dir); } -/** Obtain the private handler of InnoDB session specific data. -@param[in,out] thd MySQL thread handler. -@return reference to private handler */ -MY_ATTRIBUTE((warn_unused_result)) -static -innodb_session_t*& -thd_to_innodb_session( - THD* thd) -{ - innodb_session_t*& innodb_session = - *(innodb_session_t**) thd_ha_data(thd, innodb_hton_ptr); - - if (innodb_session != NULL) { - return(innodb_session); - } - - innodb_session = UT_NEW_NOKEY(innodb_session_t()); - - thd_set_ha_data(thd, innodb_hton_ptr, innodb_session); - - return(innodb_session); -} - /** Obtain the InnoDB transaction of a MySQL thread. @param[in,out] thd MySQL thread handler. @return reference to transaction pointer */ @@ -1994,10 +1967,7 @@ trx_t*& thd_to_trx( THD* thd) { - innodb_session_t*& innodb_session = thd_to_innodb_session(thd); - ut_ad(innodb_session != NULL); - - return(innodb_session->m_trx); + return(*(trx_t**) thd_ha_data(thd, innodb_hton_ptr)); } #ifdef WITH_WSREP @@ -2013,47 +1983,6 @@ thd_to_trx_id( } #endif /* WITH_WSREP */ -/** Check if statement is of type INSERT .... SELECT that involves -use of intrinsic tables. -@param[in] thd thread handler -@return true if INSERT .... SELECT statement. */ -static inline -bool -thd_is_ins_sel_stmt(THD* user_thd) -{ - /* If the session involves use of intrinsic table - and it is trying to fetch the result from non-temporary tables - it indicates "insert .... select" statement. For non-temporary - table this is verifed using the locked tables count but for - intrinsic table as external_lock is not invoked this count is - not updated. - - Why is this needed ? - Use of AHI is blocked if statement is insert .... select statement. */ - innodb_session_t* innodb_priv = thd_to_innodb_session(user_thd); - return(innodb_priv->count_register_table_handler() > 0 ? true : false); -} - -/** Add the table handler to thread cache. -Obtain the InnoDB transaction of a MySQL thread. -@param[in,out] table table handler -@param[in,out] heap heap for allocating system columns. -@param[in,out] thd MySQL thread handler */ -static inline -void -add_table_to_thread_cache( - dict_table_t* table, - mem_heap_t* heap, - THD* thd) -{ - dict_table_add_system_columns(table, heap); - - dict_table_set_big_rows(table); - - innodb_session_t*& priv = thd_to_innodb_session(thd); - priv->register_table_handler(table->name.m_name, table); -} - /********************************************************************//** Call this function when mysqld passes control to the client. That is to avoid deadlocks on the adaptive hash S-latch possibly held by thd. For more @@ -2068,14 +1997,8 @@ innobase_release_temporary_latches( { DBUG_ASSERT(hton == innodb_hton_ptr); - if (!innodb_inited) { - - return(0); - } - - trx_t* trx = thd_to_trx(thd); - - if (trx != NULL) { + if (!srv_was_started) { + } else if (trx_t* trx = thd_to_trx(thd)) { trx_search_latch_release_if_reserved(trx); } @@ -2296,8 +2219,6 @@ convert_error_code_to_mysql( return(HA_ERR_TABLE_CORRUPT); case DB_FTS_TOO_MANY_WORDS_IN_PHRASE: return(HA_ERR_FTS_TOO_MANY_WORDS_IN_PHRASE); - case DB_WRONG_FILE_NAME: - return(HA_ERR_GENERIC); // when can this happen? case DB_COMPUTE_VALUE_FAILED: return(HA_ERR_GENERIC); // impossible } @@ -2508,7 +2429,7 @@ innobase_get_stmt_unsafe( LEX_STRING* stmt; const char* query=NULL; - stmt = thd_query_string(thd); + stmt = thd ? thd_query_string(thd) : NULL; // MySQL 5.7 //stmt = thd_query_unsafe(thd); @@ -2541,11 +2462,11 @@ innobase_get_stmt_safe( ut_ad(buflen > 1); - stmt = thd_query_string(thd); + stmt = thd ? thd_query_string(thd) : NULL; if (stmt && stmt->str) { - length = stmt->length > buflen ? buflen : stmt->length; - memcpy(buf, stmt->str, length-1); + length = stmt->length >= buflen ? buflen - 1 : stmt->length; + memcpy(buf, stmt->str, length); buf[length]='\0'; } else { buf[0]='\0'; @@ -2700,11 +2621,10 @@ innobase_mysql_tmpfile( char errbuf[MYSYS_STRERROR_SIZE]; DBUG_PRINT("error",("Got error %d on dup",fd2)); set_my_errno(errno); - my_strerror(errbuf, sizeof(errbuf), my_errno); my_error(EE_OUT_OF_FILERESOURCES, MYF(0), - "ib*", my_errno, - errbuf); + "ib*", errno, + my_strerror(errbuf, sizeof(errbuf), errno)); } my_close(fd, MYF(MY_WME)); } @@ -2771,118 +2691,6 @@ innobase_raw_format( return(ut_str_sql_format(buf_tmp, buf_tmp_used, buf, buf_size)); } -#ifdef MYSQL_COMPRESSION -/** Check if the string is "empty" or "none". -@param[in] algorithm Compression algorithm to check -@return true if no algorithm requested */ -bool -Compression::is_none(const char* algorithm) -{ - /* NULL is the same as NONE */ - if (algorithm == NULL - || *algorithm == 0 - || innobase_strcasecmp(algorithm, "none") == 0) { - return(true); - } - - return(false); -} - -/** Check for supported COMPRESS := (ZLIB | LZ4 | NONE) supported values -@param[in] name Name of the compression algorithm -@param[out] compression The compression algorithm -@return DB_SUCCESS or DB_UNSUPPORTED */ -dberr_t -Compression::check( - const char* algorithm, - Compression* compression) -{ - if (is_none(algorithm)) { - - compression->m_type = NONE; - - } else if (innobase_strcasecmp(algorithm, "zlib") == 0) { - - compression->m_type = ZLIB; - - } else if (innobase_strcasecmp(algorithm, "lz4") == 0) { - - compression->m_type = LZ4; - - } else { - return(DB_UNSUPPORTED); - } - - return(DB_SUCCESS); -} - -/** Check for supported COMPRESS := (ZLIB | LZ4 | NONE) supported values -@param[in] name Name of the compression algorithm -@param[out] compression The compression algorithm -@return DB_SUCCESS or DB_UNSUPPORTED */ -dberr_t -Compression::validate(const char* algorithm) -{ - Compression compression; - - return(check(algorithm, &compression)); -} -#endif /* MYSQL_COMPRESSION */ - -#ifdef MYSQL_ENCRYPTION -/** Check if the string is "" or "n". -@param[in] algorithm Encryption algorithm to check -@return true if no algorithm requested */ -bool -Encryption::is_none(const char* algorithm) -{ - /* NULL is the same as NONE */ - if (algorithm == NULL - || innobase_strcasecmp(algorithm, "n") == 0 - || innobase_strcasecmp(algorithm, "") == 0) { - return(true); - } - - return(false); -} - -/** Check the encryption option and set it -@param[in] option encryption option -@param[in/out] encryption The encryption algorithm -@return DB_SUCCESS or DB_UNSUPPORTED */ -dberr_t -Encryption::set_algorithm( - const char* option, - Encryption* encryption) -{ - if (is_none(option)) { - - encryption->m_type = NONE; - - } else if (innobase_strcasecmp(option, "y") == 0) { - - encryption->m_type = AES; - - } else { - return(DB_UNSUPPORTED); - } - - return(DB_SUCCESS); -} - -/** Check for supported ENCRYPT := (Y | N) supported values -@param[in] option Encryption option -@param[out] encryption The encryption algorithm -@return DB_SUCCESS or DB_UNSUPPORTED */ -dberr_t -Encryption::validate(const char* option) -{ - Encryption encryption; - - return(encryption.set_algorithm(option, &encryption)); -} -#endif /* MYSQL_ENCRYPTION */ - /*********************************************************************//** Compute the next autoinc value. @@ -3110,16 +2918,12 @@ check_trx_exists( if (trx == NULL) { trx = innobase_trx_allocate(thd); - innodb_session_t* innodb_session = thd_to_innodb_session(thd); - innodb_session->m_trx = trx; - /* User trx can be forced to rollback, so we unset the disable flag. */ ut_ad(trx->in_innodb & TRX_FORCE_ROLLBACK_DISABLE); trx->in_innodb &= TRX_FORCE_ROLLBACK_MASK; } else { ut_a(trx->magic_n == TRX_MAGIC_N); - innobase_trx_init(thd, trx); } @@ -3168,6 +2972,20 @@ innodb_replace_trx_in_thd( } #endif /* MYSQL_REPLACE_TRX_IN_THD */ +/************************************************************************* +Gets current trx. */ +trx_t* +innobase_get_trx() +{ + THD *thd=current_thd; + if (likely(thd != 0)) { + trx_t*& trx = thd_to_trx(thd); + return(trx); + } else { + return(NULL); + } +} + /*********************************************************************//** Note that a transaction has been registered with MySQL. @return true if transaction is registered with MySQL 2PC coordinator */ @@ -3313,6 +3131,7 @@ ha_innobase::ha_innobase( | HA_CAN_VIRTUAL_COLUMNS | HA_CAN_INDEX_BLOBS | HA_CAN_SQL_HANDLER + | HA_REQUIRES_KEY_COLUMNS_FOR_DELETE | HA_PRIMARY_KEY_REQUIRED_FOR_POSITION | HA_PRIMARY_KEY_IN_READ_INDEX | HA_BINLOG_ROW_CAPABLE @@ -3362,8 +3181,7 @@ ha_innobase::update_thd( TrxInInnoDB trx_in_innodb(trx); - ut_ad(dict_table_is_intrinsic(m_prebuilt->table) - || trx_in_innodb.is_aborted() + ut_ad(trx_in_innodb.is_aborted() || (trx->dict_operation_lock_mode == 0 && trx->dict_operation == TRX_DICT_OP_NONE)); @@ -3599,7 +3417,7 @@ innobase_invalidate_query_cache( TRUE); } -/** Quote a standard SQL identifier like tablespace, index or column name. +/** Quote a standard SQL identifier like index or column name. @param[in] file output stream @param[in] trx InnoDB transaction, or NULL @param[in] id identifier to quote */ @@ -3877,6 +3695,13 @@ innobase_space_shutdown() } srv_tmp_space.shutdown(); +#ifdef WITH_INNODB_DISALLOW_WRITES + if (srv_allow_writes_event) { + os_event_destroy(srv_allow_writes_event); + srv_allow_writes_event = NULL; + } +#endif /* WITH_INNODB_DISALLOW_WRITES */ + DBUG_VOID_RETURN; } @@ -3893,67 +3718,6 @@ innobase_init_abort() DBUG_RETURN(1); } - -#ifdef MYSQL_ENCRYPTION -/* mutex protecting the master_key_id */ -ib_mutex_t master_key_id_mutex; - -/** Rotate the encrypted tablespace keys according to master key -rotation. -@return false on success, true on failure */ -bool -innobase_encryption_key_rotation() -{ - byte* master_key = NULL; - bool ret = FALSE; - - /* Require the mutex to block other rotate request. */ - mutex_enter(&master_key_id_mutex); - - /* Check if keyring loaded and the currently master key - can be fetched. */ - if (Encryption::master_key_id != 0) { - ulint master_key_id; - Encryption::Version version; - - Encryption::get_master_key(&master_key_id, - &master_key, - &version); - if (master_key == NULL) { - mutex_exit(&master_key_id_mutex); - my_error(ER_CANNOT_FIND_KEY_IN_KEYRING, MYF(0)); - return(true); - } - my_free(master_key); - } - - master_key = NULL; - - /* Generate the new master key. */ - Encryption::create_master_key(&master_key); - - if (master_key == NULL) { - my_error(ER_CANNOT_FIND_KEY_IN_KEYRING, MYF(0)); - mutex_exit(&master_key_id_mutex); - return(true); - } - - ret = !fil_encryption_rotate(); - - my_free(master_key); - - /* If rotation failure, return error */ - if (ret) { - my_error(ER_CANNOT_FIND_KEY_IN_KEYRING, MYF(0)); - } - - /* Release the mutex. */ - mutex_exit(&master_key_id_mutex); - - return(ret); -} -#endif /* MYSQL_ENCRYPTION */ - /** Return partitioning flags. */ static uint innobase_partition_flags() { @@ -3985,16 +3749,48 @@ static const char* deprecated_file_format_check static const char* deprecated_file_format_max = DEPRECATED_FORMAT_PARAMETER("innodb_file_format_max"); +/** Deprecation message about innodb_use_trim */ +static const char* deprecated_use_trim + = DEPRECATED_FORMAT_PARAMETER("innodb_use_trim"); + /** Update log_checksum_algorithm_ptr with a pointer to the function corresponding to whether checksums are enabled. -@param[in] check whether redo log block checksums are enabled */ -static -void -innodb_log_checksums_func_update(bool check) +@param[in,out] thd client session, or NULL if at startup +@param[in] check whether redo log block checksums are enabled +@return whether redo log block checksums are enabled */ +static inline +bool +innodb_log_checksums_func_update(THD* thd, bool check) { - log_checksum_algorithm_ptr = check - ? log_block_calc_checksum_crc32 - : log_block_calc_checksum_none; + static const char msg[] = "innodb_encrypt_log implies" + " innodb_log_checksums"; + + ut_ad(!thd == !srv_was_started); + + if (!check) { + check = srv_encrypt_log; + if (!check) { + } else if (thd) { + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + HA_ERR_UNSUPPORTED, msg); + } else { + sql_print_warning(msg); + } + } + + if (thd) { + log_mutex_enter(); + log_checksum_algorithm_ptr = check + ? log_block_calc_checksum_crc32 + : log_block_calc_checksum_none; + log_mutex_exit(); + } else { + log_checksum_algorithm_ptr = check + ? log_block_calc_checksum_crc32 + : log_block_calc_checksum_none; + } + + return(check); } /****************************************************************//** @@ -4020,7 +3816,6 @@ innobase_init( uint format_id; ulong num_pll_degree; ulint srv_buf_pool_size_org = 0; - ulint fsp_flags =0; DBUG_ENTER("innobase_init"); handlerton* innobase_hton= (handlerton*) p; @@ -4047,17 +3842,8 @@ innobase_init( innobase_hton->commit_by_xid = innobase_commit_by_xid; innobase_hton->rollback_by_xid = innobase_rollback_by_xid; innobase_hton->commit_checkpoint_request=innobase_checkpoint_request; - -#ifdef INNOBASE_CURSOR_VIEW - innobase_hton->create_cursor_read_view = innobase_create_cursor_view; - innobase_hton->set_cursor_read_view = innobase_set_cursor_view; - innobase_hton->close_cursor_read_view = innobase_close_cursor_view; -#endif innobase_hton->create = innobase_create_handler; -#ifdef MYSQL_TABLESPACES - innobase_hton->alter_tablespace = innobase_alter_tablespace; -#endif innobase_hton->drop_database = innobase_drop_database; innobase_hton->panic = innobase_end; innobase_hton->partition_flags= innobase_partition_flags; @@ -4089,20 +3875,10 @@ innobase_init( innobase_hton->tablefile_extensions = ha_innobase_exts; } -#ifdef MYSQL_INNODB_API_CB - /* JAN: TODO: MySQL 5.7 */ - innobase_hton->data = &innodb_api_cb; -#endif - innobase_hton->table_options = innodb_table_option_list; innodb_remember_check_sysvar_funcs(); -#ifdef MYSQL_ENCRYPTION - innobase_hton->rotate_encryption_master_key = - innobase_encryption_key_rotation; -#endif - ut_a(DATA_MYSQL_TRUE_VARCHAR == (ulint)MYSQL_TYPE_VARCHAR); #ifndef DBUG_OFF @@ -4142,20 +3918,15 @@ innobase_init( any functions that could possibly allocate memory. */ ut_new_boot(); - if (UNIV_PAGE_SIZE != UNIV_PAGE_SIZE_DEF) { - fprintf(stderr, - "InnoDB: Warning: innodb_page_size has been " - "changed from default value %d to %ldd. (###EXPERIMENTAL### " - "operation)\n", UNIV_PAGE_SIZE_DEF, UNIV_PAGE_SIZE); - - /* There is hang on buffer pool when trying to get a new - page if buffer pool size is too small for large page sizes */ - if (innobase_buffer_pool_size < (24 * 1024 * 1024)) { - fprintf(stderr, "InnoDB: Error: innobase_page_size %lu requires " - "innodb_buffer_pool_size > 24M current %lld", - UNIV_PAGE_SIZE, innobase_buffer_pool_size); - goto error; - } + /* The buffer pool needs to be able to accommodate enough many + pages, even for larger pages */ + if (UNIV_PAGE_SIZE > UNIV_PAGE_SIZE_DEF + && innobase_buffer_pool_size < (24 * 1024 * 1024)) { + ib::info() << "innodb_page_size=" + << UNIV_PAGE_SIZE << " requires " + << "innodb_buffer_pool_size > 24M current " + << innobase_buffer_pool_size; + goto error; } #ifndef HAVE_LZ4 @@ -4230,7 +4001,6 @@ innobase_init( ut_a(default_path); fil_path_to_mysql_datadir = default_path; - folder_mysql_datadir = fil_path_to_mysql_datadir; /* Set InnoDB initialization parameters according to the values read from MySQL .cnf file */ @@ -4264,13 +4034,8 @@ innobase_init( page_size_t(srv_page_size, srv_page_size, false)); srv_sys_space.set_space_id(TRX_SYS_SPACE); - - /* Create the filespace flags. */ - fsp_flags = fsp_flags_init( - univ_page_size, false, false, false, false, false, 0, ATOMIC_WRITES_DEFAULT); - srv_sys_space.set_flags(fsp_flags); - - srv_sys_space.set_name(reserved_system_space_name); + srv_sys_space.set_flags(FSP_FLAGS_PAGE_SSIZE()); + srv_sys_space.set_name("innodb_system"); srv_sys_space.set_path(srv_data_home); /* Supports raw devices */ @@ -4288,13 +4053,9 @@ innobase_init( /* We set the temporary tablspace id later, after recovery. The temp tablespace doesn't support raw devices. Set the name and path. */ - srv_tmp_space.set_name(reserved_temporary_space_name); + srv_tmp_space.set_name("innodb_temporary"); srv_tmp_space.set_path(srv_data_home); - - /* Create the filespace flags with the temp flag set. */ - fsp_flags = fsp_flags_init( - univ_page_size, false, false, false, true, false, 0, ATOMIC_WRITES_DEFAULT); - srv_tmp_space.set_flags(fsp_flags); + srv_tmp_space.set_flags(FSP_FLAGS_PAGE_SSIZE()); if (!srv_tmp_space.parse_params(innobase_temp_data_file_path, false)) { DBUG_RETURN(innobase_init_abort()); @@ -4482,9 +4243,7 @@ innobase_change_buffering_inited_ok: srv_log_file_size = (ib_uint64_t) innobase_log_file_size; if (UNIV_PAGE_SIZE_DEF != srv_page_size) { - ib::warn() << "innodb-page-size has been changed from the" - " default value " << UNIV_PAGE_SIZE_DEF << " to " - << srv_page_size << "."; + ib::info() << "innodb_page_size=" << srv_page_size; } if (srv_log_write_ahead_size > srv_page_size) { @@ -4520,7 +4279,8 @@ innobase_change_buffering_inited_ok: srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_NONE; } - innodb_log_checksums_func_update(innodb_log_checksums); + innodb_log_checksums = innodb_log_checksums_func_update( + NULL, innodb_log_checksums); #ifdef HAVE_LINUX_LARGE_PAGES if ((os_use_large_pages = my_use_large_pages)) { @@ -4575,20 +4335,27 @@ innobase_change_buffering_inited_ok: data_mysql_default_charset_coll = (ulint) default_charset_info->number; innobase_commit_concurrency_init_default(); -#ifdef HAVE_POSIX_FALLOCATE - srv_use_posix_fallocate = (ibool) innobase_use_fallocate; -#endif - srv_use_atomic_writes = (ibool) innobase_use_atomic_writes; - if (innobase_use_atomic_writes) { - fprintf(stderr, "InnoDB: using atomic writes.\n"); - /* Force doublewrite buffer off, atomic writes replace it. */ - if (srv_use_doublewrite_buf) { - fprintf(stderr, "InnoDB: Switching off doublewrite buffer " - "because of atomic writes.\n"); - innobase_use_doublewrite = srv_use_doublewrite_buf = FALSE; - } - /* Force O_DIRECT on Unixes (on Windows writes are always unbuffered)*/ + if (innobase_use_fallocate) { + ib::warn() << "innodb_use_fallocate is DEPRECATED" + " and has no effect in MariaDB 10.2." + " It will be removed in MariaDB 10.3."; + } + + srv_use_atomic_writes = (ibool) (innobase_use_atomic_writes && + my_may_have_atomic_write); + if (srv_use_atomic_writes && !srv_file_per_table) + { + fprintf(stderr, "InnoDB: Disabling atomic_writes as file_per_table is not used.\n"); + srv_use_atomic_writes= 0; + } + + if (srv_use_atomic_writes) { + fprintf(stderr, "InnoDB: using atomic writes.\n"); + /* + Force O_DIRECT on Unixes (on Windows writes are always + unbuffered) + */ #ifndef _WIN32 if (!innobase_file_flush_method || !strstr(innobase_file_flush_method, "O_DIRECT")) { @@ -4597,13 +4364,6 @@ innobase_change_buffering_inited_ok: fprintf(stderr, "InnoDB: using O_DIRECT due to atomic writes.\n"); } #endif -#ifdef HAVE_POSIX_FALLOCATE - /* Due to a bug in directFS, using atomics needs - * posix_fallocate to extend the file - * pwrite() past end of the file won't work - */ - srv_use_posix_fallocate = TRUE; -#endif } #ifdef HAVE_PSI_INTERFACE @@ -4670,14 +4430,17 @@ innobase_change_buffering_inited_ok: } if (err != DB_SUCCESS) { + innodb_shutdown(); DBUG_RETURN(innobase_init_abort()); + } else if (!srv_read_only_mode) { + mysql_thread_create(thd_destructor_thread_key, + &thd_destructor_thread, + NULL, thd_destructor_proxy, NULL); + while (!srv_running) + os_thread_sleep(20); } -#ifdef MYSQL_ENCRYPTION - /* Create mutex to protect encryption master_key_id. */ - mutex_create(LATCH_ID_MASTER_KEY_ID_MUTEX, &master_key_id_mutex); -#endif - + srv_was_started = TRUE; /* Adjust the innodb_undo_logs config object */ innobase_undo_logs_init_default_max(); @@ -4696,7 +4459,6 @@ innobase_change_buffering_inited_ok: mysql_mutex_init(pending_checkpoint_mutex_key, &pending_checkpoint_mutex, MY_MUTEX_INIT_FAST); - innodb_inited= 1; #ifdef MYSQL_DYNAMIC_PLUGIN if (innobase_hton != p) { innobase_hton = reinterpret_cast<handlerton*>(p); @@ -4765,8 +4527,7 @@ innobase_end( DBUG_ENTER("innobase_end"); DBUG_ASSERT(hton == innodb_hton_ptr); - if (innodb_inited) { - + if (srv_was_started) { THD *thd= current_thd; if (thd) { // may be UNINSTALL PLUGIN statement trx_t* trx = thd_to_trx(thd); @@ -4775,19 +4536,20 @@ innobase_end( } } - srv_fast_shutdown = (ulint) innobase_fast_shutdown; - - innodb_inited = 0; hash_table_free(innobase_open_tables); innobase_open_tables = NULL; -#ifdef MYSQL_ENCRYPTION - mutex_free(&master_key_id_mutex); -#endif - if (innobase_shutdown_for_mysql() != DB_SUCCESS) { - err = 1; + if (!abort_loop && srv_running) { + // may be UNINSTALL PLUGIN statement + srv_running->abort = 1; + mysql_cond_broadcast(srv_running->current_cond); + } + + if (!srv_read_only_mode) { + pthread_join(thd_destructor_thread, NULL); } + innodb_shutdown(); innobase_space_shutdown(); mysql_mutex_destroy(&innobase_share_mutex); @@ -4809,7 +4571,7 @@ innobase_commit_low( #ifdef WITH_WSREP THD* thd = (THD*)trx->mysql_thd; const char* tmp = 0; - if (wsrep_on(thd)) { + if (thd && wsrep_on(thd)) { #ifdef WSREP_PROC_INFO char info[64]; info[sizeof(info) - 1] = '\0'; @@ -5062,7 +4824,7 @@ innobase_commit( if (!trx_is_registered_for_2pc(trx) && trx_is_started(trx)) { - sql_print_error("Transaction not registered for MySQL 2PC," + sql_print_error("Transaction not registered for MariaDB 2PC," " but transaction is active"); } @@ -5179,13 +4941,13 @@ innobase_rollback( error = trx_rollback_for_mysql(trx); if (trx->state == TRX_STATE_FORCED_ROLLBACK) { -#ifdef UNIV_DEBUG +#ifndef DBUG_OFF char buffer[1024]; - ib::info() << "Forced rollback : " - << thd_get_error_context_description(thd, - buffer, sizeof(buffer), 512); -#endif /* UNIV_DEBUG */ + DBUG_LOG("trx", "Forced rollback: " + << thd_get_error_context_description( + thd, buffer, sizeof buffer, 512)); +#endif /* !DBUG_OFF */ trx->state = TRX_STATE_NOT_STARTED; } @@ -5576,7 +5338,7 @@ innobase_close_connection( if (!trx_is_registered_for_2pc(trx) && trx_is_started(trx)) { - sql_print_error("Transaction not registered for MySQL 2PC, " + sql_print_error("Transaction not registered for MariaDB 2PC, " "but transaction is active"); } @@ -5597,7 +5359,7 @@ innobase_close_connection( } } else { sql_print_warning( - "MySQL is closing a connection that has an active " + "MariaDB is closing a connection that has an active " "InnoDB transaction. " TRX_ID_FMT " row modifications " "will roll back.", " row modifications will roll back.", @@ -5620,10 +5382,6 @@ innobase_close_connection( trx_free_for_mysql(trx); } - UT_DELETE(thd_to_innodb_session(thd)); - - thd_to_innodb_session(thd) = NULL; - DBUG_RETURN(0); } @@ -5665,9 +5423,6 @@ innobase_kill_query( /* Cancel a pending lock request if there are any */ bool lock_mutex_taken = false; bool trx_mutex_taken = false; - bool already_have_lock_mutex = false; - bool already_have_trx_mutex = false; - dberr_t err = DB_SUCCESS; if (trx->lock.wait_lock) { WSREP_DEBUG("Killing victim trx %p BF %d trx BF %d trx_id " IB_ID_FMT " ABORT %d thd %p" @@ -5681,38 +5436,24 @@ innobase_kill_query( } if (!wsrep_thd_is_BF(trx->mysql_thd, FALSE)) { - ut_ad(!lock_mutex_own()); lock_mutex_enter(); lock_mutex_taken = true; - } else { - already_have_lock_mutex = true; } if (trx->abort_type != TRX_WSREP_ABORT) { - ut_ad(!trx_mutex_own(trx)); trx_mutex_enter(trx); trx_mutex_taken = true; - } else { - already_have_trx_mutex = true; } - err = lock_trx_handle_wait(trx, - (lock_mutex_taken || already_have_lock_mutex), - (trx_mutex_taken || already_have_trx_mutex)); + lock_trx_handle_wait(trx, true, true); if (lock_mutex_taken) { - ut_ad(lock_mutex_own()); lock_mutex_exit(); } if (trx_mutex_taken) { - ut_ad(trx_mutex_own(trx)); trx_mutex_exit(trx); } - - if (err != DB_SUCCESS && err != DB_LOCK_WAIT) { - ib::warn() << "Killing connection failed " << ut_strerr(err) << "("<<err<<")"; - } } DBUG_VOID_RETURN; @@ -5768,7 +5509,7 @@ ha_innobase::table_flags() const all InnoDB features such as GEOMETRY, FULLTEXT etc. */ /* JAN: TODO: MySQL 5.7 flags &= ~(HA_INNOPART_DISABLED_TABLE_FLAGS); - + } */ } @@ -5922,6 +5663,36 @@ ha_innobase::keys_to_use_for_scanning() } /****************************************************************//** +Ensures that if there's a concurrent inplace ADD INDEX, being-indexed virtual +columns are computed. They are not marked as indexed in the old table, so the +server won't add them to the vcol_set automatically */ +void +ha_innobase::column_bitmaps_signal() +/*================================*/ +{ + if (!table->vfield || table->current_lock != F_WRLCK) { + return; + } + + dict_index_t* clust_index = dict_table_get_first_index(m_prebuilt->table); + uint num_v = 0; + for (uint j = 0; j < table->s->virtual_fields; j++) { + if (table->vfield[j]->stored_in_db()) { + continue; + } + + dict_col_t* col = &m_prebuilt->table->v_cols[num_v].m_col; + if (col->ord_part || + (dict_index_is_online_ddl(clust_index) && + row_log_col_is_indexed(clust_index, num_v))) { + table->mark_virtual_col(table->vfield[j]); + } + num_v++; + } +} + + +/****************************************************************//** Determines if table caching is supported. @return HA_CACHE_TBL_ASKTRANSACT */ @@ -6208,8 +5979,6 @@ innobase_match_index_columns( if (innodb_idx_fld >= innodb_idx_fld_end) { DBUG_RETURN(FALSE); } - - mtype = innodb_idx_fld->col->mtype; } /* MariaDB-5.5 compatibility */ @@ -6253,7 +6022,6 @@ innobase_match_index_columns( DBUG_RETURN(TRUE); } -#ifdef MYSQL_VIRTUAL_COLUMNS /** Build a template for a base column for a virtual column @param[in] table MySQL TABLE @param[in] clust_index InnoDB clustered index @@ -6313,21 +6081,6 @@ innobase_vcol_build_templ( templ->is_unsigned = col->prtype & DATA_UNSIGNED; } -/** callback used by MySQL server layer to initialize -the table virtual columns' template -@param[in] table MySQL TABLE -@param[in,out] ib_table InnoDB table */ -void -innobase_build_v_templ_callback( - const TABLE* table, - void* ib_table) -{ - const dict_table_t* t_table = static_cast<dict_table_t*>(ib_table); - - innobase_build_v_templ(table, t_table, t_table->vc_templ, NULL, - true, NULL); -} - /** Build template for the virtual columns and their base columns. This is done when the table first opened. @param[in] table MySQL TABLE @@ -6335,16 +6088,14 @@ is done when the table first opened. @param[in,out] s_templ InnoDB template structure @param[in] add_v new virtual columns added along with add index call -@param[in] locked true if dict_sys mutex is held -@param[in] share_tbl_name original MySQL table name */ +@param[in] locked true if dict_sys mutex is held */ void innobase_build_v_templ( const TABLE* table, const dict_table_t* ib_table, dict_vcol_templ_t* s_templ, const dict_add_v_col_t* add_v, - bool locked, - const char* share_tbl_name) + bool locked) { ulint ncol = ib_table->n_cols - DATA_N_SYS_COLS; ulint n_v_col = ib_table->n_v_cols; @@ -6376,15 +6127,8 @@ innobase_build_v_templ( * sizeof *s_templ->vtempl)); s_templ->n_col = ncol; s_templ->n_v_col = n_v_col; - s_templ->rec_len = table->s->stored_rec_length; - // JAN: MySQL 5.6 - // s_templ->default_rec = table->s->default_values; - - s_templ->default_rec = static_cast<byte*>( - ut_malloc_nokey(table->s->stored_rec_length)); - memcpy(s_templ->default_rec, table->s->default_values, - table->s->stored_rec_length); - + s_templ->rec_len = table->s->reclength; + s_templ->default_rec = table->s->default_values; /* Mark those columns could be base columns */ for (ulint i = 0; i < ib_table->n_v_cols; i++) { @@ -6484,12 +6228,7 @@ innobase_build_v_templ( s_templ->db_name = table->s->db.str; s_templ->tb_name = table->s->table_name.str; - - if (share_tbl_name) { - s_templ->share_name = share_tbl_name; - } } -#endif /* MYSQL_VIRTUAL_COLUMNS */ /*******************************************************************//** This function builds a translation table in INNOBASE_SHARE @@ -6586,7 +6325,7 @@ innobase_build_index_translation( if (!innobase_match_index_columns(&table->key_info[count], index_mapping[count])) { sql_print_error("Found index %s whose column info" - " does not match that of MySQL.", + " does not match that of MariaDB.", table->key_info[count].name); ret = false; goto func_exit; @@ -6699,33 +6438,35 @@ innobase_get_int_col_max_value( return(max_value); } -/************************************************************************ -Set the autoinc column max value. This should only be called once from -ha_innobase::open(). Therefore there's no need for a covering lock. */ +/** Initialize the AUTO_INCREMENT column metadata. + +Since a partial table definition for a persistent table can already be +present in the InnoDB dict_sys cache before it is accessed from SQL, +we have to initialize the AUTO_INCREMENT counter on the first +ha_innobase::open(). + +@param[in,out] table persistent table +@param[in] field the AUTO_INCREMENT column */ +static void -ha_innobase::innobase_initialize_autoinc() -/*======================================*/ +initialize_auto_increment(dict_table_t* table, const Field* field) { - ulonglong auto_inc; - const Field* field = table->found_next_number_field; + ut_ad(!dict_table_is_temporary(table)); - if (field != NULL) { - auto_inc = innobase_get_int_col_max_value(field); - ut_ad(!innobase_is_v_fld(field)); + const unsigned col_no = innodb_col_no(field); - /* autoinc column cannot be virtual column */ - ut_ad(!innobase_is_v_fld(field)); - } else { - /* We have no idea what's been passed in to us as the - autoinc column. We set it to the 0, effectively disabling - updates to the table. */ - auto_inc = 0; + dict_table_autoinc_lock(table); - ib::info() << "Unable to determine the AUTOINC column name"; - } + table->persistent_autoinc = 1 + + dict_table_get_nth_col_pos(table, col_no, NULL); - if (srv_force_recovery >= SRV_FORCE_NO_IBUF_MERGE) { + if (table->autoinc) { + /* Already initialized. Our caller checked + table->persistent_autoinc without + dict_table_autoinc_lock(), and there might be multiple + ha_innobase::open() executing concurrently. */ + } else if (srv_force_recovery >= SRV_FORCE_NO_IBUF_MERGE) { /* If the recovery level is set so high that writes are disabled we force the AUTOINC counter to 0 value effectively disabling writes to the table. @@ -6736,82 +6477,16 @@ ha_innobase::innobase_initialize_autoinc() tables can be dumped with minimal hassle. If an error were returned in this case, the first attempt to read the table would fail and subsequent SELECTs would succeed. */ - auto_inc = 0; - } else if (field == NULL) { - /* This is a far more serious error, best to avoid - opening the table and return failure. */ - my_error(ER_AUTOINC_READ_FAILED, MYF(0)); - } else { - dict_index_t* index; - const char* col_name; - ib_uint64_t read_auto_inc; - ulint err; - - update_thd(ha_thd()); - - col_name = field->field_name; - - /* For intrinsic table, name of field has to be prefixed with - table name to maintain column-name uniqueness. */ - if (m_prebuilt->table != NULL - && dict_table_is_intrinsic(m_prebuilt->table)) { - - ulint col_no = dict_col_get_no(dict_table_get_nth_col( - m_prebuilt->table, field->field_index)); - - col_name = dict_table_get_col_name( - m_prebuilt->table, col_no); - } - - index = innobase_get_index(table->s->next_number_index); - - /* Execute SELECT MAX(col_name) FROM TABLE; */ - err = row_search_max_autoinc(index, col_name, &read_auto_inc); - - switch (err) { - case DB_SUCCESS: { - ulonglong col_max_value; - - col_max_value = innobase_get_int_col_max_value(field); - - /* At the this stage we do not know the increment - nor the offset, so use a default increment of 1. */ - - auto_inc = innobase_next_autoinc( - read_auto_inc, 1, 1, 0, col_max_value); - - break; - } - case DB_RECORD_NOT_FOUND: - ib::error() << "MySQL and InnoDB data dictionaries are" - " out of sync. Unable to find the AUTOINC" - " column " << col_name << " in the InnoDB" - " table " << index->table->name << ". We set" - " the next AUTOINC column value to 0, in" - " effect disabling the AUTOINC next value" - " generation."; - - ib::info() << "You can either set the next AUTOINC" - " value explicitly using ALTER TABLE or fix" - " the data dictionary by recreating the" - " table."; - - /* This will disable the AUTOINC generation. */ - auto_inc = 0; - - /* We want the open to succeed, so that the user can - take corrective action. ie. reads should succeed but - updates should fail. */ - err = DB_SUCCESS; - break; - default: - /* row_search_max_autoinc() should only return - one of DB_SUCCESS or DB_RECORD_NOT_FOUND. */ - ut_error; - } + } else if (table->persistent_autoinc) { + table->autoinc = innobase_next_autoinc( + btr_read_autoinc_with_fallback(table, col_no), + 1 /* need */, + 1 /* auto_increment_increment */, + 0 /* auto_increment_offset */, + innobase_get_int_col_max_value(field)); } - dict_table_autoinc_initialize(m_prebuilt->table, auto_inc); + dict_table_autoinc_unlock(table); } /*****************************************************************//** @@ -6829,7 +6504,6 @@ ha_innobase::open( dict_table_t* ib_table; char norm_name[FN_REFLEN]; THD* thd; - char* is_part = NULL; dict_err_ignore_t ignore_err = DICT_ERR_IGNORE_NONE; DBUG_ENTER("ha_innobase::open"); @@ -6859,13 +6533,7 @@ ha_innobase::open( m_upd_buf = NULL; m_upd_buf_size = 0; - /* We look for pattern #P# to see if the table is partitioned - MySQL table. */ -#ifdef _WIN32 - is_part = strstr(norm_name, "#p#"); -#else - is_part = strstr(norm_name, "#P#"); -#endif /* _WIN32 */ + char* is_part = is_partition(norm_name); /* Check whether FOREIGN_KEY_CHECKS is set to 0. If so, the table can be opened even if some FK indexes are missing. If not, the table @@ -6874,30 +6542,19 @@ ha_innobase::open( ignore_err = DICT_ERR_IGNORE_FK_NOKEY; } - /* Get pointer to a table object in InnoDB dictionary cache. - For intrinsic table, get it from session private data */ - ib_table = thd_to_innodb_session(thd)->lookup_table_handler(norm_name); - - if (ib_table == NULL) { - - ib_table = open_dict_table(name, norm_name, is_part, - ignore_err); - } else { - ib_table->acquire(); - ut_ad(dict_table_is_intrinsic(ib_table)); - } + ib_table = open_dict_table(name, norm_name, is_part, ignore_err); if (ib_table != NULL && ((!DICT_TF2_FLAG_IS_SET(ib_table, DICT_TF2_FTS_HAS_DOC_ID) - && table->s->stored_fields != dict_table_get_n_tot_u_cols(ib_table)) + && table->s->fields != dict_table_get_n_tot_u_cols(ib_table)) || (DICT_TF2_FLAG_IS_SET(ib_table, DICT_TF2_FTS_HAS_DOC_ID) - && (table->s->stored_fields + && (table->s->fields != dict_table_get_n_tot_u_cols(ib_table) - 1)))) { ib::warn() << "Table " << norm_name << " contains " << dict_table_get_n_user_cols(ib_table) << " user" - " defined columns in InnoDB, but " << table->s->stored_fields - << " columns in MySQL. Please check" + " defined columns in InnoDB, but " << table->s->fields + << " columns in MariaDB. Please check" " INFORMATION_SCHEMA.INNODB_SYS_COLUMNS and " REFMAN "innodb-troubleshooting.html for how to resolve the" " issue."; @@ -6910,28 +6567,6 @@ ha_innobase::open( is_part = NULL; } -#ifdef MYSQL_ENCRYPTION - /* For encrypted table, check if the encryption info in data - file can't be retrieved properly, mark it as corrupted. */ - if (ib_table != NULL - && dict_table_is_encrypted(ib_table) - && ib_table->ibd_file_missing - && !dict_table_is_discarded(ib_table)) { - - /* Mark this table as corrupted, so the drop table - or force recovery can still use it, but not others. */ - - dict_table_close(ib_table, FALSE, FALSE); - ib_table = NULL; - is_part = NULL; - - free_share(m_share); - my_error(ER_CANNOT_FIND_KEY_IN_KEYRING, MYF(0)); - - DBUG_RETURN(HA_ERR_TABLE_CORRUPT); - } -#endif - if (NULL == ib_table) { if (is_part) { @@ -7008,9 +6643,7 @@ ha_innobase::open( bool warning_pushed = false; fil_space_crypt_t* crypt_data = ib_table->crypt_data; - if ((crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) || - (srv_encrypt_tables && - crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { + if (crypt_data && crypt_data->should_encrypt()) { if (!encryption_key_id_exists(crypt_data->key_id)) { push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, @@ -7042,7 +6675,7 @@ ha_innobase::open( DBUG_RETURN(ret_err); } - m_prebuilt = row_create_prebuilt(ib_table, table->s->stored_rec_length); + m_prebuilt = row_create_prebuilt(ib_table, table->s->reclength); m_prebuilt->default_rec = table->s->default_values; ut_ad(m_prebuilt->default_rec); @@ -7054,12 +6687,10 @@ ha_innobase::open( key_used_on_scan = m_primary_key; -#ifdef MYSQL_VIRTUAL_COLUMNS if (ib_table->n_v_cols) { mutex_enter(&dict_sys->mutex); if (ib_table->vc_templ == NULL) { ib_table->vc_templ = UT_NEW_NOKEY(dict_vcol_templ_t()); - ib_table->vc_templ->vtempl = NULL; } else if (ib_table->get_ref_count() == 1) { /* Clean and refresh the template if no one else get hold on it */ @@ -7070,12 +6701,11 @@ ha_innobase::open( if (ib_table->vc_templ->vtempl == NULL) { innobase_build_v_templ( table, ib_table, ib_table->vc_templ, NULL, - true, m_share->table_name); + true); } mutex_exit(&dict_sys->mutex); } -#endif /* MYSQL_VIRTUAL_COLUMNS */ if (!innobase_build_index_translation(table, ib_table, m_share)) { sql_print_error("Build InnoDB index translation table for" @@ -7192,22 +6822,12 @@ ha_innobase::open( dict_table_get_format(m_prebuilt->table)); } - /* Only if the table has an AUTOINC column. */ - if (m_prebuilt->table != NULL - && !m_prebuilt->table->ibd_file_missing - && table->found_next_number_field != NULL) { - dict_table_autoinc_lock(m_prebuilt->table); - - /* Since a table can already be "open" in InnoDB's internal - data dictionary, we only init the autoinc counter once, the - first time the table is loaded. We can safely reuse the - autoinc value from a previous MySQL open. */ - if (dict_table_autoinc_read(m_prebuilt->table) == 0) { - - innobase_initialize_autoinc(); - } - - dict_table_autoinc_unlock(m_prebuilt->table); + if (m_prebuilt->table == NULL + || dict_table_is_temporary(m_prebuilt->table) + || m_prebuilt->table->persistent_autoinc + || m_prebuilt->table->ibd_file_missing) { + } else if (const Field* ai = table->found_next_number_field) { + initialize_auto_increment(m_prebuilt->table, ai); } /* Set plugin parser for fulltext index */ @@ -7232,35 +6852,25 @@ ha_innobase::open( } info(HA_STATUS_NO_LOCK | HA_STATUS_VARIABLE | HA_STATUS_CONST); + DBUG_RETURN(0); +} -#ifdef MYSQL_COMPRESSION - dberr_t err = fil_set_compression(m_prebuilt->table, - table->s->compress.str); - - switch (err) { - case DB_NOT_FOUND: - case DB_UNSUPPORTED: - /* We will do another check before the create - table and push the error to the client there. */ - break; - - case DB_IO_NO_PUNCH_HOLE_TABLESPACE: - /* We did the check in the 'if' above. */ - - case DB_IO_NO_PUNCH_HOLE_FS: - /* During open we can't check whether the FS supports - punch hole or not, at least on Linux. */ - break; - - default: - ut_error; - - case DB_SUCCESS: - break; +/** Convert MySQL column number to dict_table_t::cols[] offset. +@param[in] field non-virtual column +@return column number relative to dict_table_t::cols[] */ +unsigned +innodb_col_no(const Field* field) +{ + ut_ad(!innobase_is_s_fld(field)); + const TABLE* table = field->table; + unsigned col_no = 0; + ut_ad(field == table->field[field->field_index]); + for (unsigned i = 0; i < field->field_index; i++) { + if (table->field[i]->stored_in_db()) { + col_no++; + } } -#endif - - DBUG_RETURN(0); + return(col_no); } /** Opens dictionary table object using table name. For partition, we need to @@ -7423,19 +7033,6 @@ ha_innobase::close() /* The following accessor functions should really be inside MySQL code! */ -/** Gets field offset for a field in a table. -@param[in] table MySQL table object -@param[in] field MySQL field object -@return offset */ -static inline -uint -get_field_offset( - const TABLE* table, - const Field* field) -{ - return(static_cast<uint>((field->ptr - table->record[0]))); -} - #ifdef WITH_WSREP UNIV_INTERN int @@ -7486,7 +7083,7 @@ wsrep_innobase_mysql_sort( if (charset == NULL) { sql_print_error("InnoDB needs charset %lu for doing " - "a comparison, but MySQL cannot " + "a comparison, but MariaDB cannot " "find that charset.", (ulong) charset_number); ut_a(0); @@ -7786,8 +7383,7 @@ get_innobase_type_from_mysql_type( case MYSQL_TYPE_VARCHAR: /* new >= 5.0.3 true VARCHAR */ if (field->binary()) { return(DATA_BINARY); - } else if (strcmp(field->charset()->name, - "latin1_swedish_ci") == 0) { + } else if (field->charset() == &my_charset_latin1) { return(DATA_VARCHAR); } else { return(DATA_VARMYSQL); @@ -7795,10 +7391,8 @@ get_innobase_type_from_mysql_type( case MYSQL_TYPE_BIT: case MYSQL_TYPE_STRING: if (field->binary()) { - return(DATA_FIXBINARY); - } else if (strcmp(field->charset()->name, - "latin1_swedish_ci") == 0) { + } else if (field->charset() == &my_charset_latin1) { return(DATA_CHAR); } else { return(DATA_MYSQL); @@ -8189,10 +7783,9 @@ build_template_needs_field( dict_index_t* index, /*!< in: InnoDB index to use */ const TABLE* table, /*!< in: MySQL table object */ ulint i, /*!< in: field index in InnoDB table */ - ulint sql_idx, /*!< in: field index in SQL table */ ulint num_v) /*!< in: num virtual column so far */ { - const Field* field = table->field[sql_idx]; + const Field* field = table->field[i]; if (!index_contains) { if (read_just_key) { @@ -8207,8 +7800,8 @@ build_template_needs_field( return(field); } - if (bitmap_is_set(table->read_set, static_cast<uint>(sql_idx)) - || bitmap_is_set(table->write_set, static_cast<uint>(sql_idx))) { + if (bitmap_is_set(table->read_set, static_cast<uint>(i)) + || bitmap_is_set(table->write_set, static_cast<uint>(i))) { /* This field is needed in the query */ return(field); @@ -8284,7 +7877,62 @@ build_template_field( templ->col_no = i; templ->clust_rec_field_no = dict_col_get_clust_pos( col, clust_index); - ut_a(templ->clust_rec_field_no != ULINT_UNDEFINED); + /* If clustered index record field is not found, lets print out + field names and all the rest to understand why field is not found. */ + if (templ->clust_rec_field_no == ULINT_UNDEFINED) { + const char* tb_col_name = dict_table_get_col_name(clust_index->table, i); + dict_field_t* field=NULL; + size_t size = 0; + + for(ulint j=0; j < clust_index->n_user_defined_cols; j++) { + dict_field_t* ifield = &(clust_index->fields[j]); + if (ifield && !memcmp(tb_col_name, ifield->name, + strlen(tb_col_name))) { + field = ifield; + break; + } + } + + ib::info() << "Looking for field " << i << " name " + << (tb_col_name ? tb_col_name : "NULL") + << " from table " << clust_index->table->name; + + + for(ulint j=0; j < clust_index->n_user_defined_cols; j++) { + dict_field_t* ifield = &(clust_index->fields[j]); + ib::info() << "InnoDB Table " + << clust_index->table->name + << "field " << j << " name " + << (ifield ? ifield->name() : "NULL"); + } + + for(ulint j=0; j < table->s->stored_fields; j++) { + ib::info() << "MySQL table " + << table->s->table_name.str + << " field " << j << " name " + << table->field[j]->field_name; + } + + ib::error() << "Clustered record field for column " << i + << " not found table n_user_defined " + << clust_index->n_user_defined_cols + << " index n_user_defined " + << clust_index->table->n_cols - DATA_N_SYS_COLS + << " InnoDB table " + << clust_index->table->name + << " field name " + << (field ? field->name() : "NULL") + << " MySQL table " + << table->s->table_name.str + << " field name " + << (tb_col_name ? tb_col_name : "NULL") + << " n_fields " + << table->s->stored_fields + << " query " + << innobase_get_stmt_unsafe(current_thd, &size); + + ut_a(templ->clust_rec_field_no != ULINT_UNDEFINED); + } templ->rec_field_is_prefix = FALSE; templ->rec_prefix_field_no = ULINT_UNDEFINED; @@ -8390,10 +8038,10 @@ ha_innobase::build_template( { dict_index_t* index; dict_index_t* clust_index; - ulint n_stored_fields; + ulint n_fields; ibool fetch_all_in_key = FALSE; ibool fetch_primary_key_cols = FALSE; - ulint i, sql_idx; + ulint i; if (m_prebuilt->select_lock_type == LOCK_X) { /* We always retrieve the whole clustered index record if we @@ -8446,12 +8094,11 @@ ha_innobase::build_template( /* Below we check column by column if we need to access the clustered index. */ - /* number of stored columns */ - n_stored_fields= (ulint)table->s->stored_fields; + n_fields = (ulint) table->s->fields; /* number of columns */ if (!m_prebuilt->mysql_template) { m_prebuilt->mysql_template = (mysql_row_templ_t*) - ut_malloc_nokey(n_stored_fields * sizeof(mysql_row_templ_t)); + ut_malloc_nokey(n_fields * sizeof(mysql_row_templ_t)); } m_prebuilt->template_type = whole_row @@ -8473,16 +8120,17 @@ ha_innobase::build_template( ulint num_v = 0; /* Push down an index condition or an end_range check. */ - for (i = 0, sql_idx = 0; i < n_stored_fields; i++, sql_idx++) { + for (i = 0; i < n_fields; i++) { ibool index_contains; - while (!table->field[sql_idx]->stored_in_db()) { - sql_idx++; - } - - if (innobase_is_v_fld(table->field[sql_idx])) { + if (innobase_is_v_fld(table->field[i])) { index_contains = dict_index_contains_col_or_prefix( index, num_v, true); + if (index_contains) + { + m_prebuilt->n_template = 0; + goto no_icp; + } } else { index_contains = dict_index_contains_col_or_prefix( index, i - num_v, false); @@ -8503,8 +8151,7 @@ ha_innobase::build_template( the subset field->part_of_key.is_set(active_index) which would be acceptable if end_range==NULL. */ - bool is_v = innobase_is_v_fld(table->field[sql_idx]); - + bool is_v = innobase_is_v_fld(table->field[i]); if (build_template_needs_field_in_icp( index, m_prebuilt, index_contains, is_v ? num_v : i - num_v, is_v)) { @@ -8513,17 +8160,17 @@ ha_innobase::build_template( mysql_row_templ_t* templ; if (whole_row) { - field = table->field[sql_idx]; + field = table->field[i]; } else { field = build_template_needs_field( index_contains, m_prebuilt->read_just_key, fetch_all_in_key, fetch_primary_key_cols, - index, table, i, sql_idx, num_v); + index, table, i, num_v); if (!field) { if (innobase_is_v_fld( - table->field[sql_idx])) { + table->field[i])) { num_v++; } continue; @@ -8605,7 +8252,7 @@ ha_innobase::build_template( < m_prebuilt->index->n_uniq); */ } - if (innobase_is_v_fld(table->field[sql_idx])) { + if (innobase_is_v_fld(table->field[i])) { num_v++; } } @@ -8617,15 +8264,11 @@ ha_innobase::build_template( /* Include the fields that are not needed in index condition pushdown. */ - for (i = 0, sql_idx = 0; i < n_stored_fields; i++, sql_idx++) { + for (i = 0; i < n_fields; i++) { mysql_row_templ_t* templ; ibool index_contains; - while (!table->field[sql_idx]->stored_in_db()) { - sql_idx++; - } - - if (innobase_is_v_fld(table->field[sql_idx])) { + if (innobase_is_v_fld(table->field[i])) { index_contains = dict_index_contains_col_or_prefix( index, num_v, true); } else { @@ -8633,7 +8276,7 @@ ha_innobase::build_template( index, i - num_v, false); } - bool is_v = innobase_is_v_fld(table->field[sql_idx]); + bool is_v = innobase_is_v_fld(table->field[i]); if (!build_template_needs_field_in_icp( index, m_prebuilt, index_contains, @@ -8642,16 +8285,16 @@ ha_innobase::build_template( const Field* field; if (whole_row) { - field = table->field[sql_idx]; + field = table->field[i]; } else { field = build_template_needs_field( index_contains, m_prebuilt->read_just_key, fetch_all_in_key, fetch_primary_key_cols, - index, table, i, sql_idx, num_v); + index, table, i, num_v); if (!field) { - if (innobase_is_v_fld(table->field[sql_idx])) { + if (innobase_is_v_fld(table->field[i])) { num_v++; } continue; @@ -8670,18 +8313,15 @@ ha_innobase::build_template( m_prebuilt->idx_cond = this; } else { +no_icp: mysql_row_templ_t* templ; ulint num_v = 0; /* No index condition pushdown */ m_prebuilt->idx_cond = NULL; - for (i = 0, sql_idx = 0; i < n_stored_fields; i++, sql_idx++) { + for (i = 0; i < n_fields; i++) { const Field* field; - while (!table->field[sql_idx]->stored_in_db()) { - sql_idx++; - } - if (whole_row) { /* Even this is whole_row, if the seach is on a virtual column, and read_just_key is @@ -8689,7 +8329,7 @@ ha_innobase::build_template( will not try to fill the value since they are not stored in such index nor in the cluster index. */ - if (innobase_is_v_fld(table->field[sql_idx]) + if (innobase_is_v_fld(table->field[i]) && m_prebuilt->read_just_key && !dict_index_contains_col_or_prefix( m_prebuilt->index, num_v, true)) @@ -8701,11 +8341,11 @@ ha_innobase::build_template( continue; } - field = table->field[sql_idx]; + field = table->field[i]; } else { ibool contain; - if (innobase_is_v_fld(table->field[sql_idx])) { + if (innobase_is_v_fld(table->field[i])) { contain = dict_index_contains_col_or_prefix( index, num_v, true); } else { @@ -8714,16 +8354,14 @@ ha_innobase::build_template( false); } - field = build_template_needs_field( contain, m_prebuilt->read_just_key, fetch_all_in_key, fetch_primary_key_cols, - index, table, i, sql_idx, num_v); - + index, table, i, num_v); if (!field) { - if (innobase_is_v_fld(table->field[sql_idx])) { + if (innobase_is_v_fld(table->field[i])) { num_v++; } continue; @@ -8766,18 +8404,10 @@ ha_innobase::innobase_lock_autoinc(void) { DBUG_ENTER("ha_innobase::innobase_lock_autoinc"); dberr_t error = DB_SUCCESS; - long lock_mode = innobase_autoinc_lock_mode; - ut_ad(!srv_read_only_mode - || dict_table_is_intrinsic(m_prebuilt->table)); - - if (dict_table_is_intrinsic(m_prebuilt->table)) { - /* Intrinsic table are not shared accorss connection - so there is no need to AUTOINC lock the table. */ - lock_mode = AUTOINC_NO_LOCKING; - } + ut_ad(!srv_read_only_mode); - switch (lock_mode) { + switch (innobase_autoinc_lock_mode) { case AUTOINC_NO_LOCKING: /* Acquire only the AUTOINC mutex. */ dict_table_autoinc_lock(m_prebuilt->table); @@ -8853,30 +8483,6 @@ ha_innobase::innobase_set_max_autoinc( return(error); } -/** Write Row interface optimized for intrinisc table. -@param[in] record a row in MySQL format. -@return 0 on success or error code */ -int -ha_innobase::intrinsic_table_write_row(uchar* record) -{ - dberr_t err; - - /* No auto-increment support for intrinsic table. */ - ut_ad(!(table->next_number_field && record == table->record[0])); - - if (m_prebuilt->mysql_template == NULL - || m_prebuilt->template_type != ROW_MYSQL_WHOLE_ROW) { - /* Build the template used in converting quickly between - the two database formats */ - build_template(true); - } - - err = row_insert_for_mysql((byte*) record, m_prebuilt); - - return(convert_error_code_to_mysql( - err, m_prebuilt->table->flags, m_user_thd)); -} - /********************************************************************//** Stores a row in an InnoDB database, to the table specified in this handle. @@ -8897,15 +8503,10 @@ ha_innobase::write_row( DBUG_ENTER("ha_innobase::write_row"); - if (dict_table_is_intrinsic(m_prebuilt->table)) { - DBUG_RETURN(intrinsic_table_write_row(record)); - } - trx_t* trx = thd_to_trx(m_user_thd); TrxInInnoDB trx_in_innodb(trx); - if (!dict_table_is_intrinsic(m_prebuilt->table) - && trx_in_innodb.is_aborted()) { + if (trx_in_innodb.is_aborted()) { innobase_rollback(ht, m_user_thd, false); @@ -9310,28 +8911,32 @@ innodb_fill_old_vcol_val( return(buf); } -/**********************************************************************//** -Checks which fields have changed in a row and stores information -of them to an update vector. +/** Calculate an update vector corresponding to the changes +between old_row and new_row. +@param[out] uvect update vector +@param[in] old_row current row in MySQL format +@param[in] new_row intended updated row in MySQL format +@param[in] table MySQL table handle +@param[in,out] upd_buff buffer to use for converted values +@param[in] buff_len length of upd_buff +@param[in,out] prebuilt InnoDB execution context +@param[out] auto_inc updated AUTO_INCREMENT value, or 0 if none @return DB_SUCCESS or error code */ static dberr_t calc_row_difference( -/*================*/ - upd_t* uvect, /*!< in/out: update vector */ - const uchar* old_row, /*!< in: old row in MySQL format */ - uchar* new_row, /*!< in: new row in MySQL format */ - TABLE* table, /*!< in: table in MySQL data - dictionary */ - uchar* upd_buff, /*!< in: buffer to use */ - ulint buff_len, /*!< in: buffer length */ - row_prebuilt_t* prebuilt, /*!< in: InnoDB prebuilt struct */ - THD* thd) /*!< in: user thread */ + upd_t* uvect, + const uchar* old_row, + uchar* new_row, + TABLE* table, + uchar* upd_buff, + ulint buff_len, + row_prebuilt_t* prebuilt, + ib_uint64_t& auto_inc) { uchar* original_upd_buff = upd_buff; Field* field; enum_field_types field_mysql_type; - uint n_fields; ulint o_len; ulint n_len; ulint col_pack_len; @@ -9345,34 +8950,30 @@ calc_row_difference( ulint n_changed = 0; dfield_t dfield; dict_index_t* clust_index; - uint sql_idx,i, innodb_idx= 0; + uint i; ibool changes_fts_column = FALSE; ibool changes_fts_doc_col = FALSE; - trx_t* trx = thd_to_trx(thd); + trx_t* const trx = prebuilt->trx; doc_id_t doc_id = FTS_NULL_DOC_ID; ulint num_v = 0; - ut_ad(!srv_read_only_mode || dict_table_is_intrinsic(prebuilt->table)); + ut_ad(!srv_read_only_mode); - n_fields = table->s->fields; clust_index = dict_table_get_first_index(prebuilt->table); + auto_inc = 0; /* We use upd_buff to convert changed fields */ buf = (byte*) upd_buff; - for (sql_idx = 0,i=0; i < n_fields; i++, sql_idx++) { - field = table->field[sql_idx]; + for (i = 0; i < table->s->fields; i++) { + field = table->field[i]; bool is_virtual = innobase_is_v_fld(field); dict_col_t* col; - if (!field->stored_in_db()) { - continue; - } - if (is_virtual) { col = &prebuilt->table->v_cols[num_v].m_col; } else { - col = &prebuilt->table->cols[innodb_idx - num_v]; + col = &prebuilt->table->cols[i - num_v]; } o_ptr = (const byte*) old_row + get_field_offset(table, field); @@ -9585,11 +9186,22 @@ calc_row_difference( dfield_set_null(vfield); } num_v++; + ut_ad(field != table->found_next_number_field); } else { ufield->field_no = dict_col_get_clust_pos( - &prebuilt->table->cols[innodb_idx - num_v], + &prebuilt->table->cols[i - num_v], clust_index); ufield->old_v_val = NULL; + if (field != table->found_next_number_field + || dfield_is_null(&ufield->new_val)) { + } else { + auto_inc = row_parse_int( + static_cast<const byte*>( + ufield->new_val.data), + ufield->new_val.len, + col->mtype, + col->prtype & DATA_UNSIGNED); + } } n_changed++; @@ -9633,10 +9245,6 @@ calc_row_difference( ut_ad(col->ord_part || online_ord_part); num_v++; } - - if (field->stored_in_db()) { - innodb_idx++; - } } /* If the update changes a column with an FTS index on it, we @@ -9644,7 +9252,7 @@ calc_row_difference( other changes. We piggy back our changes on the normal UPDATE to reduce processing and IO overhead. */ if (!prebuilt->table->fts) { - trx->fts_next_doc_id = 0; + trx->fts_next_doc_id = 0; } else if (changes_fts_column || changes_fts_doc_col) { dict_table_t* innodb_table = prebuilt->table; @@ -9801,7 +9409,7 @@ wsrep_calc_row_hash( } #endif /* WITH_WSREP */ -/* +/** Updates a row given as a parameter to a new value. Note that we are given whole rows, not just the fields which are updated: this incurs some overhead for CPU when we check which fields are actually updated. @@ -9826,7 +9434,7 @@ ha_innobase::update_row( ut_a(m_prebuilt->trx == trx); - if (high_level_read_only && !dict_table_is_intrinsic(m_prebuilt->table)) { + if (high_level_read_only) { ib_senderrf(ha_thd(), IB_LOG_LEVEL_WARN, ER_READ_ONLY_MODE); DBUG_RETURN(HA_ERR_TABLE_READONLY); } else if (!trx_is_started(trx)) { @@ -9837,19 +9445,17 @@ ha_innobase::update_row( ut_ad(m_upd_buf_size == 0); /* Create a buffer for packing the fields of a record. Why - table->stored_rec_length did not work here? Obviously, - because char fields when packed actually became 1 byte - longer, when we also stored the string length as the first - byte. */ + table->reclength did not work here? Obviously, because char + fields when packed actually became 1 byte longer, when we also + stored the string length as the first byte. */ - m_upd_buf_size = table->s->stored_rec_length + table->s->max_key_length + m_upd_buf_size = table->s->reclength + table->s->max_key_length + MAX_REF_PARTS * 3; m_upd_buf = reinterpret_cast<uchar*>( - my_malloc( - m_upd_buf_size, + my_malloc(//PSI_INSTRUMENT_ME, + m_upd_buf_size, MYF(MY_WME))); - /* JAN: TODO: MySQL 5.7: PSI_INSTRUMENT_ME,...*/ if (m_upd_buf == NULL) { m_upd_buf_size = 0; @@ -9857,27 +9463,21 @@ ha_innobase::update_row( } } - upd_t* uvect; - - if (m_prebuilt->upd_node) { - uvect = m_prebuilt->upd_node->update; - } else { - uvect = row_get_prebuilt_update_vector(m_prebuilt); - } + upd_t* uvect = row_get_prebuilt_update_vector(m_prebuilt); + ib_uint64_t autoinc; /* Build an update vector from the modified fields in the rows (uses m_upd_buf of the handle) */ error = calc_row_difference( uvect, old_row, new_row, table, m_upd_buf, m_upd_buf_size, - m_prebuilt, m_user_thd); + m_prebuilt, autoinc); if (error != DB_SUCCESS) { goto func_exit; } - if (!dict_table_is_intrinsic(m_prebuilt->table) - && TrxInInnoDB::is_aborted(trx)) { + if (TrxInInnoDB::is_aborted(trx)) { innobase_rollback(ht, m_user_thd, false); @@ -9892,42 +9492,29 @@ ha_innobase::update_row( error = row_update_for_mysql((byte*) old_row, m_prebuilt); - /* We need to do some special AUTOINC handling for the following case: - - INSERT INTO t (c1,c2) VALUES(x,y) ON DUPLICATE KEY UPDATE ... - - We need to use the AUTOINC counter that was actually used by - MySQL in the UPDATE statement, which can be different from the - value used in the INSERT statement. */ - - if (error == DB_SUCCESS - && table->next_number_field - && new_row == table->record[0] - && thd_sql_command(m_user_thd) == SQLCOM_INSERT - && trx->duplicates) { - - ulonglong auto_inc; - ulonglong col_max_value; - - auto_inc = table->next_number_field->val_int(); - - /* We need the upper limit of the col type to check for - whether we update the table autoinc counter or not. */ - col_max_value = - innobase_get_int_col_max_value(table->next_number_field); - - if (auto_inc <= col_max_value && auto_inc != 0) { + if (error == DB_SUCCESS && autoinc) { + /* A value for an AUTO_INCREMENT column + was specified in the UPDATE statement. */ - ulonglong offset; - ulonglong increment; + autoinc = innobase_next_autoinc( + autoinc, 1, + m_prebuilt->autoinc_increment, + m_prebuilt->autoinc_offset, + innobase_get_int_col_max_value( + table->found_next_number_field)); - offset = m_prebuilt->autoinc_offset; - increment = m_prebuilt->autoinc_increment; + error = innobase_set_max_autoinc(autoinc); - auto_inc = innobase_next_autoinc( - auto_inc, 1, increment, offset, col_max_value); - - error = innobase_set_max_autoinc(auto_inc); + if (m_prebuilt->table->persistent_autoinc) { + /* Update the PAGE_ROOT_AUTO_INC. Yes, we do + this even if dict_table_t::autoinc already was + greater than autoinc, because we cannot know + if any INSERT actually used (and wrote to + PAGE_ROOT_AUTO_INC) a value bigger than our + autoinc. */ + btr_write_autoinc(dict_table_get_first_index( + m_prebuilt->table), + autoinc); } } @@ -9991,8 +9578,7 @@ ha_innobase::delete_row( DBUG_ENTER("ha_innobase::delete_row"); - if (!dict_table_is_intrinsic(m_prebuilt->table) - && trx_in_innodb.is_aborted()) { + if (trx_in_innodb.is_aborted()) { innobase_rollback(ht, m_user_thd, false); @@ -10002,7 +9588,7 @@ ha_innobase::delete_row( ut_a(m_prebuilt->trx == trx); - if (high_level_read_only && !dict_table_is_intrinsic(m_prebuilt->table)) { + if (high_level_read_only) { ib_senderrf(ha_thd(), IB_LOG_LEVEL_WARN, ER_READ_ONLY_MODE); DBUG_RETURN(HA_ERR_TABLE_READONLY); } else if (!trx_is_started(trx)) { @@ -10052,28 +9638,7 @@ int ha_innobase::delete_all_rows() { DBUG_ENTER("ha_innobase::delete_all_rows"); - - /* Currently enabled only for intrinsic tables. */ - if (!dict_table_is_intrinsic(m_prebuilt->table)) { - DBUG_RETURN(HA_ERR_WRONG_COMMAND); - } - - TrxInInnoDB trx_in_innodb(m_prebuilt->trx); - - if (!dict_table_is_intrinsic(m_prebuilt->table) - && trx_in_innodb.is_aborted()) { - - DBUG_RETURN(innobase_rollback(ht, m_user_thd, false)); - } - - dberr_t error = row_delete_all_rows(m_prebuilt->table); - - if (error == DB_SUCCESS) { - dict_stats_update(m_prebuilt->table, DICT_STATS_EMPTY_TABLE); - } - - DBUG_RETURN(convert_error_code_to_mysql( - error, m_prebuilt->table->flags, m_user_thd)); + DBUG_RETURN(HA_ERR_WRONG_COMMAND); } /**********************************************************************//** @@ -10087,11 +9652,7 @@ ha_innobase::unlock_row(void) { DBUG_ENTER("ha_innobase::unlock_row"); - /* Consistent read does not take any locks, thus there is - nothing to unlock. There is no locking for intrinsic table. */ - - if (m_prebuilt->select_lock_type == LOCK_NONE - || dict_table_is_intrinsic(m_prebuilt->table)) { + if (m_prebuilt->select_lock_type == LOCK_NONE) { DBUG_VOID_RETURN; } @@ -10101,8 +9662,6 @@ ha_innobase::unlock_row(void) DBUG_VOID_RETURN; } - ut_ad(!dict_table_is_intrinsic(m_prebuilt->table)); - /* Ideally, this assert must be in the beginning of the function. But there are some calls to this function from the SQL layer when the transaction is in state TRX_STATE_NOT_STARTED. The check on @@ -10190,8 +9749,6 @@ ha_innobase::index_end(void) { DBUG_ENTER("index_end"); - m_prebuilt->index->last_sel_cur->release(); - active_index = MAX_KEY; in_range_check_pushed_down = FALSE; @@ -10393,29 +9950,17 @@ ha_innobase::index_read( innobase_srv_conc_enter_innodb(m_prebuilt); - if (!dict_table_is_intrinsic(m_prebuilt->table)) { + if (TrxInInnoDB::is_aborted(m_prebuilt->trx)) { - if (TrxInInnoDB::is_aborted(m_prebuilt->trx)) { + innobase_rollback(ht, m_user_thd, false); - innobase_rollback(ht, m_user_thd, false); - - DBUG_RETURN(convert_error_code_to_mysql( - DB_FORCED_ABORT, 0, m_user_thd)); - } - - m_prebuilt->ins_sel_stmt = thd_is_ins_sel_stmt( - m_user_thd); - - ret = row_search_mvcc( - buf, mode, m_prebuilt, match_mode, 0); - - } else { - m_prebuilt->session = thd_to_innodb_session(m_user_thd); - - ret = row_search_no_mvcc( - buf, mode, m_prebuilt, match_mode, 0); + DBUG_RETURN(convert_error_code_to_mysql( + DB_FORCED_ABORT, 0, m_user_thd)); } + ret = row_search_mvcc( + buf, mode, m_prebuilt, match_mode, 0); + innobase_srv_conc_exit_innodb(m_prebuilt); } else { @@ -10595,8 +10140,7 @@ ha_innobase::change_active_index( TrxInInnoDB trx_in_innodb(m_prebuilt->trx); - if (!dict_table_is_intrinsic(m_prebuilt->table) - && trx_in_innodb.is_aborted()) { + if (trx_in_innodb.is_aborted()) { innobase_rollback(ht, m_user_thd, false); @@ -10723,9 +10267,7 @@ ha_innobase::general_fetch( ut_ad(trx == thd_to_trx(m_user_thd)); - bool intrinsic = dict_table_is_intrinsic(m_prebuilt->table); - - if (!intrinsic && TrxInInnoDB::is_aborted(trx)) { + if (TrxInInnoDB::is_aborted(trx)) { innobase_rollback(ht, m_user_thd, false); @@ -10735,17 +10277,8 @@ ha_innobase::general_fetch( innobase_srv_conc_enter_innodb(m_prebuilt); - if (!intrinsic) { - - ret = row_search_mvcc( - buf, PAGE_CUR_UNSUPP, m_prebuilt, match_mode, - direction); - - } else { - ret = row_search_no_mvcc( - buf, PAGE_CUR_UNSUPP, m_prebuilt, match_mode, - direction); - } + ret = row_search_mvcc( + buf, PAGE_CUR_UNSUPP, m_prebuilt, match_mode, direction); innobase_srv_conc_exit_innodb(m_prebuilt); @@ -10899,8 +10432,7 @@ ha_innobase::rnd_init( { TrxInInnoDB trx_in_innodb(m_prebuilt->trx); - if (!dict_table_is_intrinsic(m_prebuilt->table) - && trx_in_innodb.is_aborted()) { + if (trx_in_innodb.is_aborted()) { return(innobase_rollback(ht, m_user_thd, false)); } @@ -11150,10 +10682,7 @@ ha_innobase::ft_init_ext( /* Allocate FTS handler, and instantiate it before return */ fts_hdl = reinterpret_cast<NEW_FT_INFO*>( - my_malloc(sizeof(NEW_FT_INFO), MYF(0))); - /* JAN: TODO: MySQL 5.7 PSI - my_malloc(PSI_INSTRUMENT_ME, sizeof(NEW_FT_INFO), MYF(0))); - */ + my_malloc(/*PSI_INSTRUMENT_ME,*/ sizeof(NEW_FT_INFO), MYF(0))); fts_hdl->please = const_cast<_ft_vft*>(&ft_vft_result); fts_hdl->could_you = const_cast<_ft_vft_ext*>(&ft_vft_ext_result); @@ -11747,7 +11276,7 @@ ha_innobase::wsrep_append_keys( keyval1[0] = (char)i; if (!tab) { - WSREP_WARN("MySQL-InnoDB key mismatch %s %s", + WSREP_WARN("MariaDB-InnoDB key mismatch %s %s", table->s->table_name.str, key_info->name); } @@ -11932,7 +11461,43 @@ create_table_check_doc_id_col( return(false); } -#ifdef MYSQL_VIRTUAL_COLUMNS + +/** Finds all base columns needed to compute a given generated column. +This is returned as a bitmap, in field->table->tmp_set. +Works for both dict_v_col_t and dict_s_col_t columns. +@param[in] table InnoDB table +@param[in] field MySQL field +@param[in,out] col virtual or stored column */ +template <typename T> +void +prepare_vcol_for_base_setup( +/*========================*/ + const dict_table_t* table, + const Field* field, + T* col) +{ + ut_ad(col->num_base == 0); + ut_ad(col->base_col == NULL); + + MY_BITMAP *old_read_set = field->table->read_set; + MY_BITMAP *old_vcol_set = field->table->vcol_set; + + field->table->read_set = field->table->vcol_set = &field->table->tmp_set; + + bitmap_clear_all(&field->table->tmp_set); + field->vcol_info->expr->walk( + &Item::register_field_in_read_map, 1, field->table); + col->num_base= bitmap_bits_set(&field->table->tmp_set); + if (col->num_base != 0) { + col->base_col = static_cast<dict_col_t**>(mem_heap_zalloc( + table->heap, col->num_base * sizeof( + * col->base_col))); + } + field->table->read_set= old_read_set; + field->table->vcol_set= old_vcol_set; +} + + /** Set up base columns for virtual column @param[in] table InnoDB table @param[in] field MySQL field @@ -11945,10 +11510,12 @@ innodb_base_col_setup( { int n = 0; + prepare_vcol_for_base_setup(table, field, v_col); + for (uint i= 0; i < field->table->s->fields; ++i) { const Field* base_field = field->table->field[i]; - if (!base_field->is_virtual_gcol() - && bitmap_is_set(&field->gcol_info->base_columns_map, i)) { + if (base_field->stored_in_db() + && bitmap_is_set(&field->table->tmp_set, i)) { ulint z; for (z = 0; z < table->n_cols; z++) { @@ -11966,10 +11533,9 @@ innodb_base_col_setup( n++; } } + v_col->num_base= n; } -#endif /* MYSQL_VIRTUAL_COLUMNS */ -#ifdef MYSQL_VIRTUAL_COLUMNS /** Set up base columns for stored column @param[in] table InnoDB table @param[in] field MySQL field @@ -11982,12 +11548,14 @@ innodb_base_col_setup_for_stored( { ulint n = 0; + prepare_vcol_for_base_setup(table, field, s_col); + for (uint i= 0; i < field->table->s->fields; ++i) { const Field* base_field = field->table->field[i]; if (!innobase_is_s_fld(base_field) && !innobase_is_v_fld(base_field) - && bitmap_is_set(&field->gcol_info->base_columns_map, + && bitmap_is_set(&field->table->tmp_set, i)) { ulint z; for (z = 0; z < table->n_cols; z++) { @@ -12009,8 +11577,8 @@ innodb_base_col_setup_for_stored( } } } + s_col->num_base= n; } -#endif /** Create a table definition to an InnoDB database. @return ER_* level error */ @@ -12020,7 +11588,6 @@ create_table_info_t::create_table_def() { dict_table_t* table; ulint n_cols; - ulint s_cols; ulint col_type; ulint col_len; ulint nulls_allowed; @@ -12029,6 +11596,7 @@ create_table_info_t::create_table_def() ulint long_true_varchar; ulint charset_no; ulint i; + ulint j = 0; ulint doc_id_col = 0; ibool has_doc_id_col = FALSE; mem_heap_t* heap; @@ -12065,9 +11633,7 @@ create_table_info_t::create_table_def() } n_cols = m_form->s->fields; - s_cols = m_form->s->stored_fields; -#ifdef MYSQL_VIRTUAL_COLUMNS /* Find out any virtual column */ for (i = 0; i < n_cols; i++) { Field* field = m_form->field[i]; @@ -12076,7 +11642,6 @@ create_table_info_t::create_table_def() num_v++; } } -#endif /* MYSQL_VIRTUAL_COLUMNS */ ut_ad(trx_state_eq(m_trx, TRX_STATE_NOT_STARTED)); @@ -12095,15 +11660,9 @@ create_table_info_t::create_table_def() /* For single-table tablespaces, we pass 0 as the space id, and then determine the actual space id when the tablespace is created. */ - if (DICT_TF_HAS_SHARED_SPACE(m_flags)) { - ut_ad(m_tablespace != NULL && m_tablespace[0] != '\0'); - - space_id = fil_space_get_id_by_name(m_tablespace); - } /* Adjust the number of columns for the FTS hidden field */ - actual_n_cols = m_form->s->stored_fields; - + actual_n_cols = n_cols; if (m_flags2 & DICT_TF2_FTS && !has_doc_id_col) { actual_n_cols += 1; } @@ -12114,12 +11673,7 @@ create_table_info_t::create_table_def() /* Set the hidden doc_id column. */ if (m_flags2 & DICT_TF2_FTS) { table->fts->doc_col = has_doc_id_col - ? doc_id_col : s_cols; - } - - if (strlen(m_temp_path) != 0) { - table->dir_path_of_temp_table = - mem_heap_strdup(table->heap, m_temp_path); + ? doc_id_col : n_cols - num_v; } if (DICT_TF_HAS_DATA_DIR(m_flags)) { @@ -12132,42 +11686,13 @@ create_table_info_t::create_table_def() table->data_dir_path = NULL; } - if (DICT_TF_HAS_SHARED_SPACE(m_flags)) { - ut_ad(strlen(m_tablespace)); - table->tablespace = mem_heap_strdup(table->heap, m_tablespace); - } else { - table->tablespace = NULL; - } - heap = mem_heap_create(1000); for (i = 0; i < n_cols; i++) { ulint is_virtual; - bool is_stored MY_ATTRIBUTE((unused)); - Field* field = m_form->field[i]; - - if (!field->stored_in_db()) { - continue; - } - - /* Generate a unique column name by pre-pending table-name for - intrinsic tables. For other tables (including normal - temporary) column names are unique. If not, MySQL layer will - block such statement. - This is work-around fix till Optimizer can handle this issue - (probably 5.7.4+). */ - char field_name[MAX_FULL_NAME_LEN + 2 + 10]; - - if (dict_table_is_intrinsic(table) && field->orig_table) { + bool is_stored = false; - ut_snprintf(field_name, sizeof(field_name), - "%s_%s_%lu", field->orig_table->alias.c_ptr(), - field->field_name, i); - - } else { - ut_snprintf(field_name, sizeof(field_name), - "%s", field->field_name); - } + Field* field = m_form->field[i]; col_type = get_innobase_type_from_mysql_type( &unsigned_type, field); @@ -12244,9 +11769,9 @@ create_table_info_t::create_table_def() /* First check whether the column to be added has a system reserved name. */ - if (dict_col_name_is_reserved(field_name)){ + if (dict_col_name_is_reserved(field->field_name)){ my_error(ER_WRONG_COLUMN_NAME, MYF(0), - field_name); + field->field_name); err_col: dict_mem_table_free(table); mem_heap_free(heap); @@ -12258,7 +11783,7 @@ err_col: if (!is_virtual) { dict_mem_table_add_col(table, heap, - field_name, col_type, + field->field_name, col_type, dtype_form_prtype( (ulint) field->type() | nulls_allowed | unsigned_type @@ -12266,35 +11791,26 @@ err_col: charset_no), col_len); } else { -#ifdef MYSQL_VIRTUAL_COLUMNS dict_mem_table_add_v_col(table, heap, - field_name, col_type, + field->field_name, col_type, dtype_form_prtype( (ulint) field->type() | nulls_allowed | unsigned_type | binary_type | long_true_varchar | is_virtual, charset_no), - col_len, i, - 0); - - field->gcol_info->non_virtual_base_columns()); -#endif - } + col_len, i, 0); + } -#ifdef MYSQL_VIRTUAL_COLUMNS if (is_stored) { ut_ad(!is_virtual); /* Added stored column in m_s_cols list. */ dict_mem_table_add_s_col( - table, - field->gcol_info->non_virtual_base_columns()); + table, 0); } -#endif } -#ifdef MYSQL_VIRTUAL_COLUMNS + if (num_v) { - ulint j = 0; for (i = 0; i < n_cols; i++) { dict_v_col_t* v_col; @@ -12335,7 +11851,6 @@ err_col: } } } -#endif /* MYSQL_VIRTUAL_COLUMNS */ /* Add the FTS doc_id hidden column. */ if (m_flags2 & DICT_TF2_FTS && !has_doc_id_col) { @@ -12349,147 +11864,34 @@ err_col: on re-start we don't need to restore temp-table and so no entry is needed in SYSTEM tables. */ if (dict_table_is_temporary(table)) { -#ifdef MYSQL_COMPRESSION - if (m_create_info->compress.length > 0) { - push_warning_printf( - m_thd, - Sql_condition::WARN_LEVEL_WARN, - HA_ERR_UNSUPPORTED, - "InnoDB: Compression not supported for " - "temporary tables"); - - err = DB_UNSUPPORTED; - - dict_mem_table_free(table); - } else if (m_create_info->encrypt_type.length > 0 - && !Encryption::is_none( - m_create_info->encrypt_type.str)) { - - my_error(ER_TABLESPACE_CANNOT_ENCRYPT, MYF(0)); - err = DB_UNSUPPORTED; - dict_mem_table_free(table); - */ - } else { -#endif /* MYSQL_COMPRESSION */ - /* Get a new table ID */ - dict_table_assign_new_id(table, m_trx); + /* Get a new table ID */ + dict_table_assign_new_id(table, m_trx); - /* Create temp tablespace if configured. */ - err = dict_build_tablespace_for_table(table, NULL); + /* Create temp tablespace if configured. */ + err = dict_build_tablespace_for_table(table, NULL); - if (err == DB_SUCCESS) { - /* Temp-table are maintained in memory and so - can_be_evicted is FALSE. */ - mem_heap_t* temp_table_heap; + if (err == DB_SUCCESS) { + /* Temp-table are maintained in memory and so + can_be_evicted is FALSE. */ + mem_heap_t* temp_table_heap; - temp_table_heap = mem_heap_create(256); + temp_table_heap = mem_heap_create(256); - /* For intrinsic table (given that they are - not shared beyond session scope), add - it to session specific THD structure - instead of adding it to dictionary cache. */ - if (dict_table_is_intrinsic(table)) { - add_table_to_thread_cache( - table, temp_table_heap, m_thd); + dict_table_add_to_cache( + table, FALSE, temp_table_heap); - } else { - dict_table_add_to_cache( - table, FALSE, temp_table_heap); - } + DBUG_EXECUTE_IF("ib_ddl_crash_during_create2", + DBUG_SUICIDE();); - DBUG_EXECUTE_IF("ib_ddl_crash_during_create2", - DBUG_SUICIDE();); - - mem_heap_free(temp_table_heap); - } -#ifdef MYSQL_COMPRESSION + mem_heap_free(temp_table_heap); } -#endif - } else { - const char* algorithm = NULL; - -#if MYSQL_COMPRESSION - const char* algorithm = m_create_info->compress.str; - - if (!(m_flags2 & DICT_TF2_USE_FILE_PER_TABLE) - && m_create_info->compress.length > 0 - && !Compression::is_none(algorithm)) { - - push_warning_printf( - m_thd, - Sql_condition::WARN_LEVEL_WARN, - HA_ERR_UNSUPPORTED, - "InnoDB: Compression not supported for " - "shared tablespaces"); - - algorithm = NULL; - - err = DB_UNSUPPORTED; - dict_mem_table_free(table); - - } else if (Compression::validate(algorithm) != DB_SUCCESS - || m_form->s->row_type == ROW_TYPE_COMPRESSED - || m_create_info->key_block_size > 0) { - - algorithm = NULL; - } - - const char* encrypt = m_create_info->encrypt_type.str; - - if (!(m_flags2 & DICT_TF2_USE_FILE_PER_TABLE) - && m_create_info->encrypt_type.length > 0 - && !Encryption::is_none(encrypt)) { - - my_error(ER_TABLESPACE_CANNOT_ENCRYPT, MYF(0)); - err = DB_UNSUPPORTED; - dict_mem_table_free(table); - - } else if (!Encryption::is_none(encrypt)) { - /* Set the encryption flag. */ - byte* master_key = NULL; - ulint master_key_id; - Encryption::Version version; - - /* Check if keyring is ready. */ - Encryption::get_master_key(&master_key_id, - &master_key, - &version); - - if (master_key == NULL) { - my_error(ER_CANNOT_FIND_KEY_IN_KEYRING, - MYF(0)); - err = DB_UNSUPPORTED; - dict_mem_table_free(table); - } else { - my_free(master_key); - DICT_TF2_FLAG_SET(table, - DICT_TF2_ENCRYPTION); - } - } -#endif /* MYSQL_COMPRESSION */ - if (err == DB_SUCCESS) { err = row_create_table_for_mysql( - table, algorithm, m_trx, false, + table, m_trx, false, (fil_encryption_t)options->encryption, - options->encryption_key_id); - - } - - if (err == DB_IO_NO_PUNCH_HOLE_FS) { - - ut_ad(!dict_table_in_shared_tablespace(table)); - - push_warning_printf( - m_thd, - Sql_condition::WARN_LEVEL_WARN, - HA_ERR_UNSUPPORTED, - "InnoDB: Punch hole not supported by the " - "file system or the tablespace page size " - "is not large enough. Compression disabled"); + (ulint)options->encryption_key_id); - err = DB_SUCCESS; } DBUG_EXECUTE_IF("ib_crash_during_create_for_encryption", @@ -12578,7 +11980,7 @@ create_index( DBUG_RETURN(convert_error_code_to_mysql( row_create_index_for_mysql( - index, trx, NULL, NULL), + index, trx, NULL), flags, NULL)); } @@ -12593,13 +11995,7 @@ create_index( ind_type |= DICT_UNIQUE; } - /* JAN: TODO: MySQL 5.7 PSI - field_lengths = (ulint*) my_malloc(PSI_INSTRUMENT_ME, - key->user_defined_key_parts * sizeof * - field_lengths, MYF(MY_FAE)); - */ - - field_lengths = (ulint*) my_malloc( + field_lengths = (ulint*) my_malloc(//PSI_INSTRUMENT_ME, key->user_defined_key_parts * sizeof * field_lengths, MYF(MY_FAE)); @@ -12609,26 +12005,6 @@ create_index( index = dict_mem_index_create(table_name, key->name, 0, ind_type, key->user_defined_key_parts); - innodb_session_t*& priv = thd_to_innodb_session(trx->mysql_thd); - dict_table_t* handler = priv->lookup_table_handler(table_name); - - if (handler != NULL) { - /* This setting will enforce SQL NULL == SQL NULL. - For now this is turned-on for intrinsic tables - only but can be turned on for other tables if needed arises. */ - index->nulls_equal = - (key->flags & HA_NULL_ARE_EQUAL) ? true : false; - - /* Disable use of AHI for intrinsic table indexes as AHI - validates the predicated entry using index-id which has to be - system-wide unique that is not the case with indexes of - intrinsic table for performance reason. - Also given the lifetime of these tables and frequent delete - and update AHI would not help on performance front as it does - with normal tables. */ - index->disable_ahi = true; - } - for (ulint i = 0; i < key->user_defined_key_parts; i++) { KEY_PART_INFO* key_part = key->key_part + i; ulint prefix_len; @@ -12652,13 +12028,6 @@ create_index( ut_error; const char* field_name = key_part->field->field_name; - if (handler != NULL && dict_table_is_intrinsic(handler)) { - - ut_ad(!innobase_is_v_fld(key_part->field)); - ulint col_no = dict_col_get_no(dict_table_get_nth_col( - handler, key_part->field->field_index)); - field_name = dict_table_get_col_name(handler, col_no); - } col_type = get_innobase_type_from_mysql_type( &is_unsigned, key_part->field); @@ -12679,7 +12048,7 @@ create_index( case DATA_DOUBLE: case DATA_DECIMAL: sql_print_error( - "MySQL is trying to create a column" + "MariaDB is trying to create a column" " prefix index field, on an" " inappropriate data type. Table" " name %s, column name %s.", @@ -12708,13 +12077,9 @@ create_index( sure we don't create too long indexes. */ error = convert_error_code_to_mysql( - row_create_index_for_mysql(index, trx, field_lengths, handler), + row_create_index_for_mysql(index, trx, field_lengths), flags, NULL); - if (error && handler != NULL) { - priv->unregister_table_handler(table_name); - } - my_free(field_lengths); DBUG_RETURN(error); @@ -12740,26 +12105,7 @@ create_clustered_index_when_no_primary( innobase_index_reserve_name, 0, DICT_CLUSTERED, 0); - innodb_session_t*& priv = thd_to_innodb_session(trx->mysql_thd); - - dict_table_t* handler = priv->lookup_table_handler(table_name); - - if (handler != NULL) { - /* Disable use of AHI for intrinsic table indexes as AHI - validates the predicated entry using index-id which has to be - system-wide unique that is not the case with indexes of - intrinsic table for performance reason. - Also given the lifetime of these tables and frequent delete - and update AHI would not help on performance front as it does - with normal tables. */ - index->disable_ahi = true; - } - - error = row_create_index_for_mysql(index, trx, NULL, handler); - - if (error != DB_SUCCESS && handler != NULL) { - priv->unregister_table_handler(table_name); - } + error = row_create_index_for_mysql(index, trx, NULL); return(convert_error_code_to_mysql(error, flags, NULL)); } @@ -12802,7 +12148,7 @@ create_table_info_t::create_option_data_directory_is_valid() && m_create_info->data_file_name[0] != '\0'); /* Use DATA DIRECTORY only with file-per-table. */ - if (!m_use_shared_space && !m_allow_file_per_table) { + if (!m_allow_file_per_table) { push_warning( m_thd, Sql_condition::WARN_LEVEL_WARN, ER_ILLEGAL_HA_CREATE_OPTION, @@ -12827,273 +12173,6 @@ create_table_info_t::create_option_data_directory_is_valid() return(is_valid); } - -#ifdef MYSQL_TABLESPACES -#define IDENT_NAME_OK 0 -static int check_tablespace_name(const char *name) -{ - CHARSET_INFO *cs= system_charset_info; - return cs->cset->numchars(cs, name, name + strlen(name)) > NAME_CHAR_LEN; -} - - -/** Validate the tablespace name provided for a tablespace DDL -@param[in] name A proposed tablespace name -@param[in] for_table Caller is putting a table here -@return MySQL handler error code like HA_... */ -static -int -validate_tablespace_name( - const char* name, - bool for_table) -{ - int err = 0; - - /* This prefix is reserved by InnoDB for use in internal tablespace names. */ - const char reserved_space_name_prefix[] = "innodb_"; - - if (check_tablespace_name(name) != IDENT_NAME_OK) { - err = HA_WRONG_CREATE_OPTION; - } - - /* The tablespace name cannot start with `innodb_`. */ - if (strlen(name) >= sizeof(reserved_space_name_prefix) - 1 - && 0 == memcmp(name, reserved_space_name_prefix, - sizeof(reserved_space_name_prefix) - 1)) { - - /* Use a different message for reserved names */ - if (0 == strcmp(name, reserved_file_per_table_space_name) - || 0 == strcmp(name, reserved_system_space_name) - || 0 == strcmp(name, reserved_temporary_space_name)) { - /* Allow these names if the caller is putting a - table into one of these by CREATE/ALTER TABLE */ - if (!for_table) { - my_printf_error( - ER_WRONG_TABLESPACE_NAME, - "InnoDB: `%s` is a reserved" - " tablespace name.", - MYF(0), name); - err = HA_WRONG_CREATE_OPTION; - } - } else { - my_printf_error( - ER_WRONG_TABLESPACE_NAME, - "InnoDB: A general tablespace" - " name cannot start with `%s`.", - MYF(0), reserved_space_name_prefix); - err = HA_WRONG_CREATE_OPTION; - } - } - - /* The tablespace name cannot contain a '/'. */ - if (memchr(name, '/', strlen(name)) != NULL) { - my_printf_error(ER_WRONG_TABLESPACE_NAME, - "InnoDB: A general tablespace name cannot" - " contain '/'.", MYF(0)); - err = HA_WRONG_CREATE_OPTION; - } - - return(err); -} - -/** Validate TABLESPACE option. -@return true if valid, false if not. */ -bool -create_table_info_t::create_option_tablespace_is_valid() -{ - if (!m_use_shared_space) { - return(true); - } - - if (0 != validate_tablespace_name(m_create_info->tablespace, true)) { - return(false); - } - - /* Look up the tablespace name in the fil_system. */ - ulint space_id = fil_space_get_id_by_name( - m_create_info->tablespace); - - if (space_id == ULINT_UNDEFINED) { - my_printf_error(ER_TABLESPACE_MISSING, - "InnoDB: A general tablespace named" - " `%s` cannot be found.", MYF(0), - m_create_info->tablespace); - return(false); - } - - /* Cannot add a second table to a file-per-table tablespace. */ - ulint fsp_flags = fil_space_get_flags(space_id); - if (fsp_is_file_per_table(space_id, fsp_flags)) { - my_printf_error(ER_ILLEGAL_HA_CREATE_OPTION, - "InnoDB: Tablespace `%s` is file-per-table so no" - " other table can be added to it.", MYF(0), - m_create_info->tablespace); - return(false); - } - - /* If TABLESPACE=innodb_file_per_table this function is not called - since tablespace_is_shared_space() will return false. Any other - tablespace is incompatible with the DATA DIRECTORY phrase. - On any ALTER TABLE that contains a DATA DIRECTORY, MySQL will issue - a warning like "<DATA DIRECTORY> option ignored." The check below is - needed for CREATE TABLE only. ALTER TABLE may be moving remote - file-per-table table to a general tablespace, in which case the - create_info->data_file_name is not null. */ - bool is_create_table = (thd_sql_command(m_thd) == SQLCOM_CREATE_TABLE); - if (is_create_table - && m_create_info->data_file_name != NULL - && m_create_info->data_file_name[0] != '\0') { - my_printf_error(ER_ILLEGAL_HA_CREATE_OPTION, - "InnoDB: DATA DIRECTORY cannot be used" - " with a TABLESPACE assignment.", MYF(0)); - return(false); - } - - /* Temp tables only belong in temp tablespaces. */ - if (m_create_info->options & HA_LEX_CREATE_TMP_TABLE) { - if (!FSP_FLAGS_GET_TEMPORARY(fsp_flags)) { - my_printf_error(ER_ILLEGAL_HA_CREATE_OPTION, - "InnoDB: Tablespace `%s` cannot contain" - " TEMPORARY tables.", MYF(0), - m_create_info->tablespace); - return(false); - } - - /* Restrict Compressed Temporary General tablespaces. */ - if (m_create_info->key_block_size - || m_create_info->row_type == ROW_TYPE_COMPRESSED) { - my_printf_error(ER_ILLEGAL_HA_CREATE_OPTION, - "InnoDB: Temporary tablespace `%s` cannot" - " contain COMPRESSED tables.", MYF(0), - m_create_info->tablespace); - return(false); - } - } else if (FSP_FLAGS_GET_TEMPORARY(fsp_flags)) { - my_printf_error(ER_ILLEGAL_HA_CREATE_OPTION, - "InnoDB: Tablespace `%s` can only contain" - " TEMPORARY tables.", MYF(0), - m_create_info->tablespace); - return(false); - } - - /* Make sure the physical page size of the table matches the - file block size of the tablespace. */ - ulint block_size_needed; - bool table_is_compressed; - if (m_create_info->key_block_size) { - block_size_needed = m_create_info->key_block_size * 1024; - table_is_compressed = true; - } else if (m_create_info->row_type == ROW_TYPE_COMPRESSED) { - block_size_needed = ut_min( - UNIV_PAGE_SIZE / 2, - static_cast<ulint>(UNIV_ZIP_SIZE_MAX)); - table_is_compressed = true; - } else { - block_size_needed = UNIV_PAGE_SIZE; - table_is_compressed = false; - } - - const page_size_t page_size(fsp_flags); - - /* The compression code needs some work in order for a general - tablespace to contain both compressed and non-compressed tables - together in the same tablespace. The problem seems to be that - each page is either compressed or not based on the fsp flags, - which is shared by all tables in that general tablespace. */ - if (table_is_compressed && page_size.physical() == UNIV_PAGE_SIZE) { - my_printf_error(ER_ILLEGAL_HA_CREATE_OPTION, - "InnoDB: Tablespace `%s` cannot contain a" - " COMPRESSED table", MYF(0), - m_create_info->tablespace); - return(false); - } - - if (block_size_needed != page_size.physical()) { - my_printf_error(ER_ILLEGAL_HA_CREATE_OPTION, - "InnoDB: Tablespace `%s` uses block size " ULINTPF - " and cannot contain a table with physical" - " page size " ULINTPF, MYF(0), - m_create_info->tablespace, - page_size.physical(), - block_size_needed); - return(false); - } - - return(true); -} -#endif - -#ifdef MYSQL_COMPRESSION -/** Validate the COPMRESSION option. -@return true if valid, false if not. */ -bool -create_table_info_t::create_option_compression_is_valid() -{ - dberr_t err; - Compression compression; - - if (m_create_info->compress.length == 0) { - return(true); - } - - err = Compression::check(m_create_info->compress.str, &compression); - - if (err == DB_UNSUPPORTED) { - push_warning_printf( - m_thd, - Sql_condition::WARN_LEVEL_WARN, - ER_UNSUPPORTED_EXTENSION, - "InnoDB: Unsupported compression algorithm '%s'", - m_create_info->compress.str); - return(false); - } - - /* Allow Compression=NONE on any tablespace or row format. */ - if (compression.m_type == Compression::NONE) { - return(true); - } - - static char intro[] = "InnoDB: Page Compression is not supported"; - - if (m_create_info->key_block_size != 0 - || m_create_info->row_type == ROW_TYPE_COMPRESSED) { - push_warning_printf( - m_thd, Sql_condition::WARN_LEVEL_WARN, - ER_UNSUPPORTED_EXTENSION, - "%s with row_format=compressed or" - " key_block_size > 0", intro); - return(false); - } - - if (m_create_info->options & HA_LEX_CREATE_TMP_TABLE) { - push_warning_printf( - m_thd, Sql_condition::WARN_LEVEL_WARN, - HA_ERR_UNSUPPORTED, - "%s for temporary tables", intro); - return(false); - } - - if (tablespace_is_general_space(m_create_info)) { - push_warning_printf( - m_thd, Sql_condition::WARN_LEVEL_WARN, - HA_ERR_UNSUPPORTED, - "%s for shared general tablespaces", intro); - return(false); - } - - /* The only non-file-per-table tablespace left is the system space. */ - if (!m_use_file_per_table) { - push_warning_printf( - m_thd, Sql_condition::WARN_LEVEL_WARN, - HA_ERR_UNSUPPORTED, - "%s for the system tablespace", intro); - return(false); - } - - return(true); -} -#endif /* MYSQL_COMPRESSION */ - /** Validate the create options. Check that the options KEY_BLOCK_SIZE, ROW_FORMAT, DATA DIRECTORY, TEMPORARY & TABLESPACE are compatible with each other and other settings. These CREATE OPTIONS are not validated @@ -13108,29 +12187,24 @@ create_table_info_t::create_options_are_invalid() const char* ret = NULL; enum row_type row_format = m_create_info->row_type; + const bool is_temp + = m_create_info->options & HA_LEX_CREATE_TMP_TABLE; ut_ad(m_thd != NULL); - ut_ad(m_create_info != NULL); -#ifdef MYSQL_TABLESPACES - /* The TABLESPACE designation on a CREATE TABLE is not subject to - non-strict-mode. If it is incorrect or is incompatible with other - options, then we will return an error. Make sure the tablespace exists - and is compatible with this table */ - if (!create_option_tablespace_is_valid()) { - return("TABLESPACE"); - } -#endif - - /* If innodb_strict_mode is not set don't do any more validation. - Also, if this table is being put into a shared general tablespace - we ALWAYS act like strict mode is ON. */ - if (!m_use_shared_space && !(THDVAR(m_thd, strict_mode))) { + /* If innodb_strict_mode is not set don't do any more validation. */ + if (!THDVAR(m_thd, strict_mode)) { return(NULL); } /* Check if a non-zero KEY_BLOCK_SIZE was specified. */ if (has_key_block_size) { + if (is_temp) { + my_error(ER_UNSUPPORT_COMPRESSED_TEMPORARY_TABLE, + MYF(0)); + return("KEY_BLOCK_SIZE"); + } + switch (m_create_info->key_block_size) { ulint kbs_max; case 1: @@ -13156,11 +12230,6 @@ create_table_info_t::create_options_are_invalid() ret = "KEY_BLOCK_SIZE"; } - /* The following checks do not appy to shared tablespaces */ - if (m_use_shared_space) { - break; - } - /* Valid KEY_BLOCK_SIZE, check its dependencies. */ if (!m_allow_file_per_table) { push_warning( @@ -13195,47 +12264,39 @@ create_table_info_t::create_options_are_invalid() other incompatibilities. */ switch (row_format) { case ROW_TYPE_COMPRESSED: - if (!m_use_shared_space) { - if (!m_allow_file_per_table) { - push_warning_printf( - m_thd, Sql_condition::WARN_LEVEL_WARN, - ER_ILLEGAL_HA_CREATE_OPTION, - "InnoDB: ROW_FORMAT=%s requires" - " innodb_file_per_table.", - get_row_format_name(row_format)); - ret = "ROW_FORMAT"; - } - if (srv_file_format < UNIV_FORMAT_B) { - push_warning_printf( - m_thd, Sql_condition::WARN_LEVEL_WARN, - ER_ILLEGAL_HA_CREATE_OPTION, - "InnoDB: ROW_FORMAT=%s requires" - " innodb_file_format > Antelope.", - get_row_format_name(row_format)); - ret = "ROW_FORMAT"; - } + if (is_temp) { + my_error(ER_UNSUPPORT_COMPRESSED_TEMPORARY_TABLE, + MYF(0)); + return("ROW_FORMAT"); + } + if (!m_allow_file_per_table) { + push_warning_printf( + m_thd, Sql_condition::WARN_LEVEL_WARN, + ER_ILLEGAL_HA_CREATE_OPTION, + "InnoDB: ROW_FORMAT=%s requires" + " innodb_file_per_table.", + get_row_format_name(row_format)); + ret = "ROW_FORMAT"; + } + if (srv_file_format < UNIV_FORMAT_B) { + push_warning_printf( + m_thd, Sql_condition::WARN_LEVEL_WARN, + ER_ILLEGAL_HA_CREATE_OPTION, + "InnoDB: ROW_FORMAT=%s requires" + " innodb_file_format > Antelope.", + get_row_format_name(row_format)); + ret = "ROW_FORMAT"; } break; case ROW_TYPE_DYNAMIC: - if (!m_use_shared_space) { - if (!m_allow_file_per_table) { - push_warning_printf( - m_thd, Sql_condition::WARN_LEVEL_WARN, - ER_ILLEGAL_HA_CREATE_OPTION, - "InnoDB: ROW_FORMAT=%s requires" - " innodb_file_per_table.", - get_row_format_name(row_format)); - ret = "ROW_FORMAT"; - } - if (srv_file_format < UNIV_FORMAT_B) { - push_warning_printf( - m_thd, Sql_condition::WARN_LEVEL_WARN, - ER_ILLEGAL_HA_CREATE_OPTION, - "InnoDB: ROW_FORMAT=%s requires" - " innodb_file_format > Antelope.", - get_row_format_name(row_format)); - ret = "ROW_FORMAT"; - } + if (!is_temp && srv_file_format < UNIV_FORMAT_B) { + push_warning_printf( + m_thd, Sql_condition::WARN_LEVEL_WARN, + ER_ILLEGAL_HA_CREATE_OPTION, + "InnoDB: ROW_FORMAT=%s requires" + " innodb_file_format > Antelope.", + get_row_format_name(row_format)); + ret = "ROW_FORMAT"; } /* FALLTRHOUGH */ case ROW_TYPE_COMPACT: @@ -13293,25 +12354,6 @@ create_table_info_t::create_options_are_invalid() } } -#ifdef MYSQL_COMPRESSION - /* Validate the page compression parameter. */ - if (!create_option_compression_is_valid()) { - return("COMPRESSION"); - } - - /* Check the encryption option. */ - if (ret == NULL && m_create_info->encrypt_type.length > 0) { - dberr_t err; - - err = Encryption::validate(m_create_info->encrypt_type.str); - - if (err == DB_UNSUPPORTED) { - my_error(ER_INVALID_ENCRYPTION_OPTION, MYF(0)); - ret = "ENCRYPTION"; - } - } -#endif - return(ret); } @@ -13323,8 +12365,22 @@ create_table_info_t::check_table_options() { enum row_type row_format = m_form->s->row_type; ha_table_option_struct *options= m_form->s->option_struct; - atomic_writes_t awrites = (atomic_writes_t)options->atomic_writes; fil_encryption_t encrypt = (fil_encryption_t)options->encryption; + bool should_encrypt = (encrypt == FIL_SPACE_ENCRYPTION_ON); + + /* Currently we do not support encryption for + spatial indexes thus do not allow creating table with forced + encryption */ + for(ulint i = 0; i < m_form->s->keys; i++) { + const KEY* key = m_form->key_info + i; + if (key->flags & HA_SPATIAL && should_encrypt) { + push_warning_printf(m_thd, Sql_condition::WARN_LEVEL_WARN, + HA_ERR_UNSUPPORTED, + "InnoDB: ENCRYPTED=ON not supported for table because " + "it contains spatial index."); + return "ENCRYPTED"; + } + } if (encrypt != FIL_SPACE_ENCRYPTION_DEFAULT && !m_allow_file_per_table) { push_warning( @@ -13457,19 +12513,6 @@ create_table_info_t::check_table_options() } } - /* Check atomic writes requirements */ - if (awrites == ATOMIC_WRITES_ON || - (awrites == ATOMIC_WRITES_DEFAULT && srv_use_atomic_writes)) { - if (!m_allow_file_per_table) { - push_warning( - m_thd, Sql_condition::WARN_LEVEL_WARN, - HA_WRONG_CREATE_OPTION, - "InnoDB: ATOMIC_WRITES requires" - " innodb_file_per_table."); - return "ATOMIC_WRITES"; - } - } - return NULL; } @@ -13486,24 +12529,16 @@ ha_innobase::update_create_info( create_info->auto_increment_value = stats.auto_increment_value; } + if (dict_table_is_temporary(m_prebuilt->table)) { + return; + } + /* Update the DATA DIRECTORY name from SYS_DATAFILES. */ dict_get_and_save_data_dir_path(m_prebuilt->table, false); if (m_prebuilt->table->data_dir_path) { create_info->data_file_name = m_prebuilt->table->data_dir_path; } - - /* Update the TABLESPACE name from the Data Dictionary. */ - dict_get_and_save_space_name(m_prebuilt->table, false); - - /* Put this tablespace name into the create_info structure so that - SHOW CREATE TABLE will display TABLESPACE=name. This also affects - an ALTER TABLE which must know the current TABLESPACE so that the - table will stay there. */ - if (m_prebuilt->table->tablespace != NULL - && create_info->tablespace == NULL) { - create_info->tablespace = m_prebuilt->table->tablespace; - } } /*****************************************************************//** @@ -13522,8 +12557,7 @@ innobase_fts_load_stopword( THDVAR(thd, ft_enable_stopword), FALSE)); } -/** Parse the table name into normal name and either temp path or remote path -if needed. +/** Parse the table name into normal name and remote path if needed. @param[in] name Table name (db/table or full path). @return 0 if successful, otherwise, error number */ int @@ -13556,18 +12590,7 @@ create_table_info_t::parse_table_name( } #endif - m_temp_path[0] = '\0'; m_remote_path[0] = '\0'; - m_tablespace[0] = '\0'; - - /* A full path is provided by the server for TEMPORARY tables not - targeted for a tablespace or when DATA DIRECTORY is given. - So these two are not compatible. Likewise, DATA DIRECTORY is not - compatible with a TABLESPACE assignment. */ - if ((m_create_info->options & HA_LEX_CREATE_TMP_TABLE) - && !m_use_shared_space) { - strncpy(m_temp_path, name, FN_REFLEN - 1); - } /* Make sure DATA DIRECTORY is compatible with other options and set the remote path. In the case of either; @@ -13596,21 +12619,11 @@ create_table_info_t::parse_table_name( "INDEX DIRECTORY"); } - /* The TABLESPACE designation has already been validated by - create_option_tablespace_is_valid() irregardless of strict-mode. - So it only needs to be copied now. */ - if (m_use_shared_space) { - strncpy(m_tablespace, m_create_info->tablespace, - NAME_LEN - 1); - } - DBUG_RETURN(0); } /** Determine InnoDB table flags. If strict_mode=OFF, this will adjust the flags to what should be assumed. -However, if an existing general tablespace is being targeted, we will NOT -assume anything or adjust these flags. @retval true if successful, false if error */ bool create_table_info_t::innobase_table_flags() @@ -13618,11 +12631,14 @@ create_table_info_t::innobase_table_flags() DBUG_ENTER("innobase_table_flags"); const char* fts_doc_id_index_bad = NULL; - bool zip_allowed = true; ulint zip_ssize = 0; enum row_type row_type; rec_format_t innodb_row_format = get_row_format(innodb_default_row_format); + const bool is_temp + = m_create_info->options & HA_LEX_CREATE_TMP_TABLE; + bool zip_allowed + = !is_temp; const ulint zip_ssize_max = ut_min(static_cast<ulint>(UNIV_PAGE_SSIZE_MAX), @@ -13641,40 +12657,6 @@ create_table_info_t::innobase_table_flags() m_flags = 0; m_flags2 = 0; -#ifdef MYSQL_COMPRESSION - /* Validate the page compression parameter. */ - if (!create_option_compression_is_valid()) { - /* No need to do anything. Warnings were issued. - The compresion setting will be ignored later. - If inodb_strict_mode=ON, this is called twice unless - there was a problem before. - If inodb_strict_mode=OFF, this is the only call. */ - } -#endif - -#ifdef MYSQL_ENCRYPTION - /* Validate the page encryption parameter. */ - if (m_create_info->encrypt_type.length > 0) { - - const char* encryption = m_create_info->encrypt_type.str; - - if (Encryption::validate(encryption) != DB_SUCCESS) { - /* Incorrect encryption option */ - my_error(ER_INVALID_ENCRYPTION_OPTION, MYF(0)); - DBUG_RETURN(false); - } - - if (m_use_shared_space - || (m_create_info->options & HA_LEX_CREATE_TMP_TABLE)) { - if (!Encryption::is_none(encryption)) { - /* Can't encrypt shared tablespace */ - my_error(ER_TABLESPACE_CANNOT_ENCRYPT, MYF(0)); - DBUG_RETURN(false); - } - } - } -#endif /* MYSQL_ENCRYPTION */ - /* Check if there are any FTS indexes defined on this table. */ for (uint i = 0; i < m_form->s->keys; i++) { const KEY* key = &m_form->key_info[i]; @@ -13684,8 +12666,7 @@ create_table_info_t::innobase_table_flags() /* We don't support FTS indexes in temporary tables. */ - if (m_create_info->options & HA_LEX_CREATE_TMP_TABLE) { - + if (is_temp) { my_error(ER_INNODB_NO_FT_TEMP_TABLE, MYF(0)); DBUG_RETURN(false); } @@ -13693,12 +12674,6 @@ create_table_info_t::innobase_table_flags() if (fts_doc_id_index_bad) { goto index_bad; } - } else if (key->flags & HA_SPATIAL) { - if (m_create_info->options & HA_LEX_CREATE_TMP_TABLE - && !m_use_file_per_table) { - my_error(ER_TABLE_CANT_HANDLE_SPKEYS, MYF(0)); - DBUG_RETURN(false); - } } if (innobase_strcasecmp(key->name, FTS_DOC_ID_INDEX_NAME)) { @@ -13721,8 +12696,6 @@ index_bad: } } - //rec_format_t row_format = m_form->s->row_type; - if (m_create_info->key_block_size > 0) { /* The requested compressed page size (key_block_size) is given in kilobytes. If it is a valid number, store @@ -13740,7 +12713,14 @@ index_bad: } /* Make sure compressed row format is allowed. */ - if (!m_allow_file_per_table && !m_use_shared_space) { + if (is_temp) { + push_warning( + m_thd, Sql_condition::WARN_LEVEL_WARN, + ER_ILLEGAL_HA_CREATE_OPTION, + "InnoDB: KEY_BLOCK_SIZE is ignored" + " for TEMPORARY TABLE."); + zip_allowed = false; + } else if (!m_allow_file_per_table) { push_warning( m_thd, Sql_condition::WARN_LEVEL_WARN, ER_ILLEGAL_HA_CREATE_OPTION, @@ -13749,8 +12729,7 @@ index_bad: zip_allowed = false; } - if (file_format_allowed < UNIV_FORMAT_B - && !m_use_shared_space) { + if (file_format_allowed < UNIV_FORMAT_B) { push_warning( m_thd, Sql_condition::WARN_LEVEL_WARN, ER_ILLEGAL_HA_CREATE_OPTION, @@ -13811,37 +12790,29 @@ index_bad: innodb_row_format = REC_FORMAT_COMPACT; break; case ROW_TYPE_COMPRESSED: - /* ROW_FORMAT=COMPRESSED requires file_per_table and - file_format=Barracuda unless there is a target tablespace. */ - if (!m_allow_file_per_table - && !m_use_shared_space) { + if (is_temp) { + push_warning_printf( + m_thd, Sql_condition::WARN_LEVEL_WARN, + ER_ILLEGAL_HA_CREATE_OPTION, + "InnoDB: ROW_FORMAT=%s is ignored for" + " TEMPORARY TABLE.", + get_row_format_name(row_type)); + } else if (!m_allow_file_per_table) { push_warning_printf( m_thd, Sql_condition::WARN_LEVEL_WARN, ER_ILLEGAL_HA_CREATE_OPTION, "InnoDB: ROW_FORMAT=COMPRESSED requires" " innodb_file_per_table."); - } else if (file_format_allowed == UNIV_FORMAT_A - && !m_use_shared_space) { + } else if (file_format_allowed == UNIV_FORMAT_A) { push_warning_printf( m_thd, Sql_condition::WARN_LEVEL_WARN, ER_ILLEGAL_HA_CREATE_OPTION, "InnoDB: ROW_FORMAT=COMPRESSED requires" " innodb_file_format > Antelope."); - } else { - switch(row_type) { - case ROW_TYPE_COMPRESSED: - innodb_row_format = REC_FORMAT_COMPRESSED; - break; - case ROW_TYPE_DYNAMIC: - innodb_row_format = REC_FORMAT_DYNAMIC; - break; - default: - /* Not possible, avoid compiler warning */ - break; - } - break; /* Correct row_format */ + innodb_row_format = REC_FORMAT_COMPRESSED; + break; } zip_allowed = false; /* fall through to set row_type = DYNAMIC */ @@ -13869,43 +12840,27 @@ index_bad: zip_allowed = false; } + ut_ad(!is_temp || !zip_allowed); + ut_ad(!is_temp || innodb_row_format != REC_FORMAT_COMPRESSED); + /* Set the table flags */ if (!zip_allowed) { zip_ssize = 0; } - if (m_create_info->options & HA_LEX_CREATE_TMP_TABLE) { + if (is_temp) { m_flags2 |= DICT_TF2_TEMPORARY; - /* Intrinsic tables reside only in the shared temporary - tablespace and we will always use ROW_FORMAT=DYNAMIC. */ - -#ifdef MYSQL_COMPRESSION - if ((m_create_info->options & HA_LEX_CREATE_INTERNAL_TMP_TABLE) - && !m_use_file_per_table) { - - /* We do not allow compressed instrinsic - temporary tables. */ - - ut_ad(zip_ssize == 0); - m_flags2 |= DICT_TF2_INTRINSIC; - innodb_row_format = REC_FORMAT_DYNAMIC; - } -#endif - + } else if (m_use_file_per_table) { + m_flags2 |= DICT_TF2_USE_FILE_PER_TABLE; } /* Set the table flags */ dict_tf_set(&m_flags, innodb_row_format, zip_ssize, - m_use_data_dir, m_use_shared_space, + m_use_data_dir, options->page_compressed, options->page_compression_level == 0 ? - default_compression_level : options->page_compression_level, - options->atomic_writes); - - if (m_use_file_per_table) { - ut_ad(!m_use_shared_space); - m_flags2 |= DICT_TF2_USE_FILE_PER_TABLE; - } + default_compression_level : static_cast<ulint>(options->page_compression_level), + 0); /* Set the flags2 when create table or alter tables */ m_flags2 |= DICT_TF2_FTS_AUX_HEX_NAME; @@ -14089,42 +13044,25 @@ void create_table_info_t::set_tablespace_type( bool table_being_altered_is_file_per_table) { - /* Note whether this table will be created using a shared, - general or system tablespace. */ - m_use_shared_space = tablespace_is_shared_space(m_create_info); - /** Allow file_per_table for this table either because: 1) the setting innodb_file_per_table=on, - 2) the table being altered is currently file_per_table - 3) explicitly requested by tablespace=innodb_file_per_table. */ + 2) the table being altered is currently file_per_table */ m_allow_file_per_table = m_innodb_file_per_table - || table_being_altered_is_file_per_table - || tablespace_is_file_per_table(m_create_info); - - /* All noncompresed temporary tables will be put into the - system temporary tablespace. */ - bool is_noncompressed_temporary = - m_create_info->options & HA_LEX_CREATE_TMP_TABLE - && !(m_create_info->row_type == ROW_TYPE_COMPRESSED - || m_create_info->key_block_size > 0); + || table_being_altered_is_file_per_table; /* Ignore the current innodb-file-per-table setting if we are - creating a temporary, non-compressed table or if the - TABLESPACE= phrase is using an existing shared tablespace. */ + creating a temporary table. */ m_use_file_per_table = m_allow_file_per_table - && !is_noncompressed_temporary - && !m_use_shared_space; + && !(m_create_info->options & HA_LEX_CREATE_TMP_TABLE); /* DATA DIRECTORY must have m_use_file_per_table but cannot be used with TEMPORARY tables. */ m_use_data_dir = m_use_file_per_table - && !(m_create_info->options & HA_LEX_CREATE_TMP_TABLE) && (m_create_info->data_file_name != NULL) && (m_create_info->data_file_name[0] != '\0'); - ut_ad(!(m_use_shared_space && m_use_data_dir)); } /** Initialize the create_table_info_t object. @@ -14165,6 +13103,30 @@ create_table_info_t::initialize() } +/** Check if a virtual column is part of a fulltext or spatial index. */ +bool +create_table_info_t::gcols_in_fulltext_or_spatial() +{ + for (ulint i = 0; i < m_form->s->keys; i++) { + const KEY* key = m_form->key_info + i; + if (key->flags & (HA_SPATIAL | HA_FULLTEXT)) { + for (ulint j = 0; j < key->user_defined_key_parts; j++) { + const KEY_PART_INFO* key_part = key->key_part + j; + + /* We do not support special (Fulltext or + Spatial) index on virtual columns */ + if (innobase_is_v_fld(key_part->field)) { + my_error(ER_UNSUPPORTED_ACTION_ON_GENERATED_COLUMN, MYF(0)); + return true; + } + } + } + + } + return false; +} + + /** Prepare to create a new table to an InnoDB database. @param[in] name Table name @return error number */ @@ -14202,10 +13164,14 @@ create_table_info_t::prepare_create_table( DBUG_RETURN(HA_WRONG_CREATE_OPTION); } - if (high_level_read_only && !is_intrinsic_temp_table()) { + if (high_level_read_only) { DBUG_RETURN(HA_ERR_TABLE_READONLY); } + if (gcols_in_fulltext_or_spatial()) { + DBUG_RETURN(HA_ERR_UNSUPPORTED); + } + DBUG_RETURN(parse_table_name(name)); } @@ -14346,17 +13312,7 @@ create_table_info_t::create_table() stmt = innobase_get_stmt_unsafe(m_thd, &stmt_len); - innodb_session_t*& priv = - thd_to_innodb_session(m_trx->mysql_thd); - dict_table_t* handler = - priv->lookup_table_handler(m_table_name); - - ut_ad(handler == NULL - || (handler != NULL && dict_table_is_intrinsic(handler))); - - /* There is no concept of foreign key for intrinsic tables. */ - if (stmt && (handler == NULL)) { - + if (stmt) { dberr_t err = row_table_add_foreign_constraints( m_trx, stmt, stmt_len, m_table_name, m_create_info->options & HA_LEX_CREATE_TMP_TABLE); @@ -14382,19 +13338,15 @@ create_table_info_t::create_table() " table where referencing columns appear" " as the first columns.\n", m_table_name); break; -#ifdef MYSQL_VIRTUAL_COLUMNS - case DB_NO_FK_ON_V_BASE_COL: + case DB_NO_FK_ON_S_BASE_COL: push_warning_printf( m_thd, Sql_condition::WARN_LEVEL_WARN, HA_ERR_CANNOT_ADD_FOREIGN, "Create table '%s' with foreign key constraint" " failed. Cannot add foreign key constraint" - " placed on the base column of indexed" - " virtual column, or constraint placed" - " on columns being part of virtual index.\n", + " placed on the base column of stored" + " column. \n", m_table_name); - break; -#endif default: break; } @@ -14402,23 +13354,15 @@ create_table_info_t::create_table() error = convert_error_code_to_mysql(err, m_flags, NULL); if (error) { - if (handler != NULL) { - priv->unregister_table_handler(m_table_name); - } DBUG_RETURN(error); } } - if (!is_intrinsic_temp_table()) { - innobase_table = dict_table_open_on_name( - m_table_name, TRUE, FALSE, DICT_ERR_IGNORE_NONE); - - if (innobase_table != NULL) { - dict_table_close(innobase_table, TRUE, FALSE); - } + innobase_table = dict_table_open_on_name( + m_table_name, TRUE, FALSE, DICT_ERR_IGNORE_NONE); - } else { - innobase_table = NULL; + if (innobase_table != NULL) { + dict_table_close(innobase_table, TRUE, FALSE); } DBUG_RETURN(0); @@ -14433,17 +13377,8 @@ create_table_info_t::create_table_update_dict() DBUG_ENTER("create_table_update_dict"); - innobase_table = thd_to_innodb_session(m_thd)->lookup_table_handler( - m_table_name); - - - if (innobase_table == NULL) { - innobase_table = dict_table_open_on_name( - m_table_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); - } else { - innobase_table->acquire(); - ut_ad(dict_table_is_intrinsic(innobase_table)); - } + innobase_table = dict_table_open_on_name( + m_table_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); DBUG_ASSERT(innobase_table != 0); if (innobase_table->fts != NULL) { @@ -14486,34 +13421,40 @@ create_table_info_t::create_table_update_dict() } } - /* Note: We can't call update_thd() as m_prebuilt will not be - setup at this stage and so we use thd. */ - - /* We need to copy the AUTOINC value from the old table if - this is an ALTER|OPTIMIZE TABLE or CREATE INDEX because CREATE INDEX - does a table copy too. If query was one of : + if (const Field* ai = m_form->found_next_number_field) { + ut_ad(!innobase_is_v_fld(ai)); - CREATE TABLE ...AUTO_INCREMENT = x; or - ALTER TABLE...AUTO_INCREMENT = x; or - OPTIMIZE TABLE t; or - CREATE INDEX x on t(...); + ib_uint64_t autoinc = m_create_info->auto_increment_value; - Find out a table definition from the dictionary and get - the current value of the auto increment field. Set a new - value to the auto increment field if the value is greater - than the maximum value in the column. */ + if (autoinc == 0) { + autoinc = 1; + } - if (((m_create_info->used_fields & HA_CREATE_USED_AUTO) - || thd_sql_command(m_thd) == SQLCOM_ALTER_TABLE - || thd_sql_command(m_thd) == SQLCOM_OPTIMIZE - || thd_sql_command(m_thd) == SQLCOM_CREATE_INDEX) - && m_create_info->auto_increment_value > 0) { - ib_uint64_t auto_inc_value; + dict_table_autoinc_lock(innobase_table); + dict_table_autoinc_initialize(innobase_table, autoinc); - auto_inc_value = m_create_info->auto_increment_value; + if (dict_table_is_temporary(innobase_table)) { + /* AUTO_INCREMENT is not persistent for + TEMPORARY TABLE. Temporary tables are never + evicted. Keep the counter in memory only. */ + } else { + const unsigned col_no = innodb_col_no(ai); + + innobase_table->persistent_autoinc = 1 + + dict_table_get_nth_col_pos( + innobase_table, col_no, NULL); + + /* Persist the "last used" value, which + typically is AUTO_INCREMENT - 1. + In btr_create(), the value 0 was already written. */ + if (--autoinc) { + btr_write_autoinc( + dict_table_get_first_index( + innobase_table), + autoinc); + } + } - dict_table_autoinc_lock(innobase_table); - dict_table_autoinc_initialize(innobase_table, auto_inc_value); dict_table_autoinc_unlock(innobase_table); } @@ -14547,9 +13488,7 @@ ha_innobase::create( { int error; char norm_name[FN_REFLEN]; /* {database}/{tablename} */ - char temp_path[FN_REFLEN]; /* Absolute path of temp frm */ char remote_path[FN_REFLEN]; /* Absolute path of table */ - char tablespace[NAME_LEN]; /* Tablespace name identifier */ trx_t* trx; DBUG_ENTER("ha_innobase::create"); @@ -14557,9 +13496,7 @@ ha_innobase::create( form, create_info, norm_name, - temp_path, - remote_path, - tablespace); + remote_path); /* Initialize the object. */ if ((error = info.initialize())) { @@ -14577,13 +13514,8 @@ ha_innobase::create( /* Latch the InnoDB data dictionary exclusively so that no deadlocks or lock waits can happen in it during a table create operation. - Drop table etc. do this latching in row0mysql.cc. - Avoid locking dictionary if table is intrinsic. - Table Object for such table is cached in THD instead of storing it - to dictionary. */ - if (!info.is_intrinsic_temp_table()) { - row_mysql_lock_data_dictionary(trx); - } + Drop table etc. do this latching in row0mysql.cc. */ + row_mysql_lock_data_dictionary(trx); if ((error = info.create_table())) { goto cleanup; @@ -14591,14 +13523,12 @@ ha_innobase::create( innobase_commit_low(trx); - if (!info.is_intrinsic_temp_table()) { - ut_ad(!srv_read_only_mode); - row_mysql_unlock_data_dictionary(trx); - /* Flush the log to reduce probability that the .frm files and - the InnoDB data dictionary get out-of-sync if the user runs - with innodb_flush_log_at_trx_commit = 0 */ - log_buffer_flush_to_disk(); - } + ut_ad(!srv_read_only_mode); + row_mysql_unlock_data_dictionary(trx); + /* Flush the log to reduce probability that the .frm files and + the InnoDB data dictionary get out-of-sync if the user runs + with innodb_flush_log_at_trx_commit = 0 */ + log_buffer_flush_to_disk(); error = info.create_table_update_dict(); @@ -14613,38 +13543,7 @@ ha_innobase::create( cleanup: trx_rollback_for_mysql(trx); - - if (!info.is_intrinsic_temp_table()) { - row_mysql_unlock_data_dictionary(trx); - } else { - THD* thd = info.thd(); - - dict_table_t* intrinsic_table = - thd_to_innodb_session(thd)->lookup_table_handler( - info.table_name()); - - if (intrinsic_table != NULL) { - thd_to_innodb_session(thd)->unregister_table_handler( - info.table_name()); - - for (;;) { - dict_index_t* index; - index = UT_LIST_GET_FIRST( - intrinsic_table->indexes); - if (index == NULL) { - break; - } - rw_lock_free(&index->lock); - UT_LIST_REMOVE(intrinsic_table->indexes, index); - dict_mem_index_free(index); - index = NULL; - } - - dict_mem_table_free(intrinsic_table); - intrinsic_table = NULL; - } - } - + row_mysql_unlock_data_dictionary(trx); trx_free_for_mysql(trx); DBUG_RETURN(error); @@ -14690,16 +13589,6 @@ ha_innobase::discard_or_import_tablespace( DBUG_RETURN(HA_ERR_TABLE_NEEDS_UPGRADE); } - if (DICT_TF_HAS_SHARED_SPACE(dict_table->flags)) { - my_printf_error(ER_NOT_ALLOWED_COMMAND, - "InnoDB: Cannot %s table `%s` because it is in" - " a general tablespace. It must be file-per-table.", - MYF(0), discard ? "discard" : "import", - dict_table->name.m_name); - - DBUG_RETURN(HA_ERR_NOT_ALLOWED_COMMAND); - } - TrxInInnoDB trx_in_innodb(m_prebuilt->trx); if (trx_in_innodb.is_aborted()) { @@ -14799,11 +13688,6 @@ ha_innobase::truncate() { DBUG_ENTER("ha_innobase::truncate"); - /* Truncate of intrinsic table is not allowed truncate for now. */ - if (dict_table_is_intrinsic(m_prebuilt->table)) { - DBUG_RETURN(HA_ERR_WRONG_COMMAND); - } - if (high_level_read_only) { DBUG_RETURN(HA_ERR_TABLE_READONLY); } @@ -14882,17 +13766,7 @@ ha_innobase::delete_table( extension, in contrast to ::create */ normalize_table_name(norm_name, name); - innodb_session_t*& priv = thd_to_innodb_session(thd); - dict_table_t* handler = priv->lookup_table_handler(norm_name); - - if (handler != NULL) { - for (dict_index_t* index = UT_LIST_GET_FIRST(handler->indexes); - index != NULL; - index = UT_LIST_GET_NEXT(indexes, index)) { - index->last_ins_cur->release(); - index->last_sel_cur->release(); - } - } else if (srv_read_only_mode) { + if (srv_read_only_mode) { DBUG_RETURN(HA_ERR_TABLE_READONLY); } @@ -14938,16 +13812,11 @@ ha_innobase::delete_table( err = row_drop_table_for_mysql( norm_name, trx, thd_sql_command(thd) == SQLCOM_DROP_DB, - false, handler); + false); if (err == DB_TABLE_NOT_FOUND && innobase_get_lower_case_table_names() == 1) { - char* is_part = NULL; -#ifdef __WIN__ - is_part = strstr(norm_name, "#p#"); -#else - is_part = strstr(norm_name, "#P#"); -#endif /* __WIN__ */ + char* is_part = is_partition(norm_name); if (is_part) { char par_case_name[FN_REFLEN]; @@ -14997,9 +13866,9 @@ ha_innobase::delete_table( tbl_name.m_name = norm_name; ib::error() << "Table " << tbl_name << " does not exist in the InnoDB" - " internal data dictionary though MySQL is" + " internal data dictionary though MariaDB is" " trying to drop it. Have you copied the .frm" - " file of the table to the MySQL database" + " file of the table to the MariaDB database" " directory from another database? " << TROUBLESHOOTING_MSG; } @@ -15012,11 +13881,7 @@ ha_innobase::delete_table( native innodb partitioning is completed */ if (err == DB_TABLE_NOT_FOUND && innobase_get_lower_case_table_names() == 1) { -#ifdef _WIN32 - char* is_part = strstr(norm_name, "#p#"); -#else - char* is_part = strstr(norm_name, "#P#"); -#endif /* _WIN32 */ + char* is_part = is_partition(norm_name); if (is_part != NULL) { char par_case_name[FN_REFLEN]; @@ -15038,20 +13903,16 @@ ha_innobase::delete_table( err = row_drop_table_for_mysql( par_case_name, trx, thd_sql_command(thd) == SQLCOM_DROP_DB, - true, handler); + true); } } - if (handler == NULL) { - ut_ad(!srv_read_only_mode); - /* Flush the log to reduce probability that the .frm files and - the InnoDB data dictionary get out-of-sync if the user runs - with innodb_flush_log_at_trx_commit = 0 */ + ut_ad(!srv_read_only_mode); + /* Flush the log to reduce probability that the .frm files and + the InnoDB data dictionary get out-of-sync if the user runs + with innodb_flush_log_at_trx_commit = 0 */ - log_buffer_flush_to_disk(); - } else if (err == DB_SUCCESS) { - priv->unregister_table_handler(norm_name); - } + log_buffer_flush_to_disk(); innobase_commit_low(trx); @@ -15060,411 +13921,6 @@ ha_innobase::delete_table( DBUG_RETURN(convert_error_code_to_mysql(err, 0, NULL)); } -#ifdef MYSQL_TABLESPACES -/** Validate the parameters in st_alter_tablespace -before using them in InnoDB tablespace functions. -@param[in] thd Connection -@param[in] alter_info How to do the command. -@return MySQL handler error code like HA_... */ -static -int -validate_create_tablespace_info( - THD* thd, - st_alter_tablespace* alter_info) -{ - - int error = 0; - - /* The parser ensures that these fields are provided. */ - ut_a(alter_info->tablespace_name); - ut_a(alter_info->data_file_name); - - if (high_level_read_only) { - return(HA_ERR_INNODB_READ_ONLY); - } - - /* From this point forward, push a warning for each problem found - instead of returning immediately*/ - int error = validate_tablespace_name( - alter_info->tablespace_name, false); - - /* Make sure the tablespace is not already open. */ - space_id = fil_space_get_id_by_name(alter_info->tablespace_name); - if (space_id != ULINT_UNDEFINED) { - my_printf_error(ER_TABLESPACE_EXISTS, - "InnoDB: A tablespace named `%s`" - " already exists.", MYF(0), - alter_info->tablespace_name); - error = HA_ERR_TABLESPACE_EXISTS; - } - if (alter_info->file_block_size) { - /* Check for a bad file block size. */ - if (!ut_is_2pow(alter_info->file_block_size) - || alter_info->file_block_size < UNIV_ZIP_SIZE_MIN - || alter_info->file_block_size > UNIV_PAGE_SIZE_MAX) { - my_printf_error(ER_ILLEGAL_HA_CREATE_OPTION, - "InnoDB does not support" - " FILE_BLOCK_SIZE=%llu", MYF(0), - alter_info->file_block_size); - error = HA_WRONG_CREATE_OPTION; - - /* Don't allow a file block size larger than UNIV_PAGE_SIZE. */ - } else if (alter_info->file_block_size > UNIV_PAGE_SIZE) { - my_printf_error(ER_ILLEGAL_HA_CREATE_OPTION, - "InnoDB: Cannot create a tablespace" - " with FILE_BLOCK_SIZE=%llu because" - " INNODB_PAGE_SIZE=%lu.", MYF(0), - alter_info->file_block_size, - UNIV_PAGE_SIZE); - error = HA_WRONG_CREATE_OPTION; - - /* Don't allow a compressed tablespace when page size > 16k. */ - } else if (UNIV_PAGE_SIZE > UNIV_PAGE_SIZE_DEF - && alter_info->file_block_size != UNIV_PAGE_SIZE) { - my_printf_error(ER_ILLEGAL_HA_CREATE_OPTION, - "InnoDB: Cannot create a COMPRESSED" - " tablespace when innodb_page_size >" - " 16k.", MYF(0)); - error = HA_WRONG_CREATE_OPTION; - } - } - - /* Validate the ADD DATAFILE name. */ - char* filepath = mem_strdup(alter_info->data_file_name); - os_normalize_path(filepath); - - /* It must end with '.ibd' and contain a basename of at least - 1 character before the.ibd extension. */ - ulint dirname_len = dirname_length(filepath); - const char* basename = filepath + dirname_len; - ulint basename_len = strlen(basename); - if (basename_len < 5) { - my_error(ER_WRONG_FILE_NAME, MYF(0), - alter_info->data_file_name); - ut_free(filepath); - return(HA_WRONG_CREATE_OPTION); - } - - if (memcmp(&basename[basename_len - 4], DOT_IBD, 5)) { - my_error(ER_WRONG_FILE_NAME, MYF(0), - alter_info->data_file_name); - my_printf_error(ER_WRONG_FILE_NAME, - "An IBD filepath must end with `.ibd`.", - MYF(0)); - ut_free(filepath); - return(HA_WRONG_CREATE_OPTION); - } - - /* Do not allow an invalid colon in the file name. */ - const char* colon = strchr(filepath, ':'); - if (colon != NULL) { -#ifdef _WIN32 - /* Do not allow names like "C:name.ibd" because it - specifies the "C:" drive but allows a relative location. - It should be like "c:\". If a single colon is used it must - be the second byte the the third byte must be a separator. */ - if (colon != &filepath[1] - || (colon[1] != OS_PATH_SEPARATOR) - || NULL != strchr(&colon[1], ':')) { -#endif /* _WIN32 */ - my_error(ER_WRONG_FILE_NAME, MYF(0), - alter_info->data_file_name); - my_printf_error(ER_WRONG_FILE_NAME, - "Invalid use of ':'.", MYF(0)); - ut_free(filepath); - return(HA_WRONG_CREATE_OPTION); -#ifdef _WIN32 - } -#endif /* _WIN32 */ - } - -#ifndef _WIN32 - /* On Non-Windows platforms, '\\' is a valid file name character. - But for InnoDB datafiles, we always assume it is a directory - separator and convert these to '/' */ - if (strchr(alter_info->data_file_name, '\\') != NULL) { - ib::warn() << "Converting backslash to forward slash in" - " ADD DATAFILE " << alter_info->data_file_name; - } -#endif /* _WIN32 */ - - /* The directory path must be pre-existing. */ - Folder folder(filepath, dirname_len); - ut_free(filepath); - if (!folder.exists()) { - my_error(ER_WRONG_FILE_NAME, MYF(0), - alter_info->data_file_name); - my_printf_error(ER_WRONG_FILE_NAME, - "The directory does not exist.", MYF(0)); - return(HA_WRONG_CREATE_OPTION); - } - - /* CREATE TABLESPACE...ADD DATAFILE can be inside but not under - the datadir.*/ - if (folder_mysql_datadir > folder) { - my_error(ER_WRONG_FILE_NAME, MYF(0), - alter_info->data_file_name); - my_printf_error(ER_WRONG_FILE_NAME, - "CREATE TABLESPACE data file" - " cannot be under the datadir.", MYF(0)); - error = HA_WRONG_CREATE_OPTION; - } - - return(error); -} - -/** CREATE a tablespace. -@param[in] hton Handlerton of InnoDB -@param[in] thd Connection -@param[in] alter_info How to do the command -@return MySQL error code*/ -static -int -innobase_create_tablespace( - handlerton* hton, - THD* thd, - st_alter_tablespace* alter_info) -{ - trx_t* trx; - int error=0; - Tablespace tablespace; - - DBUG_ENTER("innobase_create_tablespace"); - DBUG_ASSERT(hton == innodb_hton_ptr); - - /* Be sure the input parameters are valid before continuing. */ - error = validate_create_tablespace_info(thd, alter_info); - if (error) { - DBUG_RETURN(error); - } - - /* Create the tablespace object. */ - tablespace.set_name(alter_info->tablespace_name); - - dberr_t err = tablespace.add_datafile(alter_info->data_file_name); - if (err != DB_SUCCESS) { - DBUG_RETURN(convert_error_code_to_mysql(err, 0, NULL)); - } - - /* Get the transaction associated with the current thd and make - sure it will not block this DDL. */ - trx_t* parent_trx = check_trx_exists(thd); - - /* In case MySQL calls this in the middle of a SELECT - query, release possible adaptive hash latch to avoid - deadlocks of threads */ - trx_search_latch_release_if_reserved(parent_trx); - - /* Allocate a new transaction for this DDL */ - trx = innobase_trx_allocate(thd); - ++trx->will_lock; - - trx_start_if_not_started(trx, true); - row_mysql_lock_data_dictionary(trx); - - /* In FSP_FLAGS, a zip_ssize of zero means that the tablespace - holds non-compresssed tables. A non-zero zip_ssize means that - the general tablespace can ONLY contain compressed tables. */ - ulint zip_size = static_cast<ulint>(alter_info->file_block_size); - ut_ad(zip_size <= UNIV_PAGE_SIZE_MAX); - if (zip_size == 0) { - zip_size = UNIV_PAGE_SIZE; - } - - bool zipped = (zip_size != UNIV_PAGE_SIZE); - page_size_t page_size(zip_size, UNIV_PAGE_SIZE, zipped); - bool atomic_blobs = page_size.is_compressed(); - - /* Create the filespace flags */ - ulint fsp_flags = fsp_flags_init( - page_size, /* page sizes and a flag if compressed */ - atomic_blobs, /* needed only for compressed tables */ - false, /* This is not a file-per-table tablespace */ - true, /* This is a general shared tablespace */ - false, /* Temporary General Tablespaces not allowed */ - false, /* Page compression is not used. */ - 0, /* Page compression level 0 */ - ATOMIC_WRITES_DEFAULT); /* No atomic writes yet */ - - tablespace.set_flags(fsp_flags); - - err = dict_build_tablespace(&tablespace); - if (err != DB_SUCCESS) { - error = convert_error_code_to_mysql(err, 0, NULL); - trx_rollback_for_mysql(trx); - goto cleanup; - } - - innobase_commit_low(trx); - -cleanup: - row_mysql_unlock_data_dictionary(trx); - trx_free_for_mysql(trx); - - DBUG_RETURN(error); -} - -/** DROP a tablespace. -@param[in] hton Handlerton of InnoDB -@param[in] thd Connection -@param[in] alter_info How to do the command -@return MySQL error code*/ -static -int -innobase_drop_tablespace( - handlerton* hton, - THD* thd, - st_alter_tablespace* alter_info) -{ - trx_t* trx; - dberr_t err; - int error = 0; - ulint space_id; - - DBUG_ENTER("innobase_drop_tablespace"); - DBUG_ASSERT(hton == innodb_hton_ptr); - - if (srv_read_only_mode) { - DBUG_RETURN(HA_ERR_INNODB_READ_ONLY); - } - - error = validate_tablespace_name(alter_info->tablespace_name, false); - if (error != 0) { - DBUG_RETURN(error); - } - - /* Be sure that this tablespace is known and valid. */ - space_id = fil_space_get_id_by_name(alter_info->tablespace_name); - if (space_id == ULINT_UNDEFINED) { - - space_id = dict_space_get_id(alter_info->tablespace_name); - if (space_id == ULINT_UNDEFINED) { - DBUG_RETURN(HA_ERR_NO_SUCH_TABLE); - // DBUG_RETURN(HA_ERR_TABLESPACE_MISSING); - } - - /* The datafile is not open but the tablespace is in - sys_tablespaces, so we can try to drop the metadata. */ - } - - /* The tablespace can only be dropped if it is empty. */ - if (!dict_space_is_empty(space_id)) { - DBUG_RETURN(HA_ERR_TABLE_READONLY); - //DBUG_RETURN(HA_ERR_TABLESPACE_IS_NOT_EMPTY); - } - - /* Get the transaction associated with the current thd and make sure - it will not block this DDL. */ - trx_t* parent_trx = check_trx_exists(thd); - - /* In case MySQL calls this in the middle of a SELECT - query, release possible adaptive hash latch to avoid - deadlocks of threads */ - trx_search_latch_release_if_reserved(parent_trx); - - /* Allocate a new transaction for this DDL */ - trx = innobase_trx_allocate(thd); - ++trx->will_lock; - - trx_start_if_not_started(trx, true); - row_mysql_lock_data_dictionary(trx); - - /* Update SYS_TABLESPACES and SYS_DATAFILES */ - err = dict_delete_tablespace_and_datafiles(space_id, trx); - if (err != DB_SUCCESS) { - ib::error() << "Unable to delete the dictionary entries" - " for tablespace `" << alter_info->tablespace_name - << "`, Space ID " << space_id; - goto have_error; - } - - /* Delete the physical files, fil_space_t & fil_node_t entries. */ - err = fil_delete_tablespace(space_id, BUF_REMOVE_FLUSH_NO_WRITE); - switch (err) { - case DB_TABLESPACE_NOT_FOUND: - /* OK if the physical file is mising. - We deleted the metadata. */ - case DB_SUCCESS: - innobase_commit_low(trx); - break; - default: - ib::error() << "Unable to delete the tablespace `" - << alter_info->tablespace_name - << "`, Space ID " << space_id; -have_error: - error = convert_error_code_to_mysql(err, 0, NULL); - trx_rollback_for_mysql(trx); - } - - row_mysql_unlock_data_dictionary(trx); - trx_free_for_mysql(trx); - - DBUG_RETURN(error); -} - -/** This API handles CREATE, ALTER & DROP commands for InnoDB tablespaces. -@param[in] hton Handlerton of InnoDB -@param[in] thd Connection -@param[in] alter_info How to do the command -@return MySQL error code*/ -static -int -innobase_alter_tablespace( - handlerton* hton, - THD* thd, - st_alter_tablespace* alter_info) -{ - int error; /* return zero for success */ - DBUG_ENTER("innobase_alter_tablespace"); - - switch (alter_info->ts_cmd_type) { - case CREATE_TABLESPACE: - error = innobase_create_tablespace(hton, thd, alter_info); - break; - - case DROP_TABLESPACE: - error = innobase_drop_tablespace(hton, thd, alter_info); - break; - - default: - error = HA_ADMIN_NOT_IMPLEMENTED; - } - - if (error) { - /* These are the most common message params */ - const char* object_type = "TABLESPACE"; - const char* object = alter_info->tablespace_name; - - /* Modify those params as needed. */ - switch (alter_info->ts_cmd_type) { - case DROP_TABLESPACE: - ib_errf(thd, IB_LOG_LEVEL_ERROR, - ER_DROP_FILEGROUP_FAILED, - "%s %s", object_type, object); - break; - case CREATE_TABLESPACE: - ib_errf(thd, IB_LOG_LEVEL_ERROR, - ER_CREATE_FILEGROUP_FAILED, - "%s %s", object_type, object); - break; - case CREATE_LOGFILE_GROUP: - my_error(ER_ILLEGAL_HA_CREATE_OPTION, MYF(0), - innobase_hton_name, "LOGFILE GROUP"); - break; - case ALTER_TABLESPACE: - case ALTER_ACCESS_MODE_TABLESPACE: - case DROP_LOGFILE_GROUP: - case ALTER_LOGFILE_GROUP: - case CHANGE_FILE_TABLESPACE: - case TS_CMD_NOT_DEFINED: - break; - } - } - - DBUG_RETURN(error); -} -#endif /* MYSQL_TABLESPACES */ - /** Remove all tables in the named database inside InnoDB. @param[in] hton handlerton from InnoDB @param[in] path Database path; Inside InnoDB the name of the last @@ -15510,9 +13966,7 @@ innobase_drop_database( } ptr++; - namebuf = (char*) my_malloc((uint) len + 2, MYF(0)); - // JAN: TODO: MySQL 5.7 - //namebuf = (char*) my_malloc(PSI_INSTRUMENT_ME, (uint) len + 2, MYF(0)); + namebuf = (char*) my_malloc(/*PSI_INSTRUMENT_ME,*/ (uint) len + 2, MYF(0)); memcpy(namebuf, ptr, len); namebuf[len] = '/'; @@ -15603,9 +14057,9 @@ innobase_rename_table( if (error == DB_TABLE_NOT_FOUND) { ib::error() << "Table " << ut_get_name(trx, norm_from) << " does not exist in the InnoDB internal" - " data dictionary though MySQL is trying to" + " data dictionary though MariaDB is trying to" " rename the table. Have you copied the .frm" - " file of the table to the MySQL database" + " file of the table to the MariaDB database" " directory from another database? " << TROUBLESHOOTING_MSG; } @@ -15613,12 +14067,7 @@ innobase_rename_table( if (error != DB_SUCCESS) { if (error == DB_TABLE_NOT_FOUND && innobase_get_lower_case_table_names() == 1) { - char* is_part = NULL; -#ifdef _WIN32 - is_part = strstr(norm_from, "#p#"); -#else - is_part = strstr(norm_from, "#P#"); -#endif /* _WIN32 */ + char* is_part = is_partition(norm_from); if (is_part) { char par_case_name[FN_REFLEN]; @@ -15767,8 +14216,9 @@ For other error codes, the server will fall back to counting records. */ #ifdef MYSQL_57_SELECT_COUNT_OPTIMIZATION int -ha_innobase::records(ha_rows* num_rows) -/*===================================*/ +ha_innobase::records( +/*==================*/ + ha_rows* num_rows) /*!< out: number of rows */ { DBUG_ENTER("ha_innobase::records()"); @@ -15840,16 +14290,15 @@ ha_innobase::records(ha_rows* num_rows) case DB_LOCK_WAIT_TIMEOUT: *num_rows = HA_POS_ERROR; DBUG_RETURN(convert_error_code_to_mysql(ret, 0, m_user_thd)); - break; case DB_INTERRUPTED: *num_rows = HA_POS_ERROR; DBUG_RETURN(HA_ERR_QUERY_INTERRUPTED); - break; default: /* No other error besides the three below is returned from row_scan_index_for_mysql(). Make a debug catch. */ *num_rows = HA_POS_ERROR; ut_ad(0); + DBUG_RETURN(-1); } m_prebuilt->trx->op_info = ""; @@ -15859,6 +14308,7 @@ ha_innobase::records(ha_rows* num_rows) DBUG_RETURN(HA_ERR_QUERY_INTERRUPTED); } + *num_rows= n_rows; DBUG_RETURN(0); } #endif /* MYSQL_57_SELECT_COUNT_OPTIMIZATION */ @@ -16231,7 +14681,7 @@ innobase_get_mysql_key_number_for_index( if (index->is_committed()) { sql_print_warning( "Found index %s in InnoDB index list" - " but not its MySQL index number." + " but not its MariaDB index number." " It could be an InnoDB internal" " index.", index->name()); @@ -16353,7 +14803,7 @@ ha_innobase::info_low( /* In case MySQL calls this in the middle of a SELECT query, release possible adaptive hash latch to avoid deadlocks of threads */ - m_prebuilt->trx->op_info = (char*)"returning various info to MySQL"; + m_prebuilt->trx->op_info = (char*)"returning various info to MariaDB"; trx_search_latch_release_if_reserved(m_prebuilt->trx); @@ -16390,7 +14840,7 @@ ha_innobase::info_low( } m_prebuilt->trx->op_info = - "returning various info to MySQL"; + "returning various info to MariaDB"; } @@ -16491,8 +14941,7 @@ ha_innobase::info_low( char errbuf[MYSYS_STRERROR_SIZE]; thd = ha_thd(); - my_strerror(errbuf, sizeof(errbuf), - errno); + push_warning_printf( thd, Sql_condition::WARN_LEVEL_WARN, @@ -16504,8 +14953,8 @@ ha_innobase::info_low( " the free space to zero." " (errno: %d - %s)", ib_table->name.m_name, errno, - errbuf); - + my_strerror(errbuf, sizeof(errbuf), + errno)); stats.delete_length = 0; } else { @@ -16610,7 +15059,7 @@ ha_innobase::info_low( sql_print_error( "Index %s of %s has %lu columns" " unique inside InnoDB, but" - " MySQL is asking statistics for" + " MariaDB is asking statistics for" " %lu columns. Have you mixed" " up .frm files from different" " installations? %s", @@ -16670,9 +15119,8 @@ ha_innobase::info_low( nor the CHECK TABLE time, nor the UPDATE or INSERT time. */ if (os_file_get_status( - path, &stat_info, false, - (dict_table_is_intrinsic(ib_table) - ? false : srv_read_only_mode)) == DB_SUCCESS) { + path, &stat_info, false, + srv_read_only_mode) == DB_SUCCESS) { stats.create_time = (ulong) stat_info.ctime; } } @@ -16702,18 +15150,7 @@ ha_innobase::info_low( } if ((flag & HA_STATUS_AUTO) && table->found_next_number_field) { - - ulonglong auto_inc_val = innobase_peek_autoinc(); - /* Initialize autoinc value if not set. */ - if (auto_inc_val == 0) { - - dict_table_autoinc_lock(m_prebuilt->table); - innobase_initialize_autoinc(); - dict_table_autoinc_unlock(m_prebuilt->table); - - auto_inc_val = innobase_peek_autoinc(); - } - stats.auto_increment_value = auto_inc_val; + stats.auto_increment_value = innobase_peek_autoinc(); } func_exit: @@ -16735,72 +15172,6 @@ ha_innobase::info( return(info_low(flag, false /* not ANALYZE */)); } -/** Enable indexes. -@param[in] mode enable index mode. -@return HA_ERR_* error code or 0 */ -int -ha_innobase::enable_indexes( - uint mode) -{ - int error = HA_ERR_WRONG_COMMAND; - - /* Enable index only for intrinsic table. Behavior for all other - table continue to remain same. */ - - if (dict_table_is_intrinsic(m_prebuilt->table)) { - ut_ad(mode == HA_KEY_SWITCH_ALL); - for (dict_index_t* index - = UT_LIST_GET_FIRST(m_prebuilt->table->indexes); - index != NULL; - index = UT_LIST_GET_NEXT(indexes, index)) { - - /* InnoDB being clustered index we can't disable/enable - clustered index itself. */ - if (dict_index_is_clust(index)) { - continue; - } - - index->allow_duplicates = false; - } - error = 0; - } - - return(error); -} - -/** Disable indexes. -@param[in] mode disable index mode. -@return HA_ERR_* error code or 0 */ -int -ha_innobase::disable_indexes( - uint mode) -{ - int error = HA_ERR_WRONG_COMMAND; - - /* Disable index only for intrinsic table. Behavior for all other - table continue to remain same. */ - - if (dict_table_is_intrinsic(m_prebuilt->table)) { - ut_ad(mode == HA_KEY_SWITCH_ALL); - for (dict_index_t* index - = UT_LIST_GET_FIRST(m_prebuilt->table->indexes); - index != NULL; - index = UT_LIST_GET_NEXT(indexes, index)) { - - /* InnoDB being clustered index we can't disable/enable - clustered index itself. */ - if (dict_index_is_clust(index)) { - continue; - } - - index->allow_duplicates = true; - } - error = 0; - } - - return(error); -} - /* Updates index cardinalities of the table, based on random dives into each index tree. This does NOT calculate exact statistics on the table. @@ -17130,7 +15501,7 @@ ha_innobase::check( if (!dict_index_is_clust(index)) { m_prebuilt->index_usable = FALSE; // row_mysql_lock_data_dictionary(m_prebuilt->trx); - dict_set_corrupted(index, m_prebuilt->trx, "dict_set_index_corrupted");; + dict_set_corrupted(index, m_prebuilt->trx, "dict_set_index_corrupted"); // row_mysql_unlock_data_dictionary(m_prebuilt->trx); }); @@ -17428,41 +15799,26 @@ get_foreign_key_info( } while (++i < foreign->n_fields); if (foreign->type & DICT_FOREIGN_ON_DELETE_CASCADE) { - len = 7; - ptr = "CASCADE"; + f_key_info.delete_method = FK_OPTION_CASCADE; } else if (foreign->type & DICT_FOREIGN_ON_DELETE_SET_NULL) { - len = 8; - ptr = "SET NULL"; + f_key_info.delete_method = FK_OPTION_SET_NULL; } else if (foreign->type & DICT_FOREIGN_ON_DELETE_NO_ACTION) { - len = 9; - ptr = "NO ACTION"; + f_key_info.delete_method = FK_OPTION_NO_ACTION; } else { - len = 8; - ptr = "RESTRICT"; + f_key_info.delete_method = FK_OPTION_RESTRICT; } - f_key_info.delete_method = thd_make_lex_string( - thd, f_key_info.delete_method, ptr, - static_cast<unsigned int>(len), 1); if (foreign->type & DICT_FOREIGN_ON_UPDATE_CASCADE) { - len = 7; - ptr = "CASCADE"; + f_key_info.update_method = FK_OPTION_CASCADE; } else if (foreign->type & DICT_FOREIGN_ON_UPDATE_SET_NULL) { - len = 8; - ptr = "SET NULL"; + f_key_info.update_method = FK_OPTION_SET_NULL; } else if (foreign->type & DICT_FOREIGN_ON_UPDATE_NO_ACTION) { - len = 9; - ptr = "NO ACTION"; + f_key_info.update_method = FK_OPTION_NO_ACTION; } else { - len = 8; - ptr = "RESTRICT"; + f_key_info.update_method = FK_OPTION_RESTRICT; } - f_key_info.update_method = thd_make_lex_string( - thd, f_key_info.update_method, ptr, - static_cast<unsigned int>(len), 1); - if (foreign->referenced_index && foreign->referenced_index->name != NULL) { referenced_key_name = thd_make_lex_string( @@ -17924,16 +16280,6 @@ ha_innobase::start_stmt( TrxInInnoDB trx_in_innodb(trx); - if (dict_table_is_intrinsic(m_prebuilt->table)) { - - if (thd_sql_command(thd) == SQLCOM_ALTER_TABLE) { - - DBUG_RETURN(HA_ERR_WRONG_COMMAND); - } - - DBUG_RETURN(0); - } - trx = m_prebuilt->trx; innobase_srv_conc_force_exit_innodb(trx); @@ -18063,18 +16409,6 @@ ha_innobase::external_lock( ut_ad(m_prebuilt->table); - if (dict_table_is_intrinsic(m_prebuilt->table)) { - - if (thd_sql_command(thd) == SQLCOM_ALTER_TABLE) { - - DBUG_RETURN(HA_ERR_WRONG_COMMAND); - } - - TrxInInnoDB::begin_stmt(trx); - - DBUG_RETURN(0); - } - /* Statement based binlogging does not work in isolation level READ UNCOMMITTED and READ COMMITTED since the necessary locks cannot be taken. In this case, we print an @@ -18329,7 +16663,7 @@ void innodb_export_status() /*==================*/ { - if (innodb_inited) { + if (srv_was_started) { srv_export_innodb_status(); } } @@ -18400,12 +16734,8 @@ innodb_show_status( /* allocate buffer for the string, and read the contents of the temporary file */ - /* JAN: TODO: MySQL 5.7 PSI */ - if (!(str = (char*) my_malloc( - usable_len + 1, MYF(0)))) { - /* if (!(str = (char*) my_malloc(PSI_INSTRUMENT_ME, + if (!(str = (char*) my_malloc(//PSI_INSTRUMENT_ME, usable_len + 1, MYF(0)))) { - */ mutex_exit(&srv_monitor_file_mutex); DBUG_RETURN(1); } @@ -18840,15 +17170,9 @@ get_share( grows too big */ share = reinterpret_cast<INNOBASE_SHARE*>( - my_malloc( - sizeof(*share) + length + 1, - MYF(MY_FAE | MY_ZEROFILL))); - /* JAN: TODO: MySQL 5.7 PSI - share = reinterpret_cast<INNOBASE_SHARE*>( - my_malloc(PSI_INSTRUMENT_ME, + my_malloc(//PSI_INSTRUMENT_ME, sizeof(*share) + length + 1, MYF(MY_FAE | MY_ZEROFILL))); - */ share->table_name = reinterpret_cast<char*>( memcpy(share + 1, table_name, length + 1)); @@ -18997,7 +17321,6 @@ ha_innobase::store_lock( const uint sql_command = thd_sql_command(thd); if (srv_read_only_mode - && !dict_table_is_intrinsic(m_prebuilt->table) && (sql_command == SQLCOM_UPDATE || sql_command == SQLCOM_INSERT || sql_command == SQLCOM_REPLACE @@ -19683,7 +18006,7 @@ innobase_xa_prepare( if (!trx_is_registered_for_2pc(trx) && trx_is_started(trx)) { - sql_print_error("Transaction not registered for MySQL 2PC," + sql_print_error("Transaction not registered for MariaDB 2PC," " but transaction is active"); } @@ -19827,64 +18150,6 @@ innobase_rollback_by_xid( } } -#ifdef INNOBASE_CURSOR_VIEW - -/*******************************************************************//** -Create a consistent view for a cursor based on current transaction -which is created if the corresponding MySQL thread still lacks one. -This consistent view is then used inside of MySQL when accessing records -using a cursor. -@return pointer to cursor view or NULL */ -static -void* -innobase_create_cursor_view( -/*========================*/ - handlerton* hton, /*!< in: innobase hton */ - THD* thd) /*!< in: user thread handle */ -{ - DBUG_ASSERT(hton == innodb_hton_ptr); - - return(read_cursor_view_create_for_mysql(check_trx_exists(thd))); -} - -/*******************************************************************//** -Close the given consistent cursor view of a transaction and restore -global read view to a transaction read view. Transaction is created if the -corresponding MySQL thread still lacks one. */ -static -void -innobase_close_cursor_view( -/*=======================*/ - handlerton* hton, /*!< in: innobase hton */ - THD* thd, /*!< in: user thread handle */ - void* curview)/*!< in: Consistent read view to be closed */ -{ - DBUG_ASSERT(hton == innodb_hton_ptr); - - read_cursor_view_close_for_mysql(check_trx_exists(thd), - (cursor_view_t*) curview); -} - -/*******************************************************************//** -Set the given consistent cursor view to a transaction which is created -if the corresponding MySQL thread still lacks one. If the given -consistent cursor view is NULL global read view of a transaction is -restored to a transaction read view. */ -static -void -innobase_set_cursor_view( -/*=====================*/ - handlerton* hton, /*!< in: innobase hton */ - THD* thd, /*!< in: user thread handle */ - void* curview)/*!< in: Consistent cursor view to be set */ -{ - DBUG_ASSERT(hton == innodb_hton_ptr); - - read_cursor_set_for_mysql(check_trx_exists(thd), - (cursor_view_t*) curview); -} -#endif /* INNOBASE_CURSOR_VIEW */ - bool ha_innobase::check_if_incompatible_data( /*====================================*/ @@ -19925,8 +18190,8 @@ ha_innobase::check_if_incompatible_data( /* Changes on engine specific table options requests a rebuild of the table. */ if (param_new->page_compressed != param_old->page_compressed || - param_new->page_compression_level != param_old->page_compression_level || - param_new->atomic_writes != param_old->atomic_writes) { + param_new->page_compression_level != param_old->page_compression_level) + { return(COMPATIBLE_DATA_NO); } @@ -21038,12 +19303,8 @@ innodb_monitor_validate( by InnoDB, so we can access it in another callback function innodb_monitor_update() and free it appropriately */ if (name) { - /* JAN: TODO: MySQL 5.7 PSI - monitor_name = my_strdup(PSI_INSTRUMENT_ME, + monitor_name = my_strdup(//PSI_INSTRUMENT_ME, name, MYF(0)); - */ - monitor_name = my_strdup( - name, MYF(0)); } else { return(1); } @@ -22004,13 +20265,8 @@ innodb_log_checksums_update( void* var_ptr, const void* save) { - my_bool check = *static_cast<my_bool*>(var_ptr) - = *static_cast<const my_bool*>(save); - - /* Make sure we are the only log user */ - mutex_enter(&log_sys->mutex); - innodb_log_checksums_func_update(check); - mutex_exit(&log_sys->mutex); + *static_cast<my_bool*>(var_ptr) = innodb_log_checksums_func_update( + thd, *static_cast<const my_bool*>(save)); } static SHOW_VAR innodb_status_variables_export[]= { @@ -22345,6 +20601,25 @@ wsrep_fake_trx_id( #endif /* WITH_WSREP */ +/** Update the innodb_use_trim parameter. +@param[in] thd thread handle +@param[in] var system variable +@param[out] var_ptr current value +@param[in] save immediate result from check function */ +static +void +innodb_use_trim_update( + THD* thd, + struct st_mysql_sys_var* var, + void* var_ptr, + const void* save) +{ + srv_use_trim = *static_cast<const my_bool*>(save); + + push_warning(thd, Sql_condition::WARN_LEVEL_WARN, + HA_ERR_WRONG_COMMAND, deprecated_use_trim); +} + /* plugin options */ static MYSQL_SYSVAR_ENUM(checksum_algorithm, srv_checksum_algorithm, @@ -22367,7 +20642,7 @@ static MYSQL_SYSVAR_ENUM(checksum_algorithm, srv_checksum_algorithm, " write a constant magic number, do not allow values other than that" " magic number when reading;" " Files updated when this option is set to crc32 or strict_crc32 will" - " not be readable by MySQL versions older than 5.6.3", + " not be readable by MariaDB versions older than 10.0.4", NULL, NULL, SRV_CHECKSUM_ALGORITHM_CRC32, &innodb_checksum_algorithm_typelib); @@ -22397,16 +20672,17 @@ static MYSQL_SYSVAR_BOOL(doublewrite, innobase_use_doublewrite, static MYSQL_SYSVAR_BOOL(use_atomic_writes, innobase_use_atomic_writes, PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, - "Prevent partial page writes, via atomic writes." - "The option is used to prevent partial writes in case of a crash/poweroff, " - "as faster alternative to doublewrite buffer." - "Currently this option works only " - "on Linux only with FusionIO device, and directFS filesystem.", - NULL, NULL, FALSE); + "Enable atomic writes, instead of using the doublewrite buffer, for files " + "on devices that supports atomic writes. " + "To use this option one must use " + "file_per_table=1, flush_method=O_DIRECT and use_fallocate=1. " + "This option only works on Linux with either FusionIO cards using " + "the directFS filesystem or with Shannon cards using any file system.", + NULL, NULL, TRUE); static MYSQL_SYSVAR_BOOL(use_fallocate, innobase_use_fallocate, PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, - "Preallocate files fast, using operating system functionality. On POSIX systems, posix_fallocate system call is used.", + "Use posix_fallocate() to allocate files. DEPRECATED, has no effect.", NULL, NULL, FALSE); static MYSQL_SYSVAR_ULONG(io_capacity, srv_io_capacity, @@ -22488,7 +20764,7 @@ static MYSQL_SYSVAR_ULONG(sync_array_size, srv_sync_array_size, 1, /* Minimum value */ 1024, 0); /* Maximum value */ -static MYSQL_SYSVAR_ULONG(fast_shutdown, innobase_fast_shutdown, +static MYSQL_SYSVAR_UINT(fast_shutdown, srv_fast_shutdown, PLUGIN_VAR_OPCMDARG, "Speeds up the shutdown process of the InnoDB storage engine. Possible" " values are 0, 1 (faster) or 2 (fastest - crash-like).", @@ -22720,7 +20996,7 @@ static MYSQL_SYSVAR_BOOL(log_compressed_pages, page_zip_log_pages, " the zlib compression algorithm changes." " When turned OFF, InnoDB will assume that the zlib" " compression algorithm doesn't change.", - NULL, NULL, FALSE); + NULL, NULL, TRUE); static MYSQL_SYSVAR_ULONG(autoextend_increment, sys_tablespace_auto_extend_increment, @@ -23001,13 +21277,6 @@ static MYSQL_SYSVAR_ULONG(force_recovery, srv_force_recovery, "Helps to save your data in case the disk image of the database becomes corrupt.", NULL, NULL, 0, 0, 6, 0); -#ifndef DBUG_OFF -static MYSQL_SYSVAR_ULONG(force_recovery_crash, srv_force_recovery_crash, - PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, - "Kills the server during crash recovery.", - NULL, NULL, 0, 0, 100, 0); -#endif /* !DBUG_OFF */ - static MYSQL_SYSVAR_ULONG(page_size, srv_page_size, PLUGIN_VAR_OPCMDARG | PLUGIN_VAR_READONLY, "Page size to use for all InnoDB tablespaces.", @@ -23027,7 +21296,7 @@ static MYSQL_SYSVAR_LONGLONG(log_file_size, innobase_log_file_size, static MYSQL_SYSVAR_ULONG(log_files_in_group, srv_n_log_files, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "Number of log files in the log group. InnoDB writes to the files in a circular fashion.", - NULL, NULL, 2, 2, SRV_N_LOG_FILES_MAX, 0); + NULL, NULL, 2, 1, SRV_N_LOG_FILES_MAX, 0); static MYSQL_SYSVAR_ULONG(log_write_ahead_size, srv_log_write_ahead_size, PLUGIN_VAR_RQCMDARG, @@ -23058,10 +21327,10 @@ static MYSQL_SYSVAR_ULONG(sync_spin_loops, srv_n_spin_wait_rounds, "Count of spin-loop rounds in InnoDB mutexes (30 by default)", NULL, NULL, 30L, 0L, ~0UL, 0); -static MYSQL_SYSVAR_ULONG(spin_wait_delay, srv_spin_wait_delay, +static MYSQL_SYSVAR_UINT(spin_wait_delay, srv_spin_wait_delay, PLUGIN_VAR_OPCMDARG, "Maximum delay between polling for a spin lock (6 by default)", - NULL, NULL, 6L, 0L, ~0UL, 0); + NULL, NULL, 6, 0, 6000, 0); static MYSQL_SYSVAR_ULONG(thread_concurrency, srv_thread_concurrency, PLUGIN_VAR_RQCMDARG, @@ -23173,43 +21442,12 @@ static MYSQL_SYSVAR_BOOL(use_native_aio, srv_use_native_aio, "Use native AIO if supported on this platform.", NULL, NULL, TRUE); -#if defined(HAVE_LIBNUMA) && defined(WITH_NUMA) +#ifdef HAVE_LIBNUMA static MYSQL_SYSVAR_BOOL(numa_interleave, srv_numa_interleave, PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, "Use NUMA interleave memory policy to allocate InnoDB buffer pool.", NULL, NULL, FALSE); -#endif /* HAVE_LIBNUMA && WITH_NUMA */ - -static MYSQL_SYSVAR_BOOL(api_enable_binlog, ib_binlog_enabled, - PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, - "Enable binlog for applications direct access InnoDB through InnoDB APIs", - NULL, NULL, FALSE); - -static MYSQL_SYSVAR_BOOL(api_enable_mdl, ib_mdl_enabled, - PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, - "Enable MDL for applications direct access InnoDB through InnoDB APIs", - NULL, NULL, FALSE); - -static MYSQL_SYSVAR_BOOL(api_disable_rowlock, ib_disable_row_lock, - PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, - "Disable row lock when direct access InnoDB through InnoDB APIs", - NULL, NULL, FALSE); - -static MYSQL_SYSVAR_ULONG(api_trx_level, ib_trx_level_setting, - PLUGIN_VAR_OPCMDARG, - "InnoDB API transaction isolation level", - NULL, NULL, - 0, /* Default setting */ - 0, /* Minimum value */ - 3, 0); /* Maximum value */ - -static MYSQL_SYSVAR_ULONG(api_bk_commit_interval, ib_bk_commit_interval, - PLUGIN_VAR_OPCMDARG, - "Background commit interval in seconds", - NULL, NULL, - 5, /* Default setting */ - 1, /* Minimum value */ - 1024 * 1024 * 1024, 0); /* Maximum value */ +#endif /* HAVE_LIBNUMA */ static MYSQL_SYSVAR_STR(change_buffering, innobase_change_buffering, PLUGIN_VAR_RQCMDARG, @@ -23244,11 +21482,6 @@ static MYSQL_SYSVAR_BOOL(disable_background_merge, PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_RQCMDARG, "Disable change buffering merges by the master thread", NULL, NULL, FALSE); - -static MYSQL_SYSVAR_ENUM(compress_debug, srv_debug_compress, - PLUGIN_VAR_RQCMDARG, - "Compress all tables, without specifying the COMPRESS table attribute", - NULL, NULL, Compression::NONE, &innodb_debug_compress_typelib); #endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */ static MYSQL_SYSVAR_ULONG(buf_dump_status_frequency, srv_buf_dump_status_frequency, @@ -23282,10 +21515,11 @@ innobase_disallow_writes_update( { *(my_bool*)var_ptr = *(my_bool*)save; ut_a(srv_allow_writes_event); - if (*(my_bool*)var_ptr) + if (*(my_bool*)var_ptr) { os_event_reset(srv_allow_writes_event); - else + } else { os_event_set(srv_allow_writes_event); + } } static MYSQL_SYSVAR_BOOL(disallow_writes, innobase_disallow_writes, @@ -23293,6 +21527,7 @@ static MYSQL_SYSVAR_BOOL(disallow_writes, innobase_disallow_writes, "Tell InnoDB to stop any writes to disk", NULL, innobase_disallow_writes_update, FALSE); #endif /* WITH_INNODB_DISALLOW_WRITES */ + static MYSQL_SYSVAR_BOOL(random_read_ahead, srv_random_read_ahead, PLUGIN_VAR_NOCMDARG, "Whether to use read ahead for random access within an extent.", @@ -23339,7 +21574,7 @@ static MYSQL_SYSVAR_BOOL(status_output_locks, srv_print_innodb_lock_monitor, static MYSQL_SYSVAR_BOOL(print_all_deadlocks, srv_print_all_deadlocks, PLUGIN_VAR_OPCMDARG, - "Print all deadlocks to MySQL error log (off by default)", + "Print all deadlocks to MariaDB error log (off by default)", NULL, NULL, FALSE); static MYSQL_SYSVAR_ULONG(compression_failure_threshold_pct, @@ -23392,6 +21627,12 @@ static MYSQL_SYSVAR_BOOL(trx_purge_view_update_only_debug, " but the each purges were not done yet.", NULL, NULL, FALSE); +static MYSQL_SYSVAR_UINT(data_file_size_debug, + srv_sys_space_size_debug, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "InnoDB system tablespace size to be set in recovery.", + NULL, NULL, 0, 0, UINT_MAX32, 0); + static MYSQL_SYSVAR_ULONG(fil_make_page_dirty_debug, srv_fil_make_page_dirty_debug, PLUGIN_VAR_OPCMDARG, "Make the first page of the given tablespace dirty.", @@ -23444,8 +21685,8 @@ static MYSQL_SYSVAR_BOOL(force_primary_key, static MYSQL_SYSVAR_BOOL(use_trim, srv_use_trim, PLUGIN_VAR_OPCMDARG, - "Use trim. Default FALSE.", - NULL, NULL, FALSE); + "Deallocate (punch_hole|trim) unused portions of the page compressed page (on by default)", + NULL, innodb_use_trim_update, TRUE); static const char *page_compression_algorithms[]= { "none", "zlib", "lz4", "lzo", "lzma", "bzip2", "snappy", 0 }; static TYPELIB page_compression_algorithms_typelib= @@ -23460,7 +21701,7 @@ static MYSQL_SYSVAR_ENUM(compression_algorithm, innodb_compression_algorithm, /* We use here the largest number of supported compression method to enable all those methods that are available. Availability of compression method is verified on innodb_compression_algorithm_validate function. */ - PAGE_UNCOMPRESSED, + PAGE_ZLIB_ALGORITHM, &page_compression_algorithms_typelib); static MYSQL_SYSVAR_LONG(mtflush_threads, srv_mtflush_threads, @@ -23599,8 +21840,6 @@ static MYSQL_SYSVAR_BOOL(instrument_semaphores, srv_instrument_semaphores, 0, 0, FALSE); static struct st_mysql_sys_var* innobase_system_variables[]= { - MYSQL_SYSVAR(api_trx_level), - MYSQL_SYSVAR(api_bk_commit_interval), MYSQL_SYSVAR(autoextend_increment), MYSQL_SYSVAR(buffer_pool_size), MYSQL_SYSVAR(buffer_pool_chunk_size), @@ -23635,9 +21874,6 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(doublewrite), MYSQL_SYSVAR(use_atomic_writes), MYSQL_SYSVAR(use_fallocate), - MYSQL_SYSVAR(api_enable_binlog), - MYSQL_SYSVAR(api_enable_mdl), - MYSQL_SYSVAR(api_disable_rowlock), MYSQL_SYSVAR(fast_shutdown), MYSQL_SYSVAR(read_io_threads), MYSQL_SYSVAR(write_io_threads), @@ -23649,9 +21885,6 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(flush_log_at_trx_commit), MYSQL_SYSVAR(flush_method), MYSQL_SYSVAR(force_recovery), -#ifndef DBUG_OFF - MYSQL_SYSVAR(force_recovery_crash), -#endif /* !DBUG_OFF */ MYSQL_SYSVAR(fill_factor), MYSQL_SYSVAR(ft_cache_size), MYSQL_SYSVAR(ft_total_cache_size), @@ -23719,9 +21952,9 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(autoinc_lock_mode), MYSQL_SYSVAR(version), MYSQL_SYSVAR(use_native_aio), -#if defined(HAVE_LIBNUMA) && defined(WITH_NUMA) +#ifdef HAVE_LIBNUMA MYSQL_SYSVAR(numa_interleave), -#endif /* HAVE_LIBNUMA && WITH_NUMA */ +#endif /* HAVE_LIBNUMA */ MYSQL_SYSVAR(change_buffering), MYSQL_SYSVAR(change_buffer_max_size), #if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG @@ -23776,9 +22009,9 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(trx_rseg_n_slots_debug), MYSQL_SYSVAR(limit_optimistic_insert_debug), MYSQL_SYSVAR(trx_purge_view_update_only_debug), + MYSQL_SYSVAR(data_file_size_debug), MYSQL_SYSVAR(fil_make_page_dirty_debug), MYSQL_SYSVAR(saved_page_number_debug), - MYSQL_SYSVAR(compress_debug), MYSQL_SYSVAR(disable_resize_buffer_pool_debug), MYSQL_SYSVAR(page_cleaner_disabled_debug), MYSQL_SYSVAR(dict_stats_disabled_debug), @@ -23813,6 +22046,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { #endif MYSQL_SYSVAR(instrument_semaphores), MYSQL_SYSVAR(buf_dump_status_frequency), + MYSQL_SYSVAR(background_thread), NULL }; @@ -23844,7 +22078,6 @@ i_s_innodb_cmp_per_index_reset, i_s_innodb_buffer_page, i_s_innodb_buffer_page_lru, i_s_innodb_buffer_stats, -i_s_innodb_temp_table_info, i_s_innodb_metrics, i_s_innodb_ft_default_stopword, i_s_innodb_ft_deleted, @@ -23987,29 +22220,29 @@ innobase_index_cond( return handler_index_cond_check(file); } -#ifdef MYSQL_VIRTUAL_COLUMNS -/** Get the computed value by supplying the base column values. -@param[in,out] table the table whose virtual column template to be built */ -void -innobase_init_vc_templ( + +/** Find or open a mysql table for the virtual column template +@param[in] thd mysql thread handle +@param[in,out] table InnoDB table whose virtual column template is to be updated +@return TABLE if successful or NULL */ +static TABLE * +innobase_find_mysql_table_for_vc( +/*=============================*/ + THD* thd, dict_table_t* table) { - char dbname[MAX_DATABASE_NAME_LEN + 1]; - char tbname[MAX_TABLE_NAME_LEN + 1]; - char* name = table->name.m_name; - ulint dbnamelen = dict_get_db_name_len(name); - ulint tbnamelen = strlen(name) - dbnamelen - 1; - char t_dbname[MAX_DATABASE_NAME_LEN + 1]; - char t_tbname[MAX_TABLE_NAME_LEN + 1]; - - mutex_enter(&dict_sys->mutex); - - if (table->vc_templ != NULL) { - mutex_exit(&dict_sys->mutex); - - return; + if (table->vc_templ->mysql_table_query_id == thd_get_query_id(thd)) { + return table->vc_templ->mysql_table; } + char dbname[MAX_DATABASE_NAME_LEN + 1]; + char tbname[MAX_TABLE_NAME_LEN + 1]; + char* name = table->name.m_name; + uint dbnamelen = dict_get_db_name_len(name); + uint tbnamelen = strlen(name) - dbnamelen - 1; + char t_dbname[MAX_DATABASE_NAME_LEN + 1]; + char t_tbname[MAX_TABLE_NAME_LEN + 1]; + strncpy(dbname, name, dbnamelen); dbname[dbnamelen] = 0; strncpy(tbname, name + dbnamelen + 1, tbnamelen); @@ -24017,36 +22250,53 @@ innobase_init_vc_templ( /* For partition table, remove the partition name and use the "main" table name to build the template */ -#ifdef _WIN32 - char* is_part = strstr(tbname, "#p#"); -#else - char* is_part = strstr(tbname, "#P#"); -#endif /* _WIN32 */ + char* is_part = is_partition(tbname); if (is_part != NULL) { *is_part = '\0'; tbnamelen = is_part - tbname; } - table->vc_templ = UT_NEW_NOKEY(dict_vcol_templ_t()); - table->vc_templ->vtempl = NULL; - dbnamelen = filename_to_tablename(dbname, t_dbname, MAX_DATABASE_NAME_LEN + 1); tbnamelen = filename_to_tablename(tbname, t_tbname, MAX_TABLE_NAME_LEN + 1); -#ifdef UNIV_DEBUG - // bool ret = -#endif /* UNIV_DEBUG */ + TABLE *mysql_table = find_fk_open_table(thd, t_dbname, dbnamelen, + t_tbname, tbnamelen); - /* JAN: TODO: MySQL: 5.7 virtual columsn - handler::my_prepare_gcolumn_template( - thd, t_dbname, t_tbname, - &innobase_build_v_templ_callback, - static_cast<void*>(table)); - ut_ad(!ret); - */ + if (!mysql_table && THDVAR(thd, background_thread)) { + /* only open the table in background purge threads */ + mysql_table = open_purge_table(thd, t_dbname, dbnamelen, + t_tbname, tbnamelen); + } + + table->vc_templ->mysql_table = mysql_table; + table->vc_templ->mysql_table_query_id = thd_get_query_id(thd); + return mysql_table; +} + +/** Get the computed value by supplying the base column values. +@param[in,out] table table whose virtual column template to be built */ +void +innobase_init_vc_templ( + dict_table_t* table) +{ + if (table->vc_templ != NULL) { + return; + } + + table->vc_templ = UT_NEW_NOKEY(dict_vcol_templ_t()); + + TABLE *mysql_table= innobase_find_mysql_table_for_vc(current_thd, table); + + ut_ad(mysql_table); + if (!mysql_table) { + return; + } + + mutex_enter(&dict_sys->mutex); + innobase_build_v_templ(mysql_table, table, table->vc_templ, NULL, true); mutex_exit(&dict_sys->mutex); } @@ -24072,11 +22322,7 @@ innobase_rename_vc_templ( /* For partition table, remove the partition name and use the "main" table name to build the template */ -#ifdef _WIN32 - char* is_part = strstr(tbname, "#p#"); -#else - char* is_part = strstr(tbname, "#P#"); -#endif /* _WIN32 */ + char* is_part = is_partition(tbname); if (is_part != NULL) { *is_part = '\0'; @@ -24160,7 +22406,6 @@ innobase_get_computed_value( upd_t* parent_update, dict_foreign_t* foreign) { - byte rec_buf1[REC_VERSION_56_MAX_INDEX_COL_LEN]; byte rec_buf2[REC_VERSION_56_MAX_INDEX_COL_LEN]; byte* mysql_rec; byte* buf; @@ -24186,15 +22431,20 @@ innobase_get_computed_value( *local_heap = mem_heap_create(UNIV_PAGE_SIZE); } - mysql_rec = static_cast<byte*>(mem_heap_alloc( - *local_heap, index->table->vc_templ->rec_len)); buf = static_cast<byte*>(mem_heap_alloc( *local_heap, index->table->vc_templ->rec_len)); } else { - mysql_rec = rec_buf1; buf = rec_buf2; } + if (!mysql_table) { + mysql_table = innobase_find_mysql_table_for_vc(thd, index->table); + } + + ut_ad(mysql_table); + + mysql_rec = mysql_table->record[0]; + for (ulint i = 0; i < col->num_base; i++) { dict_col_t* base_col = col->base_col[i]; const dfield_t* row_field = NULL; @@ -24253,46 +22503,11 @@ innobase_get_computed_value( field = dtuple_get_nth_v_field(row, col->v_pos); - /* Bitmap for specifying which virtual columns the server - should evaluate */ - MY_BITMAP column_map; - my_bitmap_map col_map_storage[bitmap_buffer_size(REC_MAX_N_FIELDS)]; - - bitmap_init(&column_map, col_map_storage, REC_MAX_N_FIELDS, false); - - /* Specify the column the server should evaluate */ - bitmap_set_bit(&column_map, col->m_col.ind); - - if (mysql_table == NULL) { - if (vctempl->type == DATA_BLOB) { - ulint max_len; - - if (vctempl->mysql_col_len - 8 == 1) { - /* This is for TINYBLOB only, which needs - only 1 byte, other BLOBs won't be affected */ - max_len = 255; - } else { - max_len = DICT_MAX_FIELD_LEN_BY_FORMAT( - index->table) + 1; - } - - byte* blob_mem = static_cast<byte*>( - mem_heap_alloc(heap, max_len)); - - row_mysql_store_blob_ref( - mysql_rec + vctempl->mysql_col_offset, - vctempl->mysql_col_len, blob_mem, max_len); - } - - ret = handler::my_eval_gcolumn_expr_with_open( - thd, index->table->vc_templ->db_name.c_str(), - index->table->vc_templ->tb_name.c_str(), &column_map, - (uchar *)mysql_rec); - } else { - ret = handler::my_eval_gcolumn_expr( - thd, mysql_table, &column_map, - (uchar *)mysql_rec); - } + my_bitmap_map* old_write_set = dbug_tmp_use_all_columns(mysql_table, mysql_table->write_set); + my_bitmap_map* old_read_set = dbug_tmp_use_all_columns(mysql_table, mysql_table->read_set); + ret = mysql_table->update_virtual_field(mysql_table->field[col->m_col.ind]); + dbug_tmp_restore_column_map(mysql_table->read_set, old_read_set); + dbug_tmp_restore_column_map(mysql_table->write_set, old_write_set); if (ret != 0) { #ifdef INNODB_VIRTUAL_DEBUG @@ -24304,11 +22519,6 @@ innobase_get_computed_value( return(NULL); } - /* we just want to store the data in passed in MySQL record */ - if (ret != 0) { - return(NULL); - } - if (vctempl->mysql_null_bit_mask && (mysql_rec[vctempl->mysql_null_byte_offset] & vctempl->mysql_null_bit_mask)) { @@ -24348,7 +22558,7 @@ innobase_get_computed_value( return(field); } -#endif /* MYSQL_VIRTUAL_COLUMNS */ + /** Attempt to push down an index condition. @param[in] keyno MySQL key number @@ -24364,6 +22574,12 @@ ha_innobase::idx_cond_push( DBUG_ASSERT(keyno != MAX_KEY); DBUG_ASSERT(idx_cond != NULL); + /* We can only evaluate the condition if all columns are stored.*/ + dict_index_t* idx = innobase_get_index(keyno); + if (idx && dict_index_has_virtual(idx)) { + DBUG_RETURN(idx_cond); + } + pushed_idx_cond = idx_cond; pushed_idx_cond_keyno = keyno; in_range_check_pushed_down = TRUE; @@ -24813,20 +23029,22 @@ ib_push_warning( const char *format,/*!< in: warning message */ ...) { - va_list args; - THD *thd = (THD *)trx->mysql_thd; - char *buf; + if (trx && trx->mysql_thd) { + THD *thd = (THD *)trx->mysql_thd; + va_list args; + char *buf; #define MAX_BUF_SIZE 4*1024 - va_start(args, format); - buf = (char *)my_malloc(MAX_BUF_SIZE, MYF(MY_WME)); - vsprintf(buf,format, args); + va_start(args, format); + buf = (char *)my_malloc(MAX_BUF_SIZE, MYF(MY_WME)); + vsprintf(buf,format, args); - push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, - convert_error_code_to_mysql((dberr_t)error, 0, thd), - buf); - my_free(buf); - va_end(args); + push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN, + convert_error_code_to_mysql((dberr_t)error, 0, thd), + buf); + my_free(buf); + va_end(args); + } } /********************************************************************//** @@ -24878,7 +23096,7 @@ ib_push_frm_error( case DICT_FRM_NO_PK: sql_print_error("Table %s has a primary key in " "InnoDB data dictionary, but not " - "in MySQL!" + "in MariaDB!" " Have you mixed up " ".frm files from different " "installations? See " @@ -24892,17 +23110,17 @@ ib_push_frm_error( "InnoDB: Table %s has a " "primary key in InnoDB data " "dictionary, but not in " - "MySQL!", ib_table->name); + "MariaDB!", ib_table->name); } break; case DICT_NO_PK_FRM_HAS: sql_print_error( "Table %s has no primary key in InnoDB data " - "dictionary, but has one in MySQL! If you " - "created the table with a MySQL version < " + "dictionary, but has one in MariaDB! If you " + "created the table with a MariaDB version < " "3.23.54 and did not define a primary key, " "but defined a unique key with all non-NULL " - "columns, then MySQL internally treats that " + "columns, then MariaDB internally treats that " "key as the primary key. You can fix this " "error by dump + DROP + CREATE + reimport " "of the table.", ib_table->name); @@ -24913,7 +23131,7 @@ ib_push_frm_error( "InnoDB: Table %s has no " "primary key in InnoDB data " "dictionary, but has one in " - "MySQL!", + "MariaDB!", ib_table->name); } break; @@ -24922,7 +23140,7 @@ ib_push_frm_error( sql_print_error("InnoDB: Table %s contains %lu " "indexes inside InnoDB, which " "is different from the number of " - "indexes %u defined in the MySQL " + "indexes %u defined in the MariaDB " " Have you mixed up " ".frm files from different " "installations? See " @@ -24937,7 +23155,7 @@ ib_push_frm_error( "InnoDB: Table %s contains %lu " "indexes inside InnoDB, which " "is different from the number of " - "indexes %u defined in the MySQL ", + "indexes %u defined in the MariaDB ", ib_table->name, n_keys, table->s->keys); } @@ -24946,7 +23164,7 @@ ib_push_frm_error( case DICT_FRM_CONSISTENT: default: sql_print_error("InnoDB: Table %s is consistent " - "on InnoDB data dictionary and MySQL " + "on InnoDB data dictionary and MariaDB " " FRM file.", ib_table->name); ut_error; diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index 84feea119c1..d11d5913f34 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -23,19 +23,6 @@ this program; if not, write to the Free Software Foundation, Inc., system clustered index when there is no primary key. */ extern const char innobase_index_reserve_name[]; -/* "innodb_file_per_table" tablespace name is reserved by InnoDB in order -to explicitly create a file_per_table tablespace for the table. */ -extern const char reserved_file_per_table_space_name[]; - -/* "innodb_system" tablespace name is reserved by InnoDB for the -system tablespace which uses space_id 0 and stores extra types of -system pages like UNDO and doublewrite. */ -extern const char reserved_system_space_name[]; - -/* "innodb_temporary" tablespace name is reserved by InnoDB for the -predefined shared temporary tablespace. */ -extern const char reserved_temporary_space_name[]; - /* Structure defines translation table between mysql index and InnoDB index structures */ struct innodb_idx_translate_t { @@ -119,6 +106,8 @@ public: const key_map* keys_to_use_for_scanning(); + void column_bitmaps_signal(); + /** Opens dictionary table object using table name. For partition, we need to try alternative lower/upper case names to support moving data files across platforms. @@ -203,9 +192,6 @@ public: int ft_read(uchar* buf); - int enable_indexes(uint mode); - int disable_indexes(uint mode); - void position(const uchar *record); int info(uint); @@ -458,7 +444,6 @@ protected: int end_stmt(); dberr_t innobase_get_autoinc(ulonglong* value); - void innobase_initialize_autoinc(); dberr_t innobase_lock_autoinc(); ulonglong innobase_peek_autoinc(); dberr_t innobase_set_max_autoinc(ulonglong auto_inc); @@ -468,9 +453,6 @@ protected: @see build_template() */ void reset_template(); - /** Write Row Interface optimized for Intrinsic table. */ - int intrinsic_table_write_row(uchar* record); - protected: inline void update_thd(THD* thd); void update_thd(); @@ -682,52 +664,6 @@ extern const char reserved_file_per_table_space_name[]; //extern "C" int wsrep_trx_is_aborting(void *thd_ptr); #endif -/** Check if the explicit tablespace targeted is file_per_table. -@param[in] create_info Metadata for the table to create. -@return true if the table is intended to use a file_per_table tablespace. */ -UNIV_INLINE -bool -tablespace_is_file_per_table( - const HA_CREATE_INFO* create_info) -{ - return(create_info->tablespace != NULL - && (0 == strcmp(create_info->tablespace, - reserved_file_per_table_space_name))); -} - -/** Check if table will be explicitly put in an existing shared general -or system tablespace. -@param[in] create_info Metadata for the table to create. -@return true if the table will use a shared general or system tablespace. */ -UNIV_INLINE -bool -tablespace_is_shared_space( -const HA_CREATE_INFO* create_info) -{ - return(create_info->tablespace != NULL - && create_info->tablespace[0] != '\0' - && (0 != strcmp(create_info->tablespace, - reserved_file_per_table_space_name))); -} - -/** Check if table will be explicitly put in a general tablespace. -@param[in] create_info Metadata for the table to create. -@return true if the table will use a general tablespace. */ -UNIV_INLINE -bool -tablespace_is_general_space( - const HA_CREATE_INFO* create_info) -{ - return(create_info->tablespace != NULL - && create_info->tablespace[0] != '\0' - && (0 != strcmp(create_info->tablespace, - reserved_file_per_table_space_name)) - && (0 != strcmp(create_info->tablespace, - reserved_temporary_space_name)) - && (0 != strcmp(create_info->tablespace, - reserved_system_space_name))); -} - /** Parse hint for table and its indexes, and update the information in dictionary. @param[in] thd Connection thread @@ -752,16 +688,12 @@ public: TABLE* form, HA_CREATE_INFO* create_info, char* table_name, - char* temp_path, - char* remote_path, - char* tablespace) + char* remote_path) :m_thd(thd), m_form(form), m_create_info(create_info), m_table_name(table_name), - m_temp_path(temp_path), m_remote_path(remote_path), - m_tablespace(tablespace), m_innodb_file_per_table(srv_file_per_table) {} @@ -787,6 +719,8 @@ public: @return NULL if valid, string name of bad option if not. */ const char* create_options_are_invalid(); + bool gcols_in_fulltext_or_spatial(); + /** Validates engine specific table options not handled by SQL-parser. @return NULL if valid, string name of bad option if not. */ @@ -798,9 +732,6 @@ public: /** Validate TABLESPACE option. */ bool create_option_tablespace_is_valid(); - /** Validate COMPRESSION option. */ - bool create_option_compression_is_valid(); - /** Prepare to create a table. */ int prepare_create_table(const char* name); @@ -833,14 +764,6 @@ public: THD* thd() const { return(m_thd); } - inline bool is_intrinsic_temp_table() const - { - /* DICT_TF2_INTRINSIC implies DICT_TF2_TEMPORARY */ - ut_ad(!(m_flags2 & DICT_TF2_INTRINSIC) - || (m_flags2 & DICT_TF2_TEMPORARY)); - return((m_flags2 & DICT_TF2_INTRINSIC) != 0); - } - /** Normalizes a table name string. A normalized name consists of the database name catenated to '/' and table name. An example: test/mytable. On Windows normalization puts @@ -879,19 +802,10 @@ private: /** Table name */ char* m_table_name; - /** If this is a table explicitly created by the user with the - TEMPORARY keyword, then this parameter is the dir path where the - table should be placed if we create an .ibd file for it - (no .ibd extension in the path, though). - Otherwise this is a zero length-string */ - char* m_temp_path; /** Remote path (DATA DIRECTORY) or zero length-string */ char* m_remote_path; - /** Tablespace name or zero length-string. */ - char* m_tablespace; - /** Local copy of srv_file_per_table. */ bool m_innodb_file_per_table; @@ -908,9 +822,6 @@ private: /** Using DATA DIRECTORY */ bool m_use_data_dir; - /** Using a Shared General Tablespace */ - bool m_use_shared_space; - /** Table flags */ ulint m_flags; @@ -1051,13 +962,9 @@ innodb_base_col_setup_for_stored( dict_s_col_t* s_col); /** whether this is a stored column */ -// JAN: TODO: MySQL 5.7 virtual fields -//#define innobase_is_s_fld(field) ((field)->gcol_info && (field)->stored_in_db) -#define innobase_is_s_fld(field) (field == NULL) -// JAN: TODO: MySQL 5.7 virtual fields +#define innobase_is_s_fld(field) ((field)->vcol_info && (field)->stored_in_db()) /** whether this is a computed virtual column */ -//#define innobase_is_v_fld(field) ((field)->gcol_info && !(field)->stored_in_db) -#define innobase_is_v_fld(field) (field == NULL) +#define innobase_is_v_fld(field) ((field)->vcol_info && !(field)->stored_in_db()) /** Release temporary latches. Call this function when mysqld passes control to the client. That is to @@ -1134,16 +1041,14 @@ innodb_rec_per_key( @param[in,out] s_templ InnoDB template structure @param[in] add_v new virtual columns added along with add index call -@param[in] locked true if innobase_share_mutex is held -@param[in] share_tbl_name original MySQL table name */ +@param[in] locked true if innobase_share_mutex is held */ void innobase_build_v_templ( const TABLE* table, const dict_table_t* ib_table, dict_vcol_templ_t* s_templ, const dict_add_v_col_t* add_v, - bool locked, - const char* share_tbl_name); + bool locked); /** callback used by MySQL server layer to initialized the table virtual columns' template @@ -1158,6 +1063,13 @@ innobase_build_v_templ_callback( the table virtual columns' template */ typedef void (*my_gcolumn_templatecallback_t)(const TABLE*, void*); +/** Convert MySQL column number to dict_table_t::cols[] offset. +@param[in] field non-virtual column +@return column number relative to dict_table_t::cols[] */ +unsigned +innodb_col_no(const Field* field) + MY_ATTRIBUTE((nonnull, warn_unused_result)); + /********************************************************************//** Helper function to push frm mismatch error to error log and if needed to sql-layer. */ @@ -1171,19 +1083,3 @@ ib_push_frm_error( ulint n_keys, /*!< in: InnoDB #keys */ bool push_warning); /*!< in: print warning ? */ -/*****************************************************************//** -Validates the create options. We may build on this function -in future. For now, it checks two specifiers: -KEY_BLOCK_SIZE and ROW_FORMAT -If innodb_strict_mode is not set then this function is a no-op -@return NULL if valid, string if not. */ -UNIV_INTERN -const char* -create_options_are_invalid( -/*=======================*/ - THD* thd, /*!< in: connection thread. */ - TABLE* form, /*!< in: information on table - columns and indexes */ - HA_CREATE_INFO* create_info, /*!< in: create info. */ - bool use_tablespace) /*!< in: srv_file_per_table */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); diff --git a/storage/innobase/handler/ha_innopart.cc b/storage/innobase/handler/ha_innopart.cc index 5fd02dfa016..6cdbd226b24 100644 --- a/storage/innobase/handler/ha_innopart.cc +++ b/storage/innobase/handler/ha_innopart.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2014, 2016, Oracle and/or its affiliates. All rights reserved. +Copyright (c) 2016, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -827,17 +828,6 @@ ha_innopart::alter_table_flags( return(HA_PARTITION_FUNCTION_SUPPORTED | HA_FAST_CHANGE_PARTITION); } -/** Internally called for initializing auto increment value. -Only called from ha_innobase::discard_or_import_table_space() -and should not do anything, since it is ha_innopart will initialize -it on first usage. */ -int -ha_innopart::innobase_initialize_autoinc() -{ - ut_ad(0); - return(0); -} - /** Set the autoinc column max value. This should only be called once from ha_innobase::open(). Therefore there's no need for a covering lock. @@ -892,78 +882,33 @@ ha_innopart::initialize_auto_increment( my_error(ER_AUTOINC_READ_FAILED, MYF(0)); error = HA_ERR_AUTOINC_READ_FAILED; } else { - dict_index_t* index; - const char* col_name; - ib_uint64_t read_auto_inc; - ib_uint64_t max_auto_inc = 0; - ulint err; - dict_table_t* ib_table; - ulonglong col_max_value; - - col_max_value = field->get_max_int_value(); + ib_uint64_t col_max_value = field->get_max_int_value(); update_thd(ha_thd()); - col_name = field->field_name; for (uint part = 0; part < m_tot_parts; part++) { - ib_table = m_part_share->get_table_part(part); + dict_table_t* ib_table + = m_part_share->get_table_part(part); dict_table_autoinc_lock(ib_table); - read_auto_inc = dict_table_autoinc_read(ib_table); - if (read_auto_inc != 0) { - set_if_bigger(max_auto_inc, read_auto_inc); - dict_table_autoinc_unlock(ib_table); - continue; - } - /* Execute SELECT MAX(col_name) FROM TABLE; */ - index = m_part_share->get_index( - part, table->s->next_number_index); - err = row_search_max_autoinc( - index, col_name, &read_auto_inc); - - switch (err) { - case DB_SUCCESS: { + ut_ad(ib_table->persistent_autoinc); + ib_uint64_t read_auto_inc + = dict_table_autoinc_read(ib_table); + if (read_auto_inc == 0) { + read_auto_inc = btr_read_autoinc( + dict_table_get_first_index(ib_table)); + /* At the this stage we do not know the increment nor the offset, so use a default increment of 1. */ - auto_inc = innobase_next_autoinc( + read_auto_inc = innobase_next_autoinc( read_auto_inc, 1, 1, 0, col_max_value); - set_if_bigger(max_auto_inc, auto_inc); dict_table_autoinc_initialize(ib_table, - auto_inc); - break; - } - case DB_RECORD_NOT_FOUND: - ib::error() << "MySQL and InnoDB data" - " dictionaries are out of sync. Unable" - " to find the AUTOINC column " - << col_name << " in the InnoDB table " - << index->table->name << ". We set the" - " next AUTOINC column value to 0, in" - " effect disabling the AUTOINC next" - " value generation."; - - ib::info() << "You can either set the next" - " AUTOINC value explicitly using ALTER" - " TABLE or fix the data dictionary by" - " recreating the table."; - - /* We want the open to succeed, so that the - user can take corrective action. ie. reads - should succeed but updates should fail. */ - - /* This will disable the AUTOINC generation. */ - auto_inc = 0; - goto done; - default: - /* row_search_max_autoinc() should only return - one of DB_SUCCESS or DB_RECORD_NOT_FOUND. */ - - ut_error; + read_auto_inc); } + set_if_bigger(auto_inc, read_auto_inc); dict_table_autoinc_unlock(ib_table); } - auto_inc = max_auto_inc; } done: @@ -1578,22 +1523,6 @@ ha_innopart::update_partition( DBUG_VOID_RETURN; } -/** Save currently highest auto increment value. -@param[in] nr Auto increment value to save. */ -void -ha_innopart::save_auto_increment( - ulonglong nr) -{ - - /* Store it in the shared dictionary of the partition. - TODO: When the new DD is done, store it in the table and make it - persistent! */ - - dict_table_autoinc_lock(m_prebuilt->table); - dict_table_autoinc_update_if_greater(m_prebuilt->table, nr + 1); - dict_table_autoinc_unlock(m_prebuilt->table); -} - /** Was the last returned row semi consistent read. In an UPDATE or DELETE, if the row under the cursor was locked by another transaction, and the engine used an optimistic read of the last @@ -2535,35 +2464,6 @@ ha_innopart::update_part_elem( } part_elem->index_file_name = NULL; - dict_get_and_save_space_name(ib_table, false); - if (ib_table->tablespace != NULL) { - ut_ad(part_elem->tablespace_name == NULL - || 0 == strcmp(part_elem->tablespace_name, - ib_table->tablespace)); - if (part_elem->tablespace_name == NULL - || strcmp(ib_table->tablespace, - part_elem->tablespace_name) != 0) { - - /* Play safe and allocate memory from TABLE and copy - instead of expose the internal data dictionary. */ - part_elem->tablespace_name = - strdup_root(&table->mem_root, - ib_table->tablespace); - } - } - else { - ut_ad(part_elem->tablespace_name == NULL - || 0 == strcmp(part_elem->tablespace_name, - "innodb_file_per_table")); - if (part_elem->tablespace_name != NULL - && 0 != strcmp(part_elem->tablespace_name, - "innodb_file_per_table")) { - - /* Update part_elem tablespace to NULL same as in - innodb data dictionary ib_table. */ - part_elem->tablespace_name = NULL; - } - } } /** Update create_info. @@ -2629,7 +2529,7 @@ ha_innopart::update_create_info( DBUG_VOID_RETURN; } - /* part_elem->data_file_name and tablespace_name should be correct from + /* part_elem->data_file_name should be correct from the .frm, but may have been changed, so update from SYS_DATAFILES. index_file_name is ignored, so remove it. */ @@ -2663,17 +2563,11 @@ set_create_info_dir( if (part_elem->data_file_name != NULL && part_elem->data_file_name[0] != '\0') { info->data_file_name = part_elem->data_file_name; - /* Also implies non-default tablespace. */ - info->tablespace = NULL; } if (part_elem->index_file_name != NULL && part_elem->index_file_name[0] != '\0') { info->index_file_name = part_elem->index_file_name; } - if (part_elem->tablespace_name != NULL - && part_elem->tablespace_name[0] != '\0') { - info->tablespace = part_elem->tablespace_name; - } } /** Set flags and append '/' to remote path if necessary. */ @@ -2713,17 +2607,13 @@ ha_innopart::create( int error; /** {database}/{tablename} */ char table_name[FN_REFLEN]; - /** absolute path of temp frm */ - char temp_path[FN_REFLEN]; /** absolute path of table */ char remote_path[FN_REFLEN]; char partition_name[FN_REFLEN]; - char tablespace_name[NAME_LEN + 1]; char* table_name_end; size_t table_name_len; char* partition_name_start; char table_data_file_name[FN_REFLEN]; - char table_level_tablespace_name[NAME_LEN + 1]; const char* index_file_name; size_t len; @@ -2731,9 +2621,7 @@ ha_innopart::create( form, create_info, table_name, - temp_path, - remote_path, - tablespace_name); + remote_path); DBUG_ENTER("ha_innopart::create"); ut_ad(create_info != NULL); @@ -2758,7 +2646,6 @@ ha_innopart::create( if (error != 0) { DBUG_RETURN(error); } - ut_ad(temp_path[0] == '\0'); strcpy(partition_name, table_name); partition_name_start = partition_name + strlen(partition_name); table_name_len = strlen(table_name); @@ -2779,11 +2666,6 @@ ha_innopart::create( table_data_file_name[0] = '\0'; } index_file_name = create_info->index_file_name; - if (create_info->tablespace != NULL) { - strcpy(table_level_tablespace_name, create_info->tablespace); - } else { - table_level_tablespace_name[0] = '\0'; - } info.allocate_trx(); @@ -2869,15 +2751,12 @@ ha_innopart::create( table_data_file_name; create_info->index_file_name = index_file_name; - create_info->tablespace = - table_level_tablespace_name; set_create_info_dir(part_elem, create_info); } } /* Reset table level DATA/INDEX DIRECTORY. */ create_info->data_file_name = table_data_file_name; create_info->index_file_name = index_file_name; - create_info->tablespace = table_level_tablespace_name; } innobase_commit_low(info.trx()); @@ -4309,27 +4188,13 @@ ha_innopart::create_new_partition( { int error; char norm_name[FN_REFLEN]; - const char* tablespace_name_backup = create_info->tablespace; const char* data_file_name_backup = create_info->data_file_name; DBUG_ENTER("ha_innopart::create_new_partition"); /* Delete by ddl_log on failure. */ normalize_table_name(norm_name, part_name); set_create_info_dir(part_elem, create_info); - /* The below check is the same as for CREATE TABLE, but since we are - doing an alter here it will not trigger the check in - create_option_tablespace_is_valid(). */ - if (tablespace_is_shared_space(create_info) - && create_info->data_file_name != NULL - && create_info->data_file_name[0] != '\0') { - my_printf_error(ER_ILLEGAL_HA_CREATE_OPTION, - "InnoDB: DATA DIRECTORY cannot be used" - " with a TABLESPACE assignment.", MYF(0)); - DBUG_RETURN(HA_WRONG_CREATE_OPTION); - } - error = ha_innobase::create(norm_name, table, create_info); - create_info->tablespace = tablespace_name_backup; create_info->data_file_name = data_file_name_backup; if (error == HA_ERR_FOUND_DUPP_KEY) { DBUG_RETURN(HA_ERR_TABLE_EXIST); diff --git a/storage/innobase/handler/ha_innopart.h b/storage/innobase/handler/ha_innopart.h index 8caa9cdd8d2..5f3f8eaa318 100644 --- a/storage/innobase/handler/ha_innopart.h +++ b/storage/innobase/handler/ha_innopart.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2014, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -723,12 +724,6 @@ private: int next_partition_index(); - /** Internally called for initializing auto increment value. - Should never be called, but defined to catch such errors. - @return 0 on success else error code. */ - int - innobase_initialize_autoinc(); - /** Get the index for the current partition @param[in] keynr MySQL index number. @return InnoDB index or NULL. */ @@ -772,12 +767,6 @@ private: initialize_auto_increment( bool /* no_lock */); - /** Save currently highest auto increment value. - @param[in] nr Auto increment value to save. */ - void - save_auto_increment( - ulonglong nr); - /** Setup the ordered record buffer and the priority queue. @param[in] used_parts Number of used partitions in query. @return false for success, else true. */ diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 9bbe3a010a4..d766dd6ada6 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -59,6 +59,10 @@ Smart ALTER TABLE #include <sql_acl.h> // PROCESS_ACL #endif +static const char *MSG_UNSUPPORTED_ALTER_ONLINE_ON_VIRTUAL_COLUMN= + "INPLACE ADD or DROP of virtual columns cannot be " + "combined with other ALTER TABLE actions"; + /* For supporting Native InnoDB Partitioning. */ /* JAN: TODO: MySQL 5.7 #include "partition_info.h" @@ -67,7 +71,8 @@ Smart ALTER TABLE /** Operations for creating secondary indexes (no rebuild needed) */ static const Alter_inplace_info::HA_ALTER_FLAGS INNOBASE_ONLINE_CREATE = Alter_inplace_info::ADD_INDEX - | Alter_inplace_info::ADD_UNIQUE_INDEX; + | Alter_inplace_info::ADD_UNIQUE_INDEX + | Alter_inplace_info::ADD_SPATIAL_INDEX; /** Operations for rebuilding a table in place */ static const Alter_inplace_info::HA_ALTER_FLAGS INNOBASE_ALTER_REBUILD @@ -77,17 +82,12 @@ static const Alter_inplace_info::HA_ALTER_FLAGS INNOBASE_ALTER_REBUILD /* CHANGE_CREATE_OPTION needs to check innobase_need_rebuild() */ | Alter_inplace_info::ALTER_COLUMN_NULLABLE | Alter_inplace_info::ALTER_COLUMN_NOT_NULLABLE - | Alter_inplace_info::ALTER_COLUMN_ORDER - | Alter_inplace_info::DROP_COLUMN - | Alter_inplace_info::ADD_COLUMN -#ifdef MYSQL_VIRTUAL_COLUMNS | Alter_inplace_info::ALTER_STORED_COLUMN_ORDER | Alter_inplace_info::DROP_STORED_COLUMN | Alter_inplace_info::ADD_STORED_BASE_COLUMN - | Alter_inplace_info::ALTER_STORED_COLUMN_TYPE -#endif | Alter_inplace_info::RECREATE_TABLE /* + | Alter_inplace_info::ALTER_STORED_COLUMN_TYPE */ ; @@ -101,9 +101,7 @@ static const Alter_inplace_info::HA_ALTER_FLAGS INNOBASE_INPLACE_IGNORE | Alter_inplace_info::ALTER_PARTITIONED | Alter_inplace_info::ALTER_COLUMN_COLUMN_FORMAT | Alter_inplace_info::ALTER_COLUMN_STORAGE_TYPE -#ifdef MYSQL_VIRTUAL_COLUMNS | Alter_inplace_info::ALTER_VIRTUAL_GCOL_EXPR -#endif | Alter_inplace_info::ALTER_RENAME; /** Operations on foreign key definitions (changing the schema only) */ @@ -121,15 +119,12 @@ static const Alter_inplace_info::HA_ALTER_FLAGS INNOBASE_ALTER_NOREBUILD | Alter_inplace_info::RENAME_INDEX #endif | Alter_inplace_info::ALTER_COLUMN_NAME - | Alter_inplace_info::ALTER_COLUMN_EQUAL_PACK_LENGTH; -#ifdef MYSQL_VIRTUAL_COLUMNS - | Alter_inplace_info::ALTER_INDEX_COMMENT + | Alter_inplace_info::ALTER_COLUMN_EQUAL_PACK_LENGTH + //| Alter_inplace_info::ALTER_INDEX_COMMENT | Alter_inplace_info::ADD_VIRTUAL_COLUMN | Alter_inplace_info::DROP_VIRTUAL_COLUMN - | Alter_inplace_info::ALTER_VIRTUAL_COLUMN_ORDER - | Alter_inplace_info::ALTER_VIRTUAL_COLUMN_TYPE -#endif - ; + | Alter_inplace_info::ALTER_VIRTUAL_COLUMN_ORDER; + struct ha_innobase_inplace_ctx : public inplace_alter_handler_ctx { /** Dummy query graph */ @@ -178,8 +173,6 @@ struct ha_innobase_inplace_ctx : public inplace_alter_handler_ctx const dtuple_t* add_cols; /** autoinc sequence to use */ ib_sequence_t sequence; - /** maximum auto-increment value */ - ulonglong max_autoinc; /** temporary table name to use for old table when renaming tables */ const char* tmp_name; /** whether the order of the clustered index is unchanged */ @@ -229,7 +222,6 @@ struct ha_innobase_inplace_ctx : public inplace_alter_handler_ctx add_cols (0), sequence(prebuilt->trx->mysql_thd, autoinc_col_min_value_arg, autoinc_col_max_value_arg), - max_autoinc (0), tmp_name (0), skip_pk_sort(false), num_to_add_vcol(0), @@ -453,7 +445,6 @@ innobase_need_rebuild( return(!!(ha_alter_info->handler_flags & INNOBASE_ALTER_REBUILD)); } -#ifdef MYSQL_VIRTUAL_COLUMNS /** Check if virtual column in old and new table are in order, excluding those dropped column. This is needed because when we drop a virtual column, ALTER_VIRTUAL_COLUMN_ORDER is also turned on, so we can't decide if this @@ -484,7 +475,7 @@ check_v_col_in_order( cf_it.rewind(); while (const Create_field* new_field = cf_it++) { - if (!new_field->is_virtual_gcol()) { + if (!innobase_is_v_fld(new_field)) { continue; } @@ -510,29 +501,15 @@ check_v_col_in_order( } for (ulint i = 0; i < table->s->fields; i++) { - Field* field = table->s->field[i]; - bool dropped = false; - Alter_drop* drop; + Field* field = table->field[i]; - if (field->stored_in_db) { + if (field->stored_in_db()) { continue; } ut_ad(innobase_is_v_fld(field)); - /* Check if this column is in drop list */ - List_iterator_fast<Alter_drop> cf_it( - ha_alter_info->alter_info->drop_list); - - while ((drop = (cf_it++)) != NULL) { - if (my_strcasecmp(system_charset_info, - field->field_name, drop->name) == 0) { - dropped = true; - break; - } - } - - if (dropped) { + if (field->flags & FIELD_IS_DROPPED) { continue; } @@ -541,7 +518,7 @@ check_v_col_in_order( while (j < altered_table->s->fields) { Field* new_field = altered_table->s->field[j]; - if (new_field->stored_in_db) { + if (new_field->stored_in_db()) { j++; continue; } @@ -567,7 +544,6 @@ check_v_col_in_order( return(true); } -#endif /* MYSQL_VIRTUAL_COLUMNS */ /** Check if InnoDB supports a particular alter table in-place @param altered_table TABLE object for new version of table. @@ -602,7 +578,7 @@ ha_innobase::check_if_supported_inplace_alter( DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); } - if (altered_table->s->stored_fields > REC_MAX_N_USER_FIELDS) { + if (altered_table->s->fields > REC_MAX_N_USER_FIELDS) { /* Deny the inplace ALTER TABLE. MySQL will try to re-create the table and ha_innobase::create() will return an error too. This is how we effectively @@ -612,21 +588,6 @@ ha_innobase::check_if_supported_inplace_alter( DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); } -#ifdef MYSQL_ENCRYPTION - /* We don't support change encryption attribute with - inplace algorithm. */ - char* old_encryption = this->table->s->encrypt_type.str; - char* new_encryption = altered_table->s->encrypt_type.str; - - if (Encryption::is_none(old_encryption) - != Encryption::is_none(new_encryption)) { - ha_alter_info->unsupported_reason = - innobase_get_err_msg( - ER_UNSUPPORTED_ALTER_ENCRYPTION_INPLACE); - DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); - } -#endif /* MYSQL_ENCRYPTION */ - update_thd(); trx_search_latch_release_if_reserved(m_prebuilt->trx); @@ -638,8 +599,7 @@ ha_innobase::check_if_supported_inplace_alter( ha_table_option_struct *old_options= table->s->option_struct; if (new_options->page_compressed != old_options->page_compressed || - new_options->page_compression_level != old_options->page_compression_level || - new_options->atomic_writes != old_options->atomic_writes) { + new_options->page_compression_level != old_options->page_compression_level) { ha_alter_info->unsupported_reason = innobase_get_err_msg( ER_ALTER_OPERATION_NOT_SUPPORTED_REASON); DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); @@ -658,14 +618,11 @@ ha_innobase::check_if_supported_inplace_alter( | INNOBASE_ALTER_NOREBUILD | INNOBASE_ALTER_REBUILD)) { -#ifdef MYSQL_VIRTUAL_COLUMNS if (ha_alter_info->handler_flags & Alter_inplace_info::ALTER_STORED_COLUMN_TYPE) { ha_alter_info->unsupported_reason = innobase_get_err_msg( ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_COLUMN_TYPE); } -#endif - DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); } @@ -806,8 +763,8 @@ ha_innobase::check_if_supported_inplace_alter( DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); } -#ifdef MYSQL_VIRTUAL_COLUMNS - // JAN: TODO: MySQL 5.7 Virtual columns + bool add_drop_v_cols = false; + /* If there is add or drop virtual columns, we will support operations with these 2 options alone with inplace interface for now */ @@ -824,8 +781,8 @@ ha_innobase::check_if_supported_inplace_alter( | Alter_inplace_info::DROP_VIRTUAL_COLUMN | Alter_inplace_info::ALTER_VIRTUAL_COLUMN_ORDER | Alter_inplace_info::ALTER_VIRTUAL_GCOL_EXPR + | Alter_inplace_info::ALTER_COLUMN_VCOL /* - | Alter_inplace_info::ALTER_STORED_COLUMN_ORDER | Alter_inplace_info::ADD_STORED_BASE_COLUMN | Alter_inplace_info::DROP_STORED_COLUMN | Alter_inplace_info::ALTER_STORED_COLUMN_ORDER @@ -840,14 +797,12 @@ ha_innobase::check_if_supported_inplace_alter( || (!check_v_col_in_order( this->table, altered_table, ha_alter_info))) { ha_alter_info->unsupported_reason = - innobase_get_err_msg( - ER_UNSUPPORTED_ALTER_INPLACE_ON_VIRTUAL_COLUMN); + MSG_UNSUPPORTED_ALTER_ONLINE_ON_VIRTUAL_COLUMN; DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); } add_drop_v_cols = true; } -#endif /* MYSQL_VIRTUAL_COLUMNS */ /* We should be able to do the operation in-place. See if we can do it online (LOCK=NONE). */ @@ -862,25 +817,23 @@ ha_innobase::check_if_supported_inplace_alter( + ha_alter_info->key_count; new_key++) { -#ifdef MYSQL_VIRTUAL_COLUMNS - /* Do not support adding/droping a vritual column, while + /* Do not support adding/droping a virtual column, while there is a table rebuild caused by adding a new FTS_DOC_ID */ if ((new_key->flags & HA_FULLTEXT) && add_drop_v_cols && !DICT_TF2_FLAG_IS_SET(m_prebuilt->table, DICT_TF2_FTS_HAS_DOC_ID)) { ha_alter_info->unsupported_reason = - innobase_get_err_msg( - ER_UNSUPPORTED_ALTER_INPLACE_ON_VIRTUAL_COLUMN); + MSG_UNSUPPORTED_ALTER_ONLINE_ON_VIRTUAL_COLUMN; DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); } -#endif /* MYSQL_VIRTUAL_COLUMNS */ for (KEY_PART_INFO* key_part = new_key->key_part; key_part < new_key->key_part + new_key->user_defined_key_parts; key_part++) { const Create_field* new_field; - DBUG_ASSERT(key_part->fieldnr < altered_table->s->fields); + DBUG_ASSERT(key_part->fieldnr + < altered_table->s->fields); cf_it.rewind(); for (uint fieldnr = 0; (new_field = cf_it++); @@ -939,31 +892,27 @@ ha_innobase::check_if_supported_inplace_alter( online = false; } -#ifdef MYSQL_VIRTUAL_COLUMNS - if (key_part->field->is_virtual_gcol()) { + if (innobase_is_v_fld(key_part->field)) { /* Do not support adding index on newly added virtual column, while there is also a drop virtual column in the same clause */ if (ha_alter_info->handler_flags & Alter_inplace_info::DROP_VIRTUAL_COLUMN) { ha_alter_info->unsupported_reason = - innobase_get_err_msg( - ER_UNSUPPORTED_ALTER_INPLACE_ON_VIRTUAL_COLUMN); + MSG_UNSUPPORTED_ALTER_ONLINE_ON_VIRTUAL_COLUMN; DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); } ha_alter_info->unsupported_reason = - innobase_get_err_msg( - ER_UNSUPPORTED_ALTER_ONLINE_ON_VIRTUAL_COLUMN); + MSG_UNSUPPORTED_ALTER_ONLINE_ON_VIRTUAL_COLUMN; online = false; } -#endif /* MYSQL_VIRTUAL_COLUMNS */ } } DBUG_ASSERT(!m_prebuilt->table->fts || m_prebuilt->table->fts->doc_col - <= table->s->stored_fields); + <= table->s->fields); DBUG_ASSERT(!m_prebuilt->table->fts || m_prebuilt->table->fts->doc_col < dict_table_get_n_user_cols(m_prebuilt->table)); @@ -1225,29 +1174,29 @@ innobase_set_foreign_key_option( switch (fk_key->delete_opt) { // JAN: TODO: ? MySQL 5.7 used enum fk_option directly from sql_lex.h - case Foreign_key::FK_OPTION_NO_ACTION: - case Foreign_key::FK_OPTION_RESTRICT: - case Foreign_key::FK_OPTION_DEFAULT: + case FK_OPTION_NO_ACTION: + case FK_OPTION_RESTRICT: + case FK_OPTION_SET_DEFAULT: foreign->type = DICT_FOREIGN_ON_DELETE_NO_ACTION; break; - case Foreign_key::FK_OPTION_CASCADE: + case FK_OPTION_CASCADE: foreign->type = DICT_FOREIGN_ON_DELETE_CASCADE; break; - case Foreign_key::FK_OPTION_SET_NULL: + case FK_OPTION_SET_NULL: foreign->type = DICT_FOREIGN_ON_DELETE_SET_NULL; break; } switch (fk_key->update_opt) { - case Foreign_key::FK_OPTION_NO_ACTION: - case Foreign_key::FK_OPTION_RESTRICT: - case Foreign_key::FK_OPTION_DEFAULT: + case FK_OPTION_NO_ACTION: + case FK_OPTION_RESTRICT: + case FK_OPTION_SET_DEFAULT: foreign->type |= DICT_FOREIGN_ON_UPDATE_NO_ACTION; break; - case Foreign_key::FK_OPTION_CASCADE: + case FK_OPTION_CASCADE: foreign->type |= DICT_FOREIGN_ON_UPDATE_CASCADE; break; - case Foreign_key::FK_OPTION_SET_NULL: + case FK_OPTION_SET_NULL: foreign->type |= DICT_FOREIGN_ON_UPDATE_SET_NULL; break; } @@ -1367,8 +1316,6 @@ next_rec: return(NULL); } -#ifdef MYSQL_VIRTUAL_COLUMNS - /** Check whether given column is a base of stored column. @param[in] col_name column name @param[in] table table @@ -1430,7 +1377,6 @@ innobase_check_fk_stored( return(false); } -#endif /* MYSQL_VIRTUAL_COLUMNS */ /** Create InnoDB foreign key structure from MySQL alter_info @param[in] ha_alter_info alter table info @@ -1678,13 +1624,14 @@ innobase_get_foreign_key_info( goto err_exit; } -#ifdef MYSQL_VIRTUAL_COLUMNS if (innobase_check_fk_stored( add_fk[num_fk], table, s_cols)) { - my_error(ER_CANNOT_ADD_FOREIGN_BASE_COL_STORED, MYF(0)); + my_printf_error( + HA_ERR_UNSUPPORTED, + "Cannot add foreign key on the base column " + "of stored column", MYF(0)); goto err_exit; } -#endif num_fk++; } @@ -1805,24 +1752,19 @@ innobase_rec_to_mysql( const ulint* offsets)/*!< in: rec_get_offsets( rec, index, ...) */ { - uint n_fields = table->s->stored_fields; - uint sql_idx = 0; + uint n_fields = table->s->fields; ut_ad(n_fields == dict_table_get_n_user_cols(index->table) - !!(DICT_TF2_FLAG_IS_SET(index->table, DICT_TF2_FTS_HAS_DOC_ID))); - for (uint i = 0; i < n_fields; i++, sql_idx++) { - Field* field; + for (uint i = 0; i < n_fields; i++) { + Field* field = table->field[i]; ulint ipos; ulint ilen; const uchar* ifield; ulint prefix_col; - while (!((field= table->field[sql_idx])->stored_in_db())) { - sql_idx++; - } - field->reset(); ipos = dict_index_get_nth_col_or_prefix_pos( @@ -1853,7 +1795,9 @@ null_field: } /*************************************************************//** -Copies an InnoDB index entry to table->record[0]. */ +Copies an InnoDB index entry to table->record[0]. +This is used in preparation for print_keydup_error() from +inline add index */ void innobase_fields_to_mysql( /*=====================*/ @@ -1861,8 +1805,7 @@ innobase_fields_to_mysql( const dict_index_t* index, /*!< in: InnoDB index */ const dfield_t* fields) /*!< in: InnoDB index fields */ { - uint n_fields = table->s->stored_fields; - uint sql_idx = 0; + uint n_fields = table->s->fields; ulint num_v = 0; ut_ad(n_fields == dict_table_get_n_user_cols(index->table) @@ -1870,16 +1813,12 @@ innobase_fields_to_mysql( - !!(DICT_TF2_FLAG_IS_SET(index->table, DICT_TF2_FTS_HAS_DOC_ID))); - for (uint i = 0; i < n_fields; i++, sql_idx++) { - Field* field; + for (uint i = 0; i < n_fields; i++) { + Field* field = table->field[i]; ulint ipos; ulint col_n; ulint prefix_col; - while (!((field= table->field[sql_idx])->stored_in_db())) { - sql_idx++; - } - field->reset(); if (innobase_is_v_fld(field)) { @@ -1913,7 +1852,9 @@ innobase_fields_to_mysql( } /*************************************************************//** -Copies an InnoDB row to table->record[0]. */ +Copies an InnoDB row to table->record[0]. +This is used in preparation for print_keydup_error() from +row_log_table_apply() */ void innobase_row_to_mysql( /*==================*/ @@ -1921,9 +1862,8 @@ innobase_row_to_mysql( const dict_table_t* itab, /*!< in: InnoDB table */ const dtuple_t* row) /*!< in: InnoDB row */ { - uint n_fields = table->s->stored_fields; - uint sql_idx = 0; - uint num_v = 0; + uint n_fields = table->s->fields; + ulint num_v = 0; /* The InnoDB row may contain an extra FTS_DOC_ID column at the end. */ ut_ad(row->n_fields == dict_table_get_n_cols(itab)); @@ -1931,12 +1871,8 @@ innobase_row_to_mysql( + dict_table_get_n_v_cols(itab) - !!(DICT_TF2_FLAG_IS_SET(itab, DICT_TF2_FTS_HAS_DOC_ID))); - for (uint i = 0; i < n_fields; i++, sql_idx++) { - Field* field; - - while (!((field= table->field[sql_idx])->stored_in_db())) { - sql_idx++; - } + for (uint i = 0; i < n_fields; i++) { + Field* field = table->field[i]; field->reset(); @@ -1960,6 +1896,11 @@ innobase_row_to_mysql( dfield_get_len(df), field); } } + if (table->vfield) { + my_bitmap_map* old_vcol_set = tmp_use_all_columns(table, table->vcol_set); + table->update_virtual_fields(table->file, VCOL_UPDATE_FOR_READ); + tmp_restore_column_map(table->vcol_set, old_vcol_set); + } } /*************************************************************//** @@ -2136,22 +2077,19 @@ name_ok: is not being created @param[in] key_part MySQL key definition @param[in,out] index_field index field -@param[in] new_clustered new cluster -@param[in] fields MySQL table fields*/ +@param[in] new_clustered new cluster */ static void innobase_create_index_field_def( const TABLE* altered_table, const KEY_PART_INFO* key_part, index_field_t* index_field, - bool new_clustered, - const Field** fields) + bool new_clustered) { const Field* field; ibool is_unsigned; ulint col_type; ulint num_v = 0; - ulint num_m_v = 0; DBUG_ENTER("innobase_create_index_field_def"); @@ -2164,34 +2102,21 @@ innobase_create_index_field_def( ut_a(field); for (ulint i = 0; i < key_part->fieldnr; i++) { - const Field* ifield = altered_table->field[i]; - if (innobase_is_v_fld(ifield)) { + if (innobase_is_v_fld(altered_table->field[i])) { num_v++; } - - if (!ifield->stored_in_db()) { - num_m_v++; - } } col_type = get_innobase_type_from_mysql_type( &is_unsigned, field); -#ifdef MYSQL_VIRTUAL_COLUMNS - if (!field->stored_in_db && field->gcol_info) { - if (!field->stored_in_db && false) { - index_field->is_v_col = true; - index_field->col_no = num_v; - } else { - index_field->is_v_col = false; - index_field->col_no = key_part->fieldnr - num_v; - } + if (innobase_is_v_fld(field)) { + index_field->is_v_col = true; + index_field->col_no = num_v; + } else { + index_field->is_v_col = false; + index_field->col_no = key_part->fieldnr - num_v; } -#else - index_field->is_v_col = false; - index_field->col_no = key_part->fieldnr - num_m_v; - index_field->col_name = altered_table ? field->field_name : fields[key_part->fieldnr]->field_name; -#endif /* MYSQL_VIRTUAL_COLUMNS */ if (DATA_LARGE_MTYPE(col_type) || (key_part->length < field->pack_length() @@ -2226,8 +2151,7 @@ innobase_create_index_def( bool new_clustered, bool key_clustered, index_def_t* index, - mem_heap_t* heap, - const Field** fields) + mem_heap_t* heap) { const KEY* key = &keys[key_number]; ulint i; @@ -2306,21 +2230,16 @@ innobase_create_index_def( index->fields[0].prefix_len = 0; index->fields[0].is_v_col = false; - #ifdef MYSQL_VIRTUAL_COLUMNS - if (!key->key_part[0].field->stored_in_db - && key->key_part[0].field->gcol_info) { + if (innobase_is_v_fld(key->key_part[0].field)) { /* Currently, the spatial index cannot be created on virtual columns. It is blocked in server layer */ - ut_ad(0); index->fields[0].is_v_col = true; } else { - index->fields[0].is_v_col = false; } -#endif /* MYSQL_VIRTUAL_COLUMNS */ } else { index->ind_type = (key->flags & HA_NOSAME) ? DICT_UNIQUE : 0; } @@ -2329,7 +2248,7 @@ innobase_create_index_def( for (i = 0; i < n_fields; i++) { innobase_create_index_field_def( altered_table, &key->key_part[i], - &index->fields[i], new_clustered, fields); + &index->fields[i], new_clustered); if (index->fields[i].is_v_col) { index->ind_type |= DICT_VIRTUAL; @@ -2360,18 +2279,13 @@ innobase_fts_check_doc_id_col( { *fts_doc_col_no = ULINT_UNDEFINED; - const uint n_cols = altered_table->s->stored_fields; - uint sql_idx = 0; + const uint n_cols = altered_table->s->fields; ulint i; *num_v = 0; - for (i = 0; i < n_cols; i++, sql_idx++) { - const Field* field; - - while (!((field= altered_table->field[sql_idx])->stored_in_db())) { - sql_idx++; - } + for (i = 0; i < n_cols; i++) { + const Field* field = altered_table->field[i]; if (innobase_is_v_fld(field)) { (*num_v)++; @@ -2684,7 +2598,7 @@ innobase_create_key_defs( /* Create the PRIMARY key index definition */ innobase_create_index_def( altered_table, key_info, primary_key_number, - TRUE, TRUE, indexdef++, heap, (const Field **)altered_table->field); + true, true, indexdef++, heap); created_clustered: n_add = 1; @@ -2696,7 +2610,7 @@ created_clustered: /* Copy the index definitions. */ innobase_create_index_def( altered_table, key_info, i, true, - false, indexdef, heap, (const Field **)altered_table->field); + false, indexdef, heap); if (indexdef->ind_type & DICT_FTS) { n_fts_add++; @@ -2713,8 +2627,7 @@ created_clustered: && !innobase_fts_check_doc_id_col( NULL, altered_table, &fts_doc_id_col, &num_v)) { - fts_doc_id_col = - altered_table->s->stored_fields; + fts_doc_id_col = altered_table->s->fields - num_v; add_fts_doc_id = true; } @@ -2743,7 +2656,7 @@ created_clustered: for (ulint i = 0; i < n_add; i++) { innobase_create_index_def( altered_table, key_info, add[i], - false, false, indexdef, heap, (const Field **)altered_table->field); + false, false, indexdef, heap); if (indexdef->ind_type & DICT_FTS) { n_fts_add++; @@ -3144,16 +3057,15 @@ innobase_build_col_map( dtuple_t* add_cols, mem_heap_t* heap) { - uint old_i, old_innobase_i; DBUG_ENTER("innobase_build_col_map"); DBUG_ASSERT(altered_table != table); DBUG_ASSERT(new_table != old_table); DBUG_ASSERT(dict_table_get_n_cols(new_table) + dict_table_get_n_v_cols(new_table) - >= altered_table->s->stored_fields + DATA_N_SYS_COLS); + >= altered_table->s->fields + DATA_N_SYS_COLS); DBUG_ASSERT(dict_table_get_n_cols(old_table) + dict_table_get_n_v_cols(old_table) - >= table->s->stored_fields + DATA_N_SYS_COLS); + >= table->s->fields + DATA_N_SYS_COLS); DBUG_ASSERT(!!add_cols == !!(ha_alter_info->handler_flags & Alter_inplace_info::ADD_COLUMN)); DBUG_ASSERT(!add_cols || dtuple_get_n_fields(add_cols) @@ -3166,14 +3078,13 @@ innobase_build_col_map( List_iterator_fast<Create_field> cf_it( ha_alter_info->alter_info->create_list); - uint i = 0, sql_idx = 0; + uint i = 0; uint num_v = 0; /* Any dropped columns will map to ULINT_UNDEFINED. */ - for (old_innobase_i = 0; - old_innobase_i + DATA_N_SYS_COLS < old_table->n_cols; - old_innobase_i++) { - col_map[old_innobase_i] = ULINT_UNDEFINED; + for (uint old_i = 0; old_i + DATA_N_SYS_COLS < old_table->n_cols; + old_i++) { + col_map[old_i] = ULINT_UNDEFINED; } for (uint old_i = 0; old_i < old_table->n_v_cols; old_i++) { @@ -3189,21 +3100,8 @@ innobase_build_col_map( ulint num_old_v = 0; - if (!new_field->stored_in_db()) - { - sql_idx++; - continue; - } - - for (old_i = 0, old_innobase_i= 0; - table->field[old_i]; - old_i++) { + for (uint old_i = 0; table->field[old_i]; old_i++) { const Field* field = table->field[old_i]; - - if (!table->field[old_i]->stored_in_db()) { - continue; - } - if (innobase_is_v_fld(field)) { if (is_v && new_field->field == field) { col_map[old_table->n_cols + num_v] @@ -3216,30 +3114,27 @@ innobase_build_col_map( } if (new_field->field == field) { - col_map[old_innobase_i] = i; + col_map[old_i - num_old_v] = i; goto found_col; } - old_innobase_i++; } ut_ad(!is_v); innobase_build_col_map_add( heap, dtuple_get_nth_field(add_cols, i), - altered_table->field[sql_idx], + altered_table->field[i + num_v], dict_table_is_comp(new_table)); found_col: if (is_v) { num_v++; } else { i++; - sql_idx++; } } - DBUG_ASSERT(i == altered_table->s->stored_fields - num_v); + DBUG_ASSERT(i == altered_table->s->fields - num_v); - i = table->s->stored_fields; - //i = table->s->fields - old_table->n_v_cols; + i = table->s->fields - old_table->n_v_cols; /* Add the InnoDB hidden FTS_DOC_ID column, if any. */ if (i + DATA_N_SYS_COLS < old_table->n_cols) { @@ -3249,21 +3144,21 @@ found_col: DICT_TF2_FTS_HAS_DOC_ID)); DBUG_ASSERT(i + DATA_N_SYS_COLS + 1 == old_table->n_cols); DBUG_ASSERT(!strcmp(dict_table_get_col_name( - old_table, table->s->stored_fields), + old_table, i), FTS_DOC_ID_COL_NAME)); - if (altered_table->s->stored_fields + DATA_N_SYS_COLS + if (altered_table->s->fields + DATA_N_SYS_COLS - new_table->n_v_cols < new_table->n_cols) { DBUG_ASSERT(DICT_TF2_FLAG_IS_SET( new_table, DICT_TF2_FTS_HAS_DOC_ID)); - DBUG_ASSERT(altered_table->s->stored_fields + DBUG_ASSERT(altered_table->s->fields + DATA_N_SYS_COLS + 1 - == new_table->n_cols); - col_map[i] = altered_table->s->stored_fields; - /* col_map[i] = altered_table->s->fields + == static_cast<ulint>( + new_table->n_cols + + new_table->n_v_cols)); + col_map[i] = altered_table->s->fields - new_table->n_v_cols; - */ } else { DBUG_ASSERT(!DICT_TF2_FLAG_IS_SET( new_table, @@ -3679,8 +3574,6 @@ innobase_check_gis_columns( DBUG_RETURN(DB_SUCCESS); } -#ifdef MYSQL_VIRTUAL_COLUMNS - /** Collect virtual column info for its addition @param[in] ha_alter_info Data used during in-place alter @param[in] altered_table MySQL table that is being altered to @@ -3746,10 +3639,8 @@ prepare_inplace_add_virtual( &is_unsigned, field); - if (!field->gcol_info || field->stored_in_db) { - my_error(ER_WRONG_KEY_COLUMN, MYF(0), - field->field_name); - return(true); + if (!innobase_is_v_fld(field)) { + continue; } col_len = field->pack_length(); @@ -3775,7 +3666,7 @@ prepare_inplace_add_virtual( charset_no += MAX_CHAR_COLL_NUM;); if (charset_no > MAX_CHAR_COLL_NUM) { - my_error(ER_WRONG_KEY_COLUMN, MYF(0), + my_error(ER_WRONG_KEY_COLUMN, MYF(0), "InnoDB", field->field_name); return(true); } @@ -3806,13 +3697,9 @@ prepare_inplace_add_virtual( ctx->add_vcol[j].m_col.len = col_len; ctx->add_vcol[j].m_col.ind = i - 1; - /* ctx->add_vcol[j].num_base = - field->gcol_info->non_virtual_base_columns(); - */ + ctx->add_vcol[j].num_base = 0; ctx->add_vcol_name[j] = field->field_name; - ctx->add_vcol[j].base_col = static_cast<dict_col_t**>( - mem_heap_alloc(ctx->heap, ctx->add_vcol[j].num_base - * sizeof *(ctx->add_vcol[j].base_col))); + ctx->add_vcol[j].base_col = NULL; ctx->add_vcol[j].v_pos = ctx->old_table->n_v_cols - ctx->num_to_drop_vcol + j; @@ -3841,12 +3728,17 @@ prepare_inplace_drop_virtual( ha_innobase_inplace_ctx* ctx; ulint i = 0; ulint j = 0; - Alter_drop *drop; ctx = static_cast<ha_innobase_inplace_ctx*> (ha_alter_info->handler_ctx); - ctx->num_to_drop_vcol = ha_alter_info->alter_info->drop_list.elements; + ctx->num_to_drop_vcol = 0; + for (i = 0; table->field[i]; i++) { + const Field* field = table->field[i]; + if (field->flags & FIELD_IS_DROPPED && !field->stored_in_db()) { + ctx->num_to_drop_vcol++; + } + } ctx->drop_vcol = static_cast<dict_v_col_t*>( mem_heap_alloc(ctx->heap, ctx->num_to_drop_vcol @@ -3855,26 +3747,9 @@ prepare_inplace_drop_virtual( mem_heap_alloc(ctx->heap, ctx->num_to_drop_vcol * sizeof *ctx->drop_vcol_name)); - List_iterator_fast<Alter_drop> cf_it( - ha_alter_info->alter_info->drop_list); - - while ((drop = (cf_it++)) != NULL) { - const Field* field; - ulint old_i; - - ut_ad(drop->type == Alter_drop::COLUMN); - - for (old_i = 0; table->field[old_i]; old_i++) { - const Field* n_field = table->field[old_i]; - if (!my_strcasecmp(system_charset_info, - n_field->field_name, drop->name)) { - break; - } - } - - i++; - - if (!table->field[old_i]) { + for (i = 0; table->field[i]; i++) { + Field *field = table->field[i]; + if (!(field->flags & FIELD_IS_DROPPED) || field->stored_in_db()) { continue; } @@ -3883,19 +3758,10 @@ prepare_inplace_drop_virtual( ulint field_type; ulint charset_no; - field = table->field[old_i]; - ulint col_type = get_innobase_type_from_mysql_type( &is_unsigned, field); - - if (!field->gcol_info || field->stored_in_db) { - my_error(ER_WRONG_KEY_COLUMN, MYF(0), - field->field_name); - return(true); - } - col_len = field->pack_length(); field_type = (ulint) field->type(); @@ -3919,7 +3785,7 @@ prepare_inplace_drop_virtual( charset_no += MAX_CHAR_COLL_NUM;); if (charset_no > MAX_CHAR_COLL_NUM) { - my_error(ER_WRONG_KEY_COLUMN, MYF(0), + my_error(ER_WRONG_KEY_COLUMN, MYF(0), "InnoDB", field->field_name); return(true); } @@ -3949,12 +3815,12 @@ prepare_inplace_drop_virtual( ctx->drop_vcol[j].m_col.len = col_len; - ctx->drop_vcol[j].m_col.ind = old_i; + ctx->drop_vcol[j].m_col.ind = i; ctx->drop_vcol_name[j] = field->field_name; dict_v_col_t* v_col = dict_table_get_nth_v_col_mysql( - ctx->old_table, old_i); + ctx->old_table, i); ctx->drop_vcol[j].v_pos = v_col->v_pos; j++; } @@ -4412,7 +4278,7 @@ innodb_v_adjust_idx_col( /* Found the field in the new table */ while (const Create_field* new_field = cf_it++) { - if (!new_field->is_virtual_gcol()) { + if (!innobase_is_v_fld(new_field)) { continue; } @@ -4432,7 +4298,7 @@ innodb_v_adjust_idx_col( ut_a(0); } - ut_ad(field->is_virtual_gcol()); + ut_ad(innobase_is_v_fld(field)); num_v = 0; @@ -4446,7 +4312,7 @@ innodb_v_adjust_idx_col( break; } - if (old_table->field[old_i]->is_virtual_gcol()) { + if (innobase_is_v_fld(old_table->field[old_i])) { num_v++; } } @@ -4454,7 +4320,6 @@ innodb_v_adjust_idx_col( ut_ad(col_found); } } -#endif /* MYSQL_VIRTUAL_COLUMNS */ /** Update internal structures with concurrent writes blocked, while preparing ALTER TABLE. @@ -4493,7 +4358,6 @@ prepare_inplace_alter_table_dict( dict_index_t* fts_index = NULL; ulint new_clustered = 0; dberr_t error; - const char* punch_hole_warning = NULL; ulint num_fts_index; dict_add_v_col_t* add_v = NULL; ha_innobase_inplace_ctx*ctx; @@ -4519,7 +4383,6 @@ prepare_inplace_alter_table_dict( trx_start_if_not_started_xa(ctx->prebuilt->trx, true); -#ifdef MYSQL_VIRTUAL_COLUMNS if (ha_alter_info->handler_flags & Alter_inplace_info::DROP_VIRTUAL_COLUMN) { if (prepare_inplace_drop_virtual( @@ -4548,12 +4411,9 @@ prepare_inplace_alter_table_dict( } } - /* - There should be no order change for virtual columns coming in - here - */ + /* There should be no order change for virtual columns coming in + here */ ut_ad(check_v_col_in_order(old_table, altered_table, ha_alter_info)); -#endif /* MYSQL_VIRTUAL_COLUMNS */ /* Create a background transaction for the operations on the data dictionary tables. */ @@ -4668,12 +4528,11 @@ prepare_inplace_alter_table_dict( ctx->new_table->id); ulint n_cols = 0; ulint n_v_cols = 0; - ulint n_mv_cols = 0; dtuple_t* add_cols; ulint space_id = 0; + ulint z = 0; ulint key_id = FIL_DEFAULT_ENCRYPTION_KEY; fil_encryption_t mode = FIL_SPACE_ENCRYPTION_DEFAULT; - const char* compression=NULL; crypt_data = fil_space_get_crypt_data(ctx->prebuilt->table->space); @@ -4694,15 +4553,11 @@ prepare_inplace_alter_table_dict( if (innobase_is_v_fld(field)) { n_v_cols++; } else { - if (field->stored_in_db()) { n_cols++; - } else { - n_mv_cols++; - } } } - ut_ad(n_cols + n_v_cols + n_mv_cols == altered_table->s->fields); + ut_ad(n_cols + n_v_cols == altered_table->s->fields); if (add_fts_doc_id) { n_cols++; @@ -4724,20 +4579,6 @@ prepare_inplace_alter_table_dict( goto new_clustered_failed; } - /* Use the old tablespace unless the tablespace - is changing. */ - if (DICT_TF_HAS_SHARED_SPACE(user_table->flags) - && (ha_alter_info->create_info->tablespace == NULL - || (0 == strcmp(ha_alter_info->create_info->tablespace, - user_table->tablespace)))) { - space_id = user_table->space; - } else if (tablespace_is_shared_space( - ha_alter_info->create_info)) { - space_id = fil_space_get_id_by_name( - ha_alter_info->create_info->tablespace); - ut_a(space_id != ULINT_UNDEFINED); - } - /* The initial space id 0 may be overridden later if this table is going to be a file_per_table tablespace. */ ctx->new_table = dict_mem_table_create( @@ -4754,21 +4595,17 @@ prepare_inplace_alter_table_dict( user_table->data_dir_path); } - for (uint i = 0, sql_idx=0; i < altered_table->s->stored_fields; i++, sql_idx++) { - Field* field; + for (uint i = 0; i < altered_table->s->fields; i++) { + const Field* field = altered_table->field[i]; ulint is_unsigned; - ulint charset_no; - ulint col_len; - - while (!((field= altered_table->field[sql_idx])->stored_in_db())) { - sql_idx++; - } - - ulint field_type = (ulint) field->type(); - bool is_virtual = innobase_is_v_fld(field); + ulint field_type + = (ulint) field->type(); ulint col_type = get_innobase_type_from_mysql_type( &is_unsigned, field); + ulint charset_no; + ulint col_len; + bool is_virtual = innobase_is_v_fld(field); /* we assume in dtype_form_prtype() that this fits in two bytes */ @@ -4792,7 +4629,7 @@ prepare_inplace_alter_table_dict( if (charset_no > MAX_CHAR_COLL_NUM) { dict_mem_table_free( ctx->new_table); - my_error(ER_WRONG_KEY_COLUMN, MYF(0), + my_error(ER_WRONG_KEY_COLUMN, MYF(0), "InnoDB", field->field_name); goto new_clustered_failed; } @@ -4835,7 +4672,6 @@ prepare_inplace_alter_table_dict( } if (is_virtual) { -#ifdef MYSQL_VIRTUAL_COLUMNS dict_mem_table_add_v_col( ctx->new_table, ctx->heap, field->field_name, @@ -4843,9 +4679,7 @@ prepare_inplace_alter_table_dict( dtype_form_prtype( field_type, charset_no) | DATA_VIRTUAL, - col_len, i, - field->gcol_info->non_virtual_base_columns()); -#endif /* MYSQL_VIRTUAL_COLUMNS */ + col_len, i, 0); } else { dict_mem_table_add_col( ctx->new_table, ctx->heap, @@ -4857,9 +4691,6 @@ prepare_inplace_alter_table_dict( } } -#ifdef MYSQL_VIRTUAL_COLUMNS - ulint z = 0; - if (n_v_cols) { for (uint i = 0; i < altered_table->s->fields; i++) { dict_v_col_t* v_col; @@ -4875,52 +4706,21 @@ prepare_inplace_alter_table_dict( ctx->new_table, field, v_col); } } -#endif /* MYSQL_VIRTUAL_COLUMNS */ if (add_fts_doc_id) { fts_add_doc_id_column(ctx->new_table, ctx->heap); ctx->new_table->fts->doc_col = fts_doc_id_col; ut_ad(fts_doc_id_col - == altered_table->s->stored_fields - n_v_cols); - ut_ad(fts_doc_id_col == altered_table->s->stored_fields); - + == altered_table->s->fields - n_v_cols); } else if (ctx->new_table->fts) { ctx->new_table->fts->doc_col = fts_doc_id_col; } -#ifdef MYSQL_COMPRESSION - compression = ha_alter_info->create_info->compress.str; - - if (Compression::validate(compression) != DB_SUCCESS) { - - compression = NULL; - } -#endif /* MYSQL_COMPRESSION */ - error = row_create_table_for_mysql( - ctx->new_table, compression, ctx->trx, false, mode, key_id); - - punch_hole_warning = - (error == DB_IO_NO_PUNCH_HOLE_FS) - ? "Punch hole is not supported by the file system" - : "Page Compression is not supported for this" - " tablespace"; + ctx->new_table, ctx->trx, false, mode, key_id); switch (error) { dict_table_t* temp_table; - case DB_IO_NO_PUNCH_HOLE_FS: - case DB_IO_NO_PUNCH_HOLE_TABLESPACE: - push_warning_printf( - ctx->prebuilt->trx->mysql_thd, - Sql_condition::WARN_LEVEL_WARN, - HA_ERR_UNSUPPORTED, - "%s. Compression disabled for '%s'", - punch_hole_warning, - ctx->new_table->name.m_name); - - error = DB_SUCCESS; - - case DB_SUCCESS: /* We need to bump up the table ref count and before we can use it we need to open the @@ -5033,14 +4833,12 @@ new_clustered_failed: for (ulint a = 0; a < ctx->num_to_add_index; a++) { -#ifdef MYSQL_VIRTUAL_COLUMNS if (index_defs[a].ind_type & DICT_VIRTUAL && ctx->num_to_drop_vcol > 0 && !new_clustered) { innodb_v_adjust_idx_col(ha_alter_info, old_table, ctx->num_to_drop_vcol, &index_defs[a]); } -#endif /* MYSQL_VIRTUAL_COLUMNS */ ctx->add_index[a] = row_merge_create_index( ctx->trx, ctx->new_table, @@ -5114,6 +4912,23 @@ new_clustered_failed: DBUG_EXECUTE_IF("innodb_alter_table_pk_assert_no_sort", DBUG_ASSERT(ctx->skip_pk_sort);); + DBUG_ASSERT(!ctx->new_table->persistent_autoinc); + if (const Field* ai = altered_table->found_next_number_field) { + const unsigned col_no = innodb_col_no(ai); + + ctx->new_table->persistent_autoinc = 1 + + dict_table_get_nth_col_pos( + ctx->new_table, col_no, NULL); + + /* Initialize the AUTO_INCREMENT sequence + to the rebuilt table from the old one. */ + if (!old_table->found_next_number_field) { + } else if (ib_uint64_t autoinc + = btr_read_autoinc(clust_index)) { + btr_write_autoinc(new_clust_index, autoinc); + } + } + if (ctx->online) { /* Allocate a log for online table rebuild. */ rw_lock_x_lock(&clust_index->lock); @@ -5351,9 +5166,6 @@ innobase_check_foreign_key_index( ulint n_drop_fk) /*!< in: Number of foreign keys to drop */ { - ut_ad(index != NULL); - ut_ad(indexed_table != NULL); - const dict_foreign_set* fks = &indexed_table->referenced_set; /* Check for all FK references from other tables to the index. */ @@ -5578,7 +5390,6 @@ rename_index_in_cache( DBUG_VOID_RETURN; } -# /** Rename all indexes in data dictionary cache of a given table that are specified in ha_alter_info. @@ -5612,7 +5423,6 @@ rename_indexes_in_cache( } #endif /* MYSQL_RENAME_INDEX */ -#ifdef MYSQL_VIRTUAL_COLUMNS /** Fill the stored column information in s_cols list. @param[in] altered_table mysql table object @param[in] table innodb table object @@ -5637,7 +5447,7 @@ alter_fill_stored_column( continue; } - ulint num_base = field->gcol_info->non_virtual_base_columns(); + ulint num_base = 0; dict_col_t* col = dict_table_get_nth_col(table, i); s_col.m_col = col; @@ -5660,7 +5470,7 @@ alter_fill_stored_column( (*s_cols)->push_back(s_col); } } -#endif /* MYSQL_VIRTUAL_COLUMNS */ + /** Allows InnoDB to update internal structures with concurrent writes blocked (provided that check_if_supported_inplace_alter() @@ -5769,37 +5579,21 @@ ha_innobase::prepare_inplace_alter_table( /* ALTER TABLE will not implicitly move a table from a single-table tablespace to the system tablespace when innodb_file_per_table=OFF. But it will implicitly move a table from the system tablespace to a - single-table tablespace if innodb_file_per_table = ON. - Tables found in a general tablespace will stay there unless ALTER - TABLE contains another TABLESPACE=name. If that is found it will - explicitly move a table to the named tablespace. - So if you specify TABLESPACE=`innodb_system` a table can be moved - into the system tablespace from either a general or file-per-table - tablespace. But from then on, it is labeled as using a shared space - (the create options have tablespace=='innodb_system' and the - SHARED_SPACE flag is set in the table flags) so it can no longer be - implicitly moved to a file-per-table tablespace. */ - bool in_system_space = is_system_tablespace(indexed_table->space); - bool is_file_per_table = !in_system_space - && !DICT_TF_HAS_SHARED_SPACE(indexed_table->flags); -#ifdef UNIV_DEBUG - bool in_general_space = !in_system_space - && DICT_TF_HAS_SHARED_SPACE(indexed_table->flags); - - /* The table being altered can only be in a system tablespace, - or its own file-per-table tablespace, or a general tablespace. */ - ut_ad(1 == in_system_space + is_file_per_table + in_general_space); -#endif /* UNIV_DEBUG */ + single-table tablespace if innodb_file_per_table = ON. */ create_table_info_t info(m_user_thd, altered_table, ha_alter_info->create_info, NULL, - NULL, - NULL, NULL); - info.set_tablespace_type(is_file_per_table); + info.set_tablespace_type(indexed_table->space != TRX_SYS_SPACE); + + if (ha_alter_info->handler_flags & Alter_inplace_info::ADD_INDEX) { + if (info.gcols_in_fulltext_or_spatial()) { + goto err_exit_no_heap; + } + } if (ha_alter_info->handler_flags & Alter_inplace_info::CHANGE_CREATE_OPTION) { @@ -6188,7 +5982,7 @@ check_if_can_drop_indexes: if (!index->to_be_dropped && dict_index_is_corrupted(index)) { my_error(ER_INDEX_CORRUPT, MYF(0), index->name()); - DBUG_RETURN(true); + goto err_exit; } } } @@ -6239,10 +6033,9 @@ check_if_can_drop_indexes: & Alter_inplace_info::ADD_FOREIGN_KEY) { ut_ad(!m_prebuilt->trx->check_foreigns); -#ifdef MYSQL_VIRTUAL_COLUMNS alter_fill_stored_column(altered_table, m_prebuilt->table, &s_cols, &s_heap); -#endif + add_fk = static_cast<dict_foreign_t**>( mem_heap_zalloc( heap, @@ -6313,7 +6106,6 @@ err_exit: } -#ifdef MYSQL_VIRTUAL_COLUMNS if ((ha_alter_info->handler_flags & Alter_inplace_info::DROP_VIRTUAL_COLUMN) && prepare_inplace_drop_virtual( @@ -6327,7 +6119,6 @@ err_exit: ha_alter_info, altered_table, table)) { DBUG_RETURN(true); } -#endif /* MYSQL_VIRTUAL_COLUMNS */ DBUG_RETURN(false); } @@ -6343,7 +6134,7 @@ err_exit: m_prebuilt->table, altered_table, &fts_doc_col_no, &num_v)) { - fts_doc_col_no = altered_table->s->stored_fields; + fts_doc_col_no = altered_table->s->fields - num_v; add_fts_doc_id = true; add_fts_doc_id_idx = true; @@ -6371,36 +6162,24 @@ err_exit: DBUG_ASSERT( doc_col_no == fts_doc_col_no || doc_col_no == ULINT_UNDEFINED - || (ha_alter_info->handler_flags)); - /* JAN: TODO: MySQL 5.7 Virtual columns + || (ha_alter_info->handler_flags & (Alter_inplace_info::ALTER_STORED_COLUMN_ORDER | Alter_inplace_info::DROP_STORED_COLUMN - | - Alter_inplace_info::ADD_STORED_COLUMN))); - */ + | Alter_inplace_info::ADD_STORED_BASE_COLUMN))); } } /* See if an AUTO_INCREMENT column was added. */ - uint i = 0, innodb_idx= 0; + uint i = 0; ulint num_v = 0; List_iterator_fast<Create_field> cf_it( ha_alter_info->alter_info->create_list); while (const Create_field* new_field = cf_it++) { const Field* field; - if (!new_field->stored_in_db()) { - i++; - continue; - } DBUG_ASSERT(i < altered_table->s->fields); - DBUG_ASSERT(innodb_idx < altered_table->s->stored_fields); for (uint old_i = 0; table->field[old_i]; old_i++) { - if (!table->field[old_i]->stored_in_db()) { - continue; - } - if (new_field->field == table->field[old_i]) { goto found_col; } @@ -6424,12 +6203,10 @@ err_exit: my_error(ER_WRONG_AUTO_KEY, MYF(0)); goto err_exit; } - add_autoinc_col_no = innodb_idx; - /* JAN: TODO: MySQL 5.7 - autoinc_col_max_value = - field->get_max_int_value(); - */ + /* Get the col no of the old table non-virtual column array */ + add_autoinc_col_no = i - num_v; + autoinc_col_max_value = innobase_get_int_col_max_value(field); } found_col: @@ -6438,7 +6215,6 @@ found_col: } i++; - innodb_idx++; } DBUG_ASSERT(heap); @@ -6612,7 +6388,6 @@ ok_exit: && alter_templ_needs_rebuild( altered_table, ha_alter_info, ctx->new_table)); -#ifdef MYSQL_VIRTUAL_COLUMNS if ((ctx->new_table->n_v_cols > 0) && rebuild_templ) { /* Save the templ if isn't NULL so as to restore the original state in case of alter operation failures. */ @@ -6620,11 +6395,9 @@ ok_exit: old_templ = ctx->new_table->vc_templ; } s_templ = UT_NEW_NOKEY(dict_vcol_templ_t()); - s_templ->vtempl = NULL; innobase_build_v_templ( - altered_table, ctx->new_table, s_templ, - NULL, false, NULL); + altered_table, ctx->new_table, s_templ, NULL, false); ctx->new_table->vc_templ = s_templ; } else if (ctx->num_to_add_vcol > 0 && ctx->num_to_drop_vcol == 0) { @@ -6641,11 +6414,8 @@ ok_exit: add_v->v_col = ctx->add_vcol; add_v->v_col_name = ctx->add_vcol_name; - s_templ->vtempl = NULL; - innobase_build_v_templ( - altered_table, ctx->new_table, s_templ, - add_v, false, NULL); + altered_table, ctx->new_table, s_templ, add_v, false); old_templ = ctx->new_table->vc_templ; ctx->new_table->vc_templ = s_templ; } @@ -6656,7 +6426,6 @@ ok_exit: if (!ctx->need_rebuild() && ctx->num_to_drop_vcol > 0) { eval_table = table; } -#endif /* MYSQL_VIRTUAL_COLUMNS */ /* Read the clustered index of the table and build indexes based on this information using temporary @@ -6800,9 +6569,10 @@ innobase_online_rebuild_log_free( /** For each user column, which is part of an index which is not going to be dropped, it checks if the column number of the column is same as col_no argument passed. -@param[in] table table object -@param[in] col_no column number of the column which is to be checked -@param[in] is_v if this is a virtual column +@param[in] table table +@param[in] col_no column number +@param[in] is_v if this is a virtual column +@param[in] only_committed whether to consider only committed indexes @retval true column exists @retval false column does not exist, true if column is system column or it is in the index. */ @@ -6811,17 +6581,21 @@ bool check_col_exists_in_indexes( const dict_table_t* table, ulint col_no, - bool is_v) + bool is_v, + bool only_committed = false) { /* This function does not check system columns */ if (!is_v && dict_table_get_nth_col(table, col_no)->mtype == DATA_SYS) { return(true); } - for (dict_index_t* index = dict_table_get_first_index(table); index; + for (const dict_index_t* index = dict_table_get_first_index(table); + index; index = dict_table_get_next_index(index)) { - if (index->to_be_dropped) { + if (only_committed + ? !index->is_committed() + : index->to_be_dropped) { continue; } @@ -7000,14 +6774,24 @@ func_exit: we do this by checking every existing column, if any current index would index them */ for (ulint i = 0; i < dict_table_get_n_cols(prebuilt->table); i++) { - if (!check_col_exists_in_indexes(prebuilt->table, i, false)) { - prebuilt->table->cols[i].ord_part = 0; + dict_col_t& col = prebuilt->table->cols[i]; + if (!col.ord_part) { + continue; + } + if (!check_col_exists_in_indexes(prebuilt->table, i, false, + true)) { + col.ord_part = 0; } } for (ulint i = 0; i < dict_table_get_n_v_cols(prebuilt->table); i++) { - if (!check_col_exists_in_indexes(prebuilt->table, i, true)) { - prebuilt->table->v_cols[i].m_col.ord_part = 0; + dict_col_t& col = prebuilt->table->v_cols[i].m_col; + if (!col.ord_part) { + continue; + } + if (!check_col_exists_in_indexes(prebuilt->table, i, true, + true)) { + col.ord_part = 0; } } @@ -7463,14 +7247,13 @@ innobase_enlarge_columns_try( List_iterator_fast<Create_field> cf_it( ha_alter_info->alter_info->create_list); ulint i = 0; - bool is_v=false; + ulint num_v = 0; + bool is_v; for (Field** fp = table->field; *fp; fp++, i++) { ulint idx; -#ifdef MYSQL_VIRTUAL_COLUMNS - ulint num_v = 0; - if ((*fp)->is_virtual_gcol()) { + if (innobase_is_v_fld(*fp)) { is_v = true; idx = num_v; num_v++; @@ -7478,10 +7261,6 @@ innobase_enlarge_columns_try( idx = i - num_v; is_v = false; } -#else - idx = i; - is_v = false; -#endif cf_it.rewind(); while (Create_field* cf = cf_it++) { @@ -7490,7 +7269,7 @@ innobase_enlarge_columns_try( == IS_EQUAL_PACK_LENGTH && innobase_enlarge_column_try( user_table, trx, table_name, - idx, cf->length, is_v)) { + idx, static_cast<ulint>(cf->length), is_v)) { return(true); } @@ -7565,85 +7344,120 @@ innobase_rename_or_enlarge_columns_cache( } } -/** Get the auto-increment value of the table on commit. +/** Set the auto-increment value of the table on commit. @param ha_alter_info Data used during in-place alter @param ctx In-place ALTER TABLE context @param altered_table MySQL table that is being altered -@param old_table MySQL table as it is before the ALTER operation -@return the next auto-increment value (0 if not present) */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -ulonglong -commit_get_autoinc( -/*===============*/ +@param old_table MySQL table as it is before the ALTER operation */ +static MY_ATTRIBUTE((nonnull)) +void +commit_set_autoinc( Alter_inplace_info* ha_alter_info, ha_innobase_inplace_ctx*ctx, const TABLE* altered_table, const TABLE* old_table) { - ulonglong max_autoinc; - DBUG_ENTER("commit_get_autoinc"); if (!altered_table->found_next_number_field) { /* There is no AUTO_INCREMENT column in the table after the ALTER operation. */ - max_autoinc = 0; } else if (ctx->add_autoinc != ULINT_UNDEFINED) { + ut_ad(ctx->need_rebuild()); /* An AUTO_INCREMENT column was added. Get the last value from the sequence, which may be based on a supplied AUTO_INCREMENT value. */ - max_autoinc = ctx->sequence.last(); + ib_uint64_t autoinc = ctx->sequence.last(); + ctx->new_table->autoinc = autoinc; + /* Bulk index creation does not update + PAGE_ROOT_AUTO_INC, so we must persist the "last used" + value here. */ + btr_write_autoinc(dict_table_get_first_index(ctx->new_table), + autoinc - 1, true); } else if ((ha_alter_info->handler_flags & Alter_inplace_info::CHANGE_CREATE_OPTION) && (ha_alter_info->create_info->used_fields & HA_CREATE_USED_AUTO)) { - /* An AUTO_INCREMENT value was supplied, but the table was not - rebuilt. Get the user-supplied value or the last value from the - sequence. */ - ib_uint64_t max_value_table; - dberr_t err; - - Field* autoinc_field = - old_table->found_next_number_field; - - dict_index_t* index = dict_table_get_index_on_first_col( - ctx->old_table, autoinc_field->field_index, - autoinc_field->field_name); - - max_autoinc = ha_alter_info->create_info->auto_increment_value; - - dict_table_autoinc_lock(ctx->old_table); - - err = row_search_max_autoinc( - index, autoinc_field->field_name, &max_value_table); - - if (err != DB_SUCCESS) { - ut_ad(0); - max_autoinc = 0; - } else if (max_autoinc <= max_value_table) { - ulonglong col_max_value; - ulonglong offset; - - col_max_value = innobase_get_int_col_max_value(autoinc_field); - // JAN: TODO: MySQL 5.7 - //col_max_value = autoinc_field->get_max_int_value(); - - offset = ctx->prebuilt->autoinc_offset; - max_autoinc = innobase_next_autoinc( - max_value_table, 1, 1, offset, - col_max_value); + /* An AUTO_INCREMENT value was supplied by the user. + It must be persisted to the data file. */ + const Field* ai = old_table->found_next_number_field; + ut_ad(!strcmp(dict_table_get_col_name(ctx->old_table, + innodb_col_no(ai)), + ai->field_name)); + + ib_uint64_t autoinc + = ha_alter_info->create_info->auto_increment_value; + if (autoinc == 0) { + autoinc = 1; + } + + if (autoinc >= ctx->old_table->autoinc) { + /* Persist the predecessor of the + AUTO_INCREMENT value as the last used one. */ + ctx->new_table->autoinc = autoinc--; + } else { + /* Mimic ALGORITHM=COPY in the following scenario: + + CREATE TABLE t (a SERIAL); + INSERT INTO t SET a=100; + ALTER TABLE t AUTO_INCREMENT = 1; + INSERT INTO t SET a=NULL; + SELECT * FROM t; + + By default, ALGORITHM=INPLACE would reset the + sequence to 1, while after ALGORITHM=COPY, the + last INSERT would use a value larger than 100. + + We could only search the tree to know current + max counter in the table and compare. */ + const dict_col_t* autoinc_col + = dict_table_get_nth_col(ctx->old_table, + innodb_col_no(ai)); + dict_index_t* index + = dict_table_get_first_index(ctx->old_table); + while (index != NULL + && index->fields[0].col != autoinc_col) { + index = dict_table_get_next_index(index); + } + + ut_ad(index); + + ib_uint64_t max_in_table = index + ? row_search_max_autoinc(index) + : 0; + + if (autoinc <= max_in_table) { + ctx->new_table->autoinc = innobase_next_autoinc( + max_in_table, 1, + ctx->prebuilt->autoinc_increment, + ctx->prebuilt->autoinc_offset, + innobase_get_int_col_max_value(ai)); + /* Persist the maximum value as the + last used one. */ + autoinc = max_in_table; + } else { + /* Persist the predecessor of the + AUTO_INCREMENT value as the last used one. */ + ctx->new_table->autoinc = autoinc--; + } } - dict_table_autoinc_unlock(ctx->old_table); - } else { - /* An AUTO_INCREMENT value was not specified. - Read the old counter value from the table. */ - ut_ad(old_table->found_next_number_field); - dict_table_autoinc_lock(ctx->old_table); - max_autoinc = ctx->old_table->autoinc; - dict_table_autoinc_unlock(ctx->old_table); + + btr_write_autoinc(dict_table_get_first_index(ctx->new_table), + autoinc, true); + } else if (ctx->need_rebuild()) { + /* No AUTO_INCREMENT value was specified. + Copy it from the old table. */ + ctx->new_table->autoinc = ctx->old_table->autoinc; + /* The persistent value was already copied in + prepare_inplace_alter_table_dict() when ctx->new_table + was created. If this was a LOCK=NONE operation, the + AUTO_INCREMENT values would be updated during + row_log_table_apply(). If this was LOCK!=NONE, + the table contents could not possibly have changed + between prepare_inplace and commit_inplace. */ } - DBUG_RETURN(max_autoinc); + DBUG_VOID_RETURN; } /** Add or drop foreign key constraints to the data dictionary tables, @@ -7708,7 +7522,7 @@ innobase_update_foreign_try( names, while the columns in ctx->old_table have not been renamed yet. */ error = dict_create_add_foreign_to_dictionary( - (dict_table_t*)ctx->old_table,ctx->old_table->name.m_name, fk, trx); + ctx->old_table->name.m_name, fk, trx); DBUG_EXECUTE_IF( "innodb_test_cannot_add_fk_system", @@ -7977,26 +7791,11 @@ commit_try_rebuild( user_table, rebuilt_table, ctx->tmp_name, trx); /* We must be still holding a table handle. */ - DBUG_ASSERT(user_table->get_ref_count() >= 1); + DBUG_ASSERT(user_table->get_ref_count() == 1); DBUG_EXECUTE_IF("ib_ddl_crash_after_rename", DBUG_SUICIDE();); DBUG_EXECUTE_IF("ib_rebuild_cannot_rename", error = DB_ERROR;); - if (user_table->get_ref_count() > 1) { - /* This should only occur when an innodb_memcached - connection with innodb_api_enable_mdl=off was started - before commit_inplace_alter_table() locked the data - dictionary. We must roll back the ALTER TABLE, because - we cannot drop a table while it is being used. */ - - /* Normally, n_ref_count must be 1, because purge - cannot be executing on this very table as we are - holding dict_operation_lock X-latch. */ - my_printf_error(ER_ILLEGAL_HA, "Cannot complete the operation " - "because table is referenced by another connection.", MYF(0)); - DBUG_RETURN(true); - } - switch (error) { case DB_SUCCESS: DBUG_RETURN(false); @@ -8224,7 +8023,6 @@ commit_try_norebuild( } #endif /* MYSQL_RENAME_INDEX */ -#ifdef MYSQL_VIRTUAL_COLUMNS if ((ha_alter_info->handler_flags & Alter_inplace_info::DROP_VIRTUAL_COLUMN) && innobase_drop_virtual_try( @@ -8240,7 +8038,6 @@ commit_try_norebuild( ctx->old_table, trx)) { DBUG_RETURN(true); } -#endif /* MYSQL_VIRTUAL_COLUMNS */ DBUG_RETURN(false); } @@ -8726,8 +8523,7 @@ ha_innobase::commit_inplace_alter_table( DBUG_ASSERT(new_clustered == ctx->need_rebuild()); - ctx->max_autoinc = commit_get_autoinc( - ha_alter_info, ctx, altered_table, table); + commit_set_autoinc(ha_alter_info, ctx, altered_table, table); if (ctx->need_rebuild()) { ctx->tmp_name = dict_mem_create_temporary_tablename( @@ -9003,15 +8799,7 @@ foreign_fail: } if (ctx0->num_to_drop_vcol || ctx0->num_to_add_vcol) { - - if (ctx0->old_table->get_ref_count() > 1) { - - row_mysql_unlock_data_dictionary(trx); - trx_free_for_mysql(trx); - my_printf_error(ER_ILLEGAL_HA, "Cannot complete the operation " - "because table is referenced by another connection.", MYF(0)); - DBUG_RETURN(true); - } + DBUG_ASSERT(ctx0->old_table->get_ref_count() == 1); trx_commit_for_mysql(m_prebuilt->trx); @@ -9065,14 +8853,6 @@ foreign_fail: (*pctx); DBUG_ASSERT(ctx->need_rebuild() == new_clustered); - if (altered_table->found_next_number_field) { - dict_table_t* t = ctx->new_table; - - dict_table_autoinc_lock(t); - dict_table_autoinc_initialize(t, ctx->max_autoinc); - dict_table_autoinc_unlock(t); - } - bool add_fts = false; /* Publish the created fulltext index, if any. @@ -9221,10 +9001,6 @@ foreign_fail: } } - /* We don't support compression for the system tablespace nor - the temporary tablespace. Only because they are shared tablespaces. - There is no other technical reason. */ - innobase_parse_hint_from_comment( m_user_thd, m_prebuilt->table, altered_table->s); diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 37bc9c1f15d..a400ee54758 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2016, Oracle and/or its affiliates. -Copyrigth (c) 2014, 2016, MariaDB Corporation +Copyrigth (c) 2014, 2017, MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -4097,287 +4097,6 @@ UNIV_INTERN struct st_maria_plugin i_s_innodb_ft_config = STRUCT_FLD(maturity, MariaDB_PLUGIN_MATURITY_STABLE), }; -/* Fields of the dynamic table INNODB_TEMP_TABLE_INFO. */ -static ST_FIELD_INFO i_s_innodb_temp_table_info_fields_info[] = -{ -#define IDX_TEMP_TABLE_ID 0 - {STRUCT_FLD(field_name, "TABLE_ID"), - STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE)}, - -#define IDX_TEMP_TABLE_NAME 1 - {STRUCT_FLD(field_name, "NAME"), - STRUCT_FLD(field_length, MAX_TABLE_UTF8_LEN), - STRUCT_FLD(field_type, MYSQL_TYPE_STRING), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_MAYBE_NULL), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE)}, - -#define IDX_TEMP_TABLE_N_COLS 2 - {STRUCT_FLD(field_name, "N_COLS"), - STRUCT_FLD(field_length, MY_INT32_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE)}, - -#define IDX_TEMP_TABLE_SPACE_ID 3 - {STRUCT_FLD(field_name, "SPACE"), - STRUCT_FLD(field_length, MY_INT32_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE)}, - -#define IDX_TEMP_TABLE_PTT 4 - {STRUCT_FLD(field_name, "PER_TABLE_TABLESPACE"), - STRUCT_FLD(field_length, 64), - STRUCT_FLD(field_type, MYSQL_TYPE_STRING), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_MAYBE_NULL), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE)}, - -#define IDX_TEMP_TABLE_IS_COMPRESSED 5 - {STRUCT_FLD(field_name, "IS_COMPRESSED"), - STRUCT_FLD(field_length, 64), - STRUCT_FLD(field_type, MYSQL_TYPE_STRING), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_MAYBE_NULL), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE)}, - END_OF_ST_FIELD_INFO -}; - -struct temp_table_info_t{ - table_id_t m_table_id; - char m_table_name[MAX_TABLE_UTF8_LEN]; - unsigned m_n_cols; - unsigned m_space_id; - char m_per_table_tablespace[64]; - char m_is_compressed[64]; -}; - -typedef std::vector<temp_table_info_t, ut_allocator<temp_table_info_t> > - temp_table_info_cache_t; - -/*******************************************************************//** -Fill Information Schema table INNODB_TEMP_TABLE_INFO for a particular -temp-table -@return 0 on success, 1 on failure */ -static -int -i_s_innodb_temp_table_info_fill( -/*=============================*/ - THD* thd, /*!< in: thread */ - TABLE_LIST* tables, /*!< in/out: tables - to fill */ - const temp_table_info_t* info) /*!< in: temp-table - information */ -{ - TABLE* table; - Field** fields; - - DBUG_ENTER("i_s_innodb_temp_table_info_fill"); - - table = tables->table; - - fields = table->field; - - OK(fields[IDX_TEMP_TABLE_ID]->store(info->m_table_id, true)); - - OK(field_store_string( - fields[IDX_TEMP_TABLE_NAME], info->m_table_name)); - - OK(fields[IDX_TEMP_TABLE_N_COLS]->store(info->m_n_cols)); - - OK(fields[IDX_TEMP_TABLE_SPACE_ID]->store(info->m_space_id)); - - OK(field_store_string( - fields[IDX_TEMP_TABLE_PTT], info->m_per_table_tablespace)); - - OK(field_store_string( - fields[IDX_TEMP_TABLE_IS_COMPRESSED], info->m_is_compressed)); - - DBUG_RETURN(schema_table_store_record(thd, table)); -} - -/*******************************************************************//** -Populate current table information to cache */ -static -void -innodb_temp_table_populate_cache( -/*=============================*/ - const dict_table_t* table, /*! in: table */ - temp_table_info_t* cache) /*! in/out: populate data in this - cache */ -{ - cache->m_table_id = table->id; - - char db_utf8[MAX_DB_UTF8_LEN]; - char table_utf8[MAX_TABLE_UTF8_LEN]; - - dict_fs2utf8(table->name.m_name, - db_utf8, sizeof(db_utf8), - table_utf8, sizeof(table_utf8)); - strcpy(cache->m_table_name, table_utf8); - - cache->m_n_cols = table->n_cols; - - cache->m_space_id = table->space; - - if (fsp_is_system_temporary(table->space)) { - strcpy(cache->m_per_table_tablespace, "FALSE"); - } else { - strcpy(cache->m_per_table_tablespace, "TRUE"); - } - - if (dict_table_page_size(table).is_compressed()) { - strcpy(cache->m_is_compressed, "TRUE"); - } else { - strcpy(cache->m_is_compressed, "FALSE"); - } -} - -/*******************************************************************//** -This function will iterate over all available table and will fill -stats for temp-tables to INNODB_TEMP_TABLE_INFO. -@return 0 on success, 1 on failure */ -static -int -i_s_innodb_temp_table_info_fill_table( -/*===================================*/ - THD* thd, /*!< in: thread */ - TABLE_LIST* tables, /*!< in/out: tables to fill */ - Item* ) /*!< in: condition (ignored) */ -{ - int status = 0; - dict_table_t* table = NULL; - - DBUG_ENTER("i_s_innodb_temp_table_info_fill_table"); - - /* Only allow the PROCESS privilege holder to access the stats */ - if (check_global_access(thd, PROCESS_ACL)) { - DBUG_RETURN(0); - } - - /* First populate all temp-table info by acquiring dict_sys->mutex. - Note: Scan is being done on NON-LRU list which mainly has system - table entries and temp-table entries. This means 2 things: list - is smaller so processing would be faster and most of the data - is relevant */ - temp_table_info_cache_t all_temp_info_cache; - all_temp_info_cache.reserve(UT_LIST_GET_LEN(dict_sys->table_non_LRU)); - - mutex_enter(&dict_sys->mutex); - for (table = UT_LIST_GET_FIRST(dict_sys->table_non_LRU); - table != NULL; - table = UT_LIST_GET_NEXT(table_LRU, table)) { - - if (!dict_table_is_temporary(table)) { - continue; - } - - temp_table_info_t current_temp_table_info; - - innodb_temp_table_populate_cache( - table, ¤t_temp_table_info); - - all_temp_info_cache.push_back(current_temp_table_info); - } - mutex_exit(&dict_sys->mutex); - - /* Now populate the info to MySQL table */ - temp_table_info_cache_t::const_iterator end = all_temp_info_cache.end(); - for (temp_table_info_cache_t::const_iterator it - = all_temp_info_cache.begin(); - it != end; - it++) { - status = i_s_innodb_temp_table_info_fill(thd, tables, &(*it)); - if (status) { - break; - } - } - - DBUG_RETURN(status); -} - -/*******************************************************************//** -Bind the dynamic table INFORMATION_SCHEMA.INNODB_TEMP_TABLE_INFO. -@return 0 on success, 1 on failure */ -static -int -i_s_innodb_temp_table_info_init( -/*=============================*/ - void* p) /*!< in/out: table schema object */ -{ - ST_SCHEMA_TABLE* schema; - - DBUG_ENTER("i_s_innodb_temp_table_info_init"); - - schema = reinterpret_cast<ST_SCHEMA_TABLE*>(p); - - schema->fields_info = i_s_innodb_temp_table_info_fields_info; - schema->fill_table = i_s_innodb_temp_table_info_fill_table; - - DBUG_RETURN(0); -} - -struct st_maria_plugin i_s_innodb_temp_table_info = -{ - /* the plugin type (a MYSQL_XXX_PLUGIN value) */ - /* int */ - STRUCT_FLD(type, MYSQL_INFORMATION_SCHEMA_PLUGIN), - - /* pointer to type-specific plugin descriptor */ - /* void* */ - STRUCT_FLD(info, &i_s_info), - - /* plugin name */ - /* const char* */ - STRUCT_FLD(name, "INNODB_TEMP_TABLE_INFO"), - - /* plugin author (for SHOW PLUGINS) */ - /* const char* */ - STRUCT_FLD(author, plugin_author), - - /* general descriptive text (for SHOW PLUGINS) */ - /* const char* */ - STRUCT_FLD(descr, "InnoDB Temp Table Stats"), - - /* the plugin license (PLUGIN_LICENSE_XXX) */ - /* int */ - STRUCT_FLD(license, PLUGIN_LICENSE_GPL), - - /* the function to invoke when plugin is loaded */ - /* int (*)(void*); */ - STRUCT_FLD(init, i_s_innodb_temp_table_info_init), - - /* the function to invoke when plugin is unloaded */ - /* int (*)(void*); */ - STRUCT_FLD(deinit, i_s_common_deinit), - - /* plugin version (for SHOW PLUGINS) */ - /* unsigned int */ - STRUCT_FLD(version, INNODB_VERSION_SHORT), - - /* struct st_mysql_show_var* */ - STRUCT_FLD(status_vars, NULL), - - /* struct st_mysql_sys_var** */ - STRUCT_FLD(system_vars, NULL), - - STRUCT_FLD(version_info, INNODB_VERSION_STR), - STRUCT_FLD(maturity, MariaDB_PLUGIN_MATURITY_GAMMA) -}; - /* Fields of the dynamic table INNODB_BUFFER_POOL_STATS. */ static ST_FIELD_INFO i_s_innodb_buffer_stats_fields_info[] = { @@ -6331,8 +6050,6 @@ i_s_dict_fill_sys_tables( if (is_system_tablespace(table->space)) { space_type = "System"; - } else if (DICT_TF_HAS_SHARED_SPACE(table->flags)) { - space_type = "General"; } else { space_type = "Single"; } @@ -8326,9 +8043,6 @@ i_s_dict_fill_sys_tablespaces( file_format = trx_sys_file_format_id_to_name(atomic_blobs); if (is_system_tablespace(space)) { row_format = "Compact or Redundant"; - } else if (fsp_is_shared_tablespace(flags) && !is_compressed) { - file_format = "Any"; - row_format = "Any"; } else if (is_compressed) { row_format = "Compressed"; } else if (atomic_blobs) { @@ -8339,8 +8053,6 @@ i_s_dict_fill_sys_tablespaces( if (is_system_tablespace(space)) { space_type = "System"; - } else if (fsp_is_shared_tablespace(flags)) { - space_type = "General"; } else { space_type = "Single"; } @@ -8370,8 +8082,7 @@ i_s_dict_fill_sys_tablespaces( space_type)); char* filepath = NULL; - if (FSP_FLAGS_HAS_DATA_DIR(flags) - || FSP_FLAGS_GET_SHARED(flags)) { + if (FSP_FLAGS_HAS_DATA_DIR(flags)) { mutex_enter(&dict_sys->mutex); filepath = dict_get_first_path(space); mutex_exit(&dict_sys->mutex); @@ -9406,11 +9117,8 @@ i_s_innodb_mutexes_fill_table( TABLE_LIST* tables, /*!< in/out: tables to fill */ Item* ) /*!< in: condition (not used) */ { - ib_mutex_t* mutex; rw_lock_t* lock; - ulint block_mutex_oswait_count = 0; ulint block_lock_oswait_count = 0; - ib_mutex_t* block_mutex = NULL; rw_lock_t* block_lock = NULL; Field** fields = tables->table->field; @@ -9425,6 +9133,9 @@ i_s_innodb_mutexes_fill_table( // mutex_enter(&mutex_list_mutex); #ifdef JAN_TODO_FIXME + ib_mutex_t* mutex; + ulint block_mutex_oswait_count = 0; + ib_mutex_t* block_mutex = NULL; for (mutex = UT_LIST_GET_FIRST(os_mutex_list); mutex != NULL; mutex = UT_LIST_GET_NEXT(list, mutex)) { if (mutex->count_os_wait == 0) { diff --git a/storage/innobase/handler/i_s.h b/storage/innobase/handler/i_s.h index d272490dd27..8d34fbf8fbb 100644 --- a/storage/innobase/handler/i_s.h +++ b/storage/innobase/handler/i_s.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyrigth (c) 2014, 2016, MariaDB Corporation +Copyrigth (c) 2014, 2017, MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -50,7 +50,6 @@ extern struct st_maria_plugin i_s_innodb_ft_config; extern struct st_maria_plugin i_s_innodb_buffer_page; extern struct st_maria_plugin i_s_innodb_buffer_page_lru; extern struct st_maria_plugin i_s_innodb_buffer_stats; -extern struct st_maria_plugin i_s_innodb_temp_table_info; extern struct st_maria_plugin i_s_innodb_sys_tables; extern struct st_maria_plugin i_s_innodb_sys_tablestats; extern struct st_maria_plugin i_s_innodb_sys_indexes; diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index e9ad1a9d8c0..e6c36b425e6 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, MariaDB Corporation. +Copyright (c) 2016, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -46,8 +46,6 @@ my_bool srv_ibuf_disable_background_merge; #include "ibuf0ibuf.ic" #endif -#ifndef UNIV_HOTBACKUP - #include "buf0buf.h" #include "buf0rea.h" #include "fsp0fsp.h" @@ -595,7 +593,6 @@ ibuf_max_size_update( } -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Initializes an ibuf bitmap page. */ void @@ -618,10 +615,7 @@ ibuf_bitmap_page_init( memset(page + IBUF_BITMAP, 0, byte_offset); /* The remaining area (up to the page trailer) is uninitialized. */ - -#ifndef UNIV_HOTBACKUP mlog_write_initial_log_record(page, MLOG_IBUF_BITMAP_INIT, mtr); -#endif /* !UNIV_HOTBACKUP */ } /*********************************************************************//** @@ -644,7 +638,7 @@ ibuf_parse_bitmap_init( return(ptr); } -#ifndef UNIV_HOTBACKUP + # ifdef UNIV_DEBUG /** Gets the desired bits for a given page from a bitmap page. @param[in] page bitmap page @@ -862,11 +856,17 @@ ibuf_set_free_bits_low( mtr_t* mtr) /*!< in/out: mtr */ { page_t* bitmap_page; + buf_frame_t* frame; ut_ad(mtr->is_named_space(block->page.id.space())); - if (!page_is_leaf(buf_block_get_frame(block))) { + if (!block) { + return; + } + + frame = buf_block_get_frame(block); + if (!frame || !page_is_leaf(frame)) { return; } @@ -1040,7 +1040,10 @@ ibuf_update_free_bits_zip( page_t* bitmap_page; ulint after; - ut_a(page_is_leaf(buf_block_get_frame(block))); + ut_a(block); + buf_frame_t* frame = buf_block_get_frame(block); + ut_a(frame); + ut_a(page_is_leaf(frame)); ut_a(block->page.size.is_compressed()); bitmap_page = ibuf_bitmap_get_map_page(block->page.id, @@ -1224,8 +1227,9 @@ ibuf_rec_get_page_no_func( const byte* field; ulint len; - ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX) - || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX)); + ut_ad(mtr_memo_contains_page_flagged(mtr, rec, + MTR_MEMO_PAGE_X_FIX + | MTR_MEMO_PAGE_S_FIX)); ut_ad(ibuf_inside(mtr)); ut_ad(rec_get_n_fields_old(rec) > 2); @@ -1262,8 +1266,8 @@ ibuf_rec_get_space_func( const byte* field; ulint len; - ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX) - || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX)); + ut_ad(mtr_memo_contains_page_flagged(mtr, rec, MTR_MEMO_PAGE_X_FIX + | MTR_MEMO_PAGE_S_FIX)); ut_ad(ibuf_inside(mtr)); ut_ad(rec_get_n_fields_old(rec) > 2); @@ -1312,8 +1316,8 @@ ibuf_rec_get_info_func( ulint info_len_local; ulint counter_local; - ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX) - || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX)); + ut_ad(mtr_memo_contains_page_flagged(mtr, rec, MTR_MEMO_PAGE_X_FIX + | MTR_MEMO_PAGE_S_FIX)); ut_ad(ibuf_inside(mtr)); fields = rec_get_n_fields_old(rec); ut_a(fields > IBUF_REC_FIELD_USER); @@ -1384,8 +1388,8 @@ ibuf_rec_get_op_type_func( { ulint len; - ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX) - || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX)); + ut_ad(mtr_memo_contains_page_flagged(mtr, rec, MTR_MEMO_PAGE_X_FIX + | MTR_MEMO_PAGE_S_FIX)); ut_ad(ibuf_inside(mtr)); ut_ad(rec_get_n_fields_old(rec) > 2); @@ -1583,8 +1587,8 @@ ibuf_build_entry_from_ibuf_rec_func( ulint comp; dict_index_t* index; - ut_ad(mtr_memo_contains_page(mtr, ibuf_rec, MTR_MEMO_PAGE_X_FIX) - || mtr_memo_contains_page(mtr, ibuf_rec, MTR_MEMO_PAGE_S_FIX)); + ut_ad(mtr_memo_contains_page_flagged(mtr, ibuf_rec, MTR_MEMO_PAGE_X_FIX + | MTR_MEMO_PAGE_S_FIX)); ut_ad(ibuf_inside(mtr)); data = rec_get_nth_field_old(ibuf_rec, IBUF_REC_FIELD_MARKER, &len); @@ -1705,8 +1709,8 @@ ibuf_rec_get_volume_func( ibuf_op_t op; ulint info_len; - ut_ad(mtr_memo_contains_page(mtr, ibuf_rec, MTR_MEMO_PAGE_X_FIX) - || mtr_memo_contains_page(mtr, ibuf_rec, MTR_MEMO_PAGE_S_FIX)); + ut_ad(mtr_memo_contains_page_flagged(mtr, ibuf_rec, MTR_MEMO_PAGE_X_FIX + | MTR_MEMO_PAGE_S_FIX)); ut_ad(ibuf_inside(mtr)); ut_ad(rec_get_n_fields_old(ibuf_rec) > 2); @@ -2288,8 +2292,8 @@ ibuf_get_merge_page_nos_func( ulint limit; ulint n_pages; - ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX) - || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX)); + ut_ad(mtr_memo_contains_page_flagged(mtr, rec, MTR_MEMO_PAGE_X_FIX + | MTR_MEMO_PAGE_S_FIX)); ut_ad(ibuf_inside(mtr)); *n_stored = 0; @@ -2745,8 +2749,6 @@ ibuf_merge_in_background( } #endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */ - - while (sum_pages < n_pages) { ulint n_bytes; @@ -2874,8 +2876,8 @@ ibuf_get_volume_buffered_count_func( const byte* types; ulint n_fields; - ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX) - || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX)); + ut_ad(mtr_memo_contains_page_flagged(mtr, rec, MTR_MEMO_PAGE_X_FIX + | MTR_MEMO_PAGE_S_FIX)); ut_ad(ibuf_inside(mtr)); n_fields = rec_get_n_fields_old(rec); @@ -3247,8 +3249,8 @@ ibuf_get_entry_counter_low_func( ulint len; ut_ad(ibuf_inside(mtr)); - ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX) - || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX)); + ut_ad(mtr_memo_contains_page_flagged(mtr, rec, MTR_MEMO_PAGE_X_FIX + | MTR_MEMO_PAGE_S_FIX)); ut_ad(rec_get_n_fields_old(rec) > 2); field = rec_get_nth_field_old(rec, IBUF_REC_FIELD_MARKER, &len); @@ -3727,9 +3729,10 @@ ibuf_insert( op, page_id.space(), page_id.page_no())); ut_ad(dtuple_check_typed(entry)); - ut_ad(page_id.space() != srv_tmp_space.space_id()); + ut_ad(page_id.space() != SRV_TMP_SPACE_ID); ut_a(!dict_index_is_clust(index)); + ut_ad(!dict_table_is_temporary(index->table)); no_counter = use <= IBUF_USE_INSERT; @@ -5122,5 +5125,3 @@ ibuf_set_bitmap_for_bulk_load( mtr_commit(&mtr); } - -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/api0api.h b/storage/innobase/include/api0api.h deleted file mode 100644 index ec02febee74..00000000000 --- a/storage/innobase/include/api0api.h +++ /dev/null @@ -1,1040 +0,0 @@ -/***************************************************************************** - -Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/**************************************************//** -@file include/api0api.h -InnoDB Native API - -2008-08-01 Created by Sunny Bains. -3/20/2011 Jimmy Yang extracted from Embedded InnoDB -*******************************************************/ - -#ifndef api0api_h -#define api0api_h - -#include "db0err.h" -#include <stdio.h> - -#if defined(__GNUC__) -#define UNIV_NO_IGNORE MY_ATTRIBUTE ((warn_unused_result)) -#else -#define UNIV_NO_IGNORE -#endif /* __GNUC__ */ - -/* See comment about ib_bool_t as to why the two macros are unsigned long. */ -/** The boolean value of "true" used internally within InnoDB */ -#define IB_TRUE 0x1UL -/** The boolean value of "false" used internally within InnoDB */ -#define IB_FALSE 0x0UL - -/* Basic types used by the InnoDB API. */ -/** All InnoDB error codes are represented by ib_err_t */ -typedef enum dberr_t ib_err_t; -/** Representation of a byte within InnoDB */ -typedef unsigned char ib_byte_t; -/** Representation of an unsigned long int within InnoDB */ -#ifdef _WIN64 -typedef unsigned __int64 ib_ulint_t; -#else -typedef unsigned long int ib_ulint_t; -#endif /* _WIN64 */ - -/* We assume C99 support except when using VisualStudio. */ -#if !defined(_MSC_VER) -#include <stdint.h> -#endif /* _MSC_VER */ - -/* Integer types used by the API. Microsft VS defines its own types -and we use the Microsoft types when building with Visual Studio. */ -#if defined(_MSC_VER) -/** A signed 8 bit integral type. */ -typedef __int8 ib_i8_t; -#else -/** A signed 8 bit integral type. */ -typedef int8_t ib_i8_t; -#endif - -#if defined(_MSC_VER) -/** An unsigned 8 bit integral type. */ -typedef unsigned __int8 ib_u8_t; -#else -/** An unsigned 8 bit integral type. */ -typedef uint8_t ib_u8_t; -#endif - -#if defined(_MSC_VER) -/** A signed 16 bit integral type. */ -typedef __int16 ib_i16_t; -#else -/** A signed 16 bit integral type. */ -typedef int16_t ib_i16_t; -#endif - -#if defined(_MSC_VER) -/** An unsigned 16 bit integral type. */ -typedef unsigned __int16 ib_u16_t; -#else -/** An unsigned 16 bit integral type. */ -typedef uint16_t ib_u16_t; -#endif - -#if defined(_MSC_VER) -/** A signed 32 bit integral type. */ -typedef __int32 ib_i32_t; -#else -/** A signed 32 bit integral type. */ -typedef int32_t ib_i32_t; -#endif - -#if defined(_MSC_VER) -/** An unsigned 32 bit integral type. */ -typedef unsigned __int32 ib_u32_t; -#else -/** An unsigned 32 bit integral type. */ -typedef uint32_t ib_u32_t; -#endif - -#if defined(_MSC_VER) -/** A signed 64 bit integral type. */ -typedef __int64 ib_i64_t; -#else -/** A signed 64 bit integral type. */ -typedef int64_t ib_i64_t; -#endif - -#if defined(_MSC_VER) -/** An unsigned 64 bit integral type. */ -typedef unsigned __int64 ib_u64_t; -#else -/** An unsigned 64 bit integral type. */ -typedef uint64_t ib_u64_t; -#endif - -typedef void* ib_opaque_t; -typedef ib_opaque_t ib_charset_t; -typedef ib_ulint_t ib_bool_t; -typedef ib_u64_t ib_id_u64_t; - -/** @enum ib_cfg_type_t Possible types for a configuration variable. */ -typedef enum { - IB_CFG_IBOOL, /*!< The configuration parameter is - of type ibool */ - - /* XXX Can we avoid having different types for ulint and ulong? - - On Win64 "unsigned long" is 32 bits - - ulong is always defined as "unsigned long" - - On Win64 ulint is defined as 64 bit integer - => On Win64 ulint != ulong. - If we typecast all ulong and ulint variables to the smaller type - ulong, then we will cut the range of the ulint variables. - This is not a problem for most ulint variables because their max - allowed values do not exceed 2^32-1 (e.g. log_groups is ulint - but its max allowed value is 10). BUT buffer_pool_size and - log_file_size allow up to 2^64-1. */ - - IB_CFG_ULINT, /*!< The configuration parameter is - of type ulint */ - - IB_CFG_ULONG, /*!< The configuration parameter is - of type ulong */ - - IB_CFG_TEXT, /*!< The configuration parameter is - of type char* */ - - IB_CFG_CB /*!< The configuration parameter is - a callback parameter */ -} ib_cfg_type_t; - -/** @enum ib_col_type_t column types that are supported. */ -typedef enum { - IB_VARCHAR = 1, /*!< Character varying length. The - column is not padded. */ - - IB_CHAR = 2, /*!< Fixed length character string. The - column is padded to the right. */ - - IB_BINARY = 3, /*!< Fixed length binary, similar to - IB_CHAR but the column is not padded - to the right. */ - - IB_VARBINARY = 4, /*!< Variable length binary */ - - IB_BLOB = 5, /*!< Binary large object, or - a TEXT type */ - - IB_INT = 6, /*!< Integer: can be any size - from 1 - 8 bytes. If the size is - 1, 2, 4 and 8 bytes then you can use - the typed read and write functions. For - other sizes you will need to use the - ib_col_get_value() function and do the - conversion yourself. */ - - IB_SYS = 8, /*!< System column, this column can - be one of DATA_TRX_ID, DATA_ROLL_PTR - or DATA_ROW_ID. */ - - IB_FLOAT = 9, /*!< C (float) floating point value. */ - - IB_DOUBLE = 10, /*!> C (double) floating point value. */ - - IB_DECIMAL = 11, /*!< Decimal stored as an ASCII - string */ - - IB_VARCHAR_ANYCHARSET = 12, /*!< Any charset, varying length */ - - IB_CHAR_ANYCHARSET = 13 /*!< Any charset, fixed length */ - -} ib_col_type_t; - -/** @enum ib_tbl_fmt_t InnoDB table format types */ -typedef enum { - IB_TBL_REDUNDANT, /*!< Redundant row format, the column - type and length is stored in the row.*/ - - IB_TBL_COMPACT, /*!< Compact row format, the column - type is not stored in the row. The - length is stored in the row but the - storage format uses a compact format - to store the length of the column data - and record data storage format also - uses less storage. */ - - IB_TBL_DYNAMIC, /*!< Compact row format. BLOB prefixes - are not stored in the clustered index */ - - IB_TBL_COMPRESSED /*!< Similar to dynamic format but - with pages compressed */ -} ib_tbl_fmt_t; - -/** @enum ib_col_attr_t InnoDB column attributes */ -typedef enum { - IB_COL_NONE = 0, /*!< No special attributes. */ - - IB_COL_NOT_NULL = 1, /*!< Column data can't be NULL. */ - - IB_COL_UNSIGNED = 2, /*!< Column is IB_INT and unsigned. */ - - IB_COL_NOT_USED = 4, /*!< Future use, reserved. */ - - IB_COL_CUSTOM1 = 8, /*!< Custom precision type, this is - a bit that is ignored by InnoDB and so - can be set and queried by users. */ - - IB_COL_CUSTOM2 = 16, /*!< Custom precision type, this is - a bit that is ignored by InnoDB and so - can be set and queried by users. */ - - IB_COL_CUSTOM3 = 32 /*!< Custom precision type, this is - a bit that is ignored by InnoDB and so - can be set and queried by users. */ -} ib_col_attr_t; - -/* Note: must match lock0types.h */ -/** @enum ib_lck_mode_t InnoDB lock modes. */ -typedef enum { - IB_LOCK_IS = 0, /*!< Intention shared, an intention - lock should be used to lock tables */ - - IB_LOCK_IX, /*!< Intention exclusive, an intention - lock should be used to lock tables */ - - IB_LOCK_S, /*!< Shared locks should be used to - lock rows */ - - IB_LOCK_X, /*!< Exclusive locks should be used to - lock rows*/ - - IB_LOCK_TABLE_X, /*!< exclusive table lock */ - - IB_LOCK_NONE, /*!< This is used internally to note - consistent read */ - - IB_LOCK_NUM = IB_LOCK_NONE /*!< number of lock modes */ -} ib_lck_mode_t; - -typedef enum { - IB_CLUSTERED = 1, /*!< clustered index */ - IB_UNIQUE = 2 /*!< unique index */ -} ib_index_type_t; - -/** @enum ib_srch_mode_t InnoDB cursor search modes for ib_cursor_moveto(). -Note: Values must match those found in page0cur.h */ -typedef enum { - IB_CUR_G = 1, /*!< If search key is not found then - position the cursor on the row that - is greater than the search key */ - - IB_CUR_GE = 2, /*!< If the search key not found then - position the cursor on the row that - is greater than or equal to the search - key */ - - IB_CUR_L = 3, /*!< If search key is not found then - position the cursor on the row that - is less than the search key */ - - IB_CUR_LE = 4 /*!< If search key is not found then - position the cursor on the row that - is less than or equal to the search - key */ -} ib_srch_mode_t; - -/** @enum ib_match_mode_t Various match modes used by ib_cursor_moveto() */ -typedef enum { - IB_CLOSEST_MATCH, /*!< Closest match possible */ - - IB_EXACT_MATCH, /*!< Search using a complete key - value */ - - IB_EXACT_PREFIX /*!< Search using a key prefix which - must match to rows: the prefix may - contain an incomplete field (the - last field in prefix may be just - a prefix of a fixed length column) */ -} ib_match_mode_t; - -/** @struct ib_col_meta_t InnoDB column meta data. */ -typedef struct { - ib_col_type_t type; /*!< Type of the column */ - - ib_col_attr_t attr; /*!< Column attributes */ - - ib_u32_t type_len; /*!< Length of type */ - - ib_u16_t client_type; /*!< 16 bits of data relevant only to - the client. InnoDB doesn't care */ - - ib_charset_t* charset; /*!< Column charset */ -} ib_col_meta_t; - -/* Note: Must be in sync with trx0trx.h */ -/** @enum ib_trx_level_t Transaction isolation levels */ -typedef enum { - IB_TRX_READ_UNCOMMITTED = 0, /*!< Dirty read: non-locking SELECTs are - performed so that we do not look at a - possible earlier version of a record; - thus they are not 'consistent' reads - under this isolation level; otherwise - like level 2 */ - - IB_TRX_READ_COMMITTED = 1, /*!< Somewhat Oracle-like isolation, - except that in range UPDATE and DELETE - we must block phantom rows with - next-key locks; SELECT ... FOR UPDATE - and ... LOCK IN SHARE MODE only lock - the index records, NOT the gaps before - them, and thus allow free inserting; - each consistent read reads its own - snapshot */ - - IB_TRX_REPEATABLE_READ = 2, /*!< All consistent reads in the same - trx read the same snapshot; full - next-key locking used in locking reads - to block insertions into gaps */ - - IB_TRX_SERIALIZABLE = 3 /*!< All plain SELECTs are converted to - LOCK IN SHARE MODE reads */ -} ib_trx_level_t; - -/** Generical InnoDB callback prototype. */ -typedef void (*ib_cb_t)(void); - -#define IB_CFG_BINLOG_ENABLED 0x1 -#define IB_CFG_MDL_ENABLED 0x2 -#define IB_CFG_DISABLE_ROWLOCK 0x4 - -/** The first argument to the InnoDB message logging function. By default -it's set to stderr. You should treat ib_msg_stream_t as a void*, since -it will probably change in the future. */ -typedef FILE* ib_msg_stream_t; - -/** All log messages are written to this function.It should have the same -behavior as fprintf(3). */ -typedef int (*ib_msg_log_t)(ib_msg_stream_t, const char*, ...); - -/* Note: This is to make it easy for API users to have type -checking for arguments to our functions. Making it ib_opaque_t -by itself will result in pointer decay resulting in subverting -of the compiler's type checking. */ - -/** InnoDB tuple handle. This handle can refer to either a cluster index -tuple or a secondary index tuple. There are two types of tuples for each -type of index, making a total of four types of tuple handles. There -is a tuple for reading the entire row contents and another for searching -on the index key. */ -typedef struct ib_tuple_t* ib_tpl_t; - -/** InnoDB transaction handle, all database operations need to be covered -by transactions. This handle represents a transaction. The handle can be -created with ib_trx_begin(), you commit your changes with ib_trx_commit() -and undo your changes using ib_trx_rollback(). If the InnoDB deadlock -monitor rolls back the transaction then you need to free the transaction -using the function ib_trx_release(). You can query the state of an InnoDB -transaction by calling ib_trx_state(). */ -typedef struct trx_t* ib_trx_t; - -/** InnoDB cursor handle */ -typedef struct ib_cursor_t* ib_crsr_t; - -/*************************************************************//** -This function is used to compare two data fields for which the data type -is such that we must use the client code to compare them. - -@param col_meta column meta data -@param p1 key -@oaram p1_len key length -@param p2 second key -@param p2_len second key length -@return 1, 0, -1, if a is greater, equal, less than b, respectively */ - -typedef int (*ib_client_cmp_t)( - const ib_col_meta_t* col_meta, - const ib_byte_t* p1, - ib_ulint_t p1_len, - const ib_byte_t* p2, - ib_ulint_t p2_len); - -/* This should be the same as univ.i */ -/** Represents SQL_NULL length */ -#define IB_SQL_NULL 0xFFFFFFFF - -/*****************************************************************//** -Start a transaction that's been rolled back. This special function -exists for the case when InnoDB's deadlock detector has rolledack -a transaction. While the transaction has been rolled back the handle -is still valid and can be reused by calling this function. If you -don't want to reuse the transaction handle then you can free the handle -by calling ib_trx_release(). -@return innobase txn handle */ -ib_err_t -ib_trx_start( -/*=========*/ - ib_trx_t ib_trx, /*!< in: transaction to restart */ - ib_trx_level_t ib_trx_level, /*!< in: trx isolation level */ - ib_bool_t read_write, /*!< in: true if read write - transaction */ - ib_bool_t auto_commit, /*!< in: auto commit after each - single DML */ - void* thd); /*!< in: THD */ - -/*****************************************************************//** -Begin a transaction. This will allocate a new transaction handle and -put the transaction in the active state. -@return innobase txn handle */ -ib_trx_t -ib_trx_begin( -/*=========*/ - ib_trx_level_t ib_trx_level, /*!< in: trx isolation level */ - ib_bool_t read_write, /*!< in: true if read write - transaction */ - ib_bool_t auto_commit); /*!< in: auto commit after each - single DML */ - -/*****************************************************************//** -Check if the transaction is read_only */ -ib_u32_t -ib_trx_read_only( -/*=============*/ - ib_trx_t ib_trx); /*!< in: trx handle */ - -/*****************************************************************//** -Release the resources of the transaction. If the transaction was -selected as a victim by InnoDB and rolled back then use this function -to free the transaction handle. -@return DB_SUCCESS or err code */ -ib_err_t -ib_trx_release( -/*===========*/ - ib_trx_t ib_trx); /*!< in: trx handle */ - -/*****************************************************************//** -Commit a transaction. This function will release the schema latches too. -It will also free the transaction handle. -@return DB_SUCCESS or err code */ -ib_err_t -ib_trx_commit( -/*==========*/ - ib_trx_t ib_trx); /*!< in: trx handle */ - -/*****************************************************************//** -Rollback a transaction. This function will release the schema latches too. -It will also free the transaction handle. -@return DB_SUCCESS or err code */ -ib_err_t -ib_trx_rollback( -/*============*/ - ib_trx_t ib_trx); /*!< in: trx handle */ - -/*****************************************************************//** -Open an InnoDB table and return a cursor handle to it. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_open_table_using_id( -/*==========================*/ - ib_id_u64_t table_id, /*!< in: table id of table to open */ - ib_trx_t ib_trx, /*!< in: Current transaction handle - can be NULL */ - ib_crsr_t* ib_crsr); /*!< out,own: InnoDB cursor */ - -/*****************************************************************//** -Open an InnoDB secondary index cursor and return a cursor handle to it. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_open_index_using_name( -/*============================*/ - ib_crsr_t ib_open_crsr, /*!< in: open/active cursor */ - const char* index_name, /*!< in: secondary index name */ - ib_crsr_t* ib_crsr, /*!< out,own: InnoDB index cursor */ - int* idx_type, /*!< out: index is cluster index */ - ib_id_u64_t* idx_id); /*!< out: index id */ - -/*****************************************************************//** -Open an InnoDB table by name and return a cursor handle to it. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_open_table( -/*=================*/ - const char* name, /*!< in: table name */ - ib_trx_t ib_trx, /*!< in: Current transaction handle - can be NULL */ - ib_crsr_t* ib_crsr); /*!< out,own: InnoDB cursor */ - -/*****************************************************************//** -Reset the cursor. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_reset( -/*============*/ - ib_crsr_t ib_crsr); /*!< in/out: InnoDB cursor */ - -/*****************************************************************//** -Close an InnoDB table and free the cursor. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_close( -/*============*/ - ib_crsr_t ib_crsr); /*!< in/out: InnoDB cursor */ - -/*****************************************************************//** -Close the table, decrement n_ref_count count. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_close_table( -/*==================*/ - ib_crsr_t ib_crsr); /*!< in/out: InnoDB cursor */ - -/*****************************************************************//** -update the cursor with new transactions and also reset the cursor -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_new_trx( -/*==============*/ - ib_crsr_t ib_crsr, /*!< in/out: InnoDB cursor */ - ib_trx_t ib_trx); /*!< in: transaction */ - -/*****************************************************************//** -Commit the transaction in a cursor -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_commit_trx( -/*=================*/ - ib_crsr_t ib_crsr, /*!< in/out: InnoDB cursor */ - ib_trx_t ib_trx); /*!< in: transaction */ - -/*****************************************************************//** -Insert a row to a table. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_insert_row( -/*=================*/ - ib_crsr_t ib_crsr, /*!< in/out: InnoDB cursor instance */ - const ib_tpl_t ib_tpl); /*!< in: tuple to insert */ - -/*****************************************************************//** -Update a row in a table. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_update_row( -/*=================*/ - ib_crsr_t ib_crsr, /*!< in: InnoDB cursor instance */ - const ib_tpl_t ib_old_tpl, /*!< in: Old tuple in table */ - const ib_tpl_t ib_new_tpl); /*!< in: New tuple to update */ - -/*****************************************************************//** -Delete a row in a table. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_delete_row( -/*=================*/ - ib_crsr_t ib_crsr); /*!< in: cursor instance */ - -/*****************************************************************//** -Read current row. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_read_row( -/*===============*/ - ib_crsr_t ib_crsr, /*!< in: InnoDB cursor instance */ - ib_tpl_t ib_tpl, /*!< out: read cols into this tuple */ - void** row_buf, /*!< in/out: row buffer */ - ib_ulint_t* row_len); /*!< in/out: row buffer len */ - -/*****************************************************************//** -Move cursor to the first record in the table. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_first( -/*============*/ - ib_crsr_t ib_crsr); /*!< in: InnoDB cursor instance */ - -/*****************************************************************//** -Move cursor to the next record in the table. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_next( -/*===========*/ - ib_crsr_t ib_crsr); /*!< in: InnoDB cursor instance */ - -/*****************************************************************//** -Search for key. -@return DB_SUCCESS or err code */ -ib_err_t -ib_cursor_moveto( -/*=============*/ - ib_crsr_t ib_crsr, /*!< in: InnoDB cursor instance */ - ib_tpl_t ib_tpl, /*!< in: Key to search for */ - ib_srch_mode_t ib_srch_mode); /*!< in: search mode */ - -/*****************************************************************//** -Set the match mode for ib_cursor_move(). */ -void -ib_cursor_set_match_mode( -/*=====================*/ - ib_crsr_t ib_crsr, /*!< in: Cursor instance */ - ib_match_mode_t match_mode); /*!< in: ib_cursor_moveto match mode */ - -/*****************************************************************//** -Set a column of the tuple. Make a copy using the tuple's heap. -@return DB_SUCCESS or error code */ -ib_err_t -ib_col_set_value( -/*=============*/ - ib_tpl_t ib_tpl, /*!< in: tuple instance */ - ib_ulint_t col_no, /*!< in: column index in tuple */ - const void* src, /*!< in: data value */ - ib_ulint_t len, /*!< in: data value len */ - ib_bool_t need_cpy); /*!< in: if need memcpy */ - - -/*****************************************************************//** -Get the size of the data available in the column the tuple. -@return bytes avail or IB_SQL_NULL */ -ib_ulint_t -ib_col_get_len( -/*===========*/ - ib_tpl_t ib_tpl, /*!< in: tuple instance */ - ib_ulint_t i); /*!< in: column index in tuple */ - -/*****************************************************************//** -Copy a column value from the tuple. -@return bytes copied or IB_SQL_NULL */ -ib_ulint_t -ib_col_copy_value( -/*==============*/ - ib_tpl_t ib_tpl, /*!< in: tuple instance */ - ib_ulint_t i, /*!< in: column index in tuple */ - void* dst, /*!< out: copied data value */ - ib_ulint_t len); /*!< in: max data value len to copy */ - -/*************************************************************//** -Read a signed int 8 bit column from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_i8( -/*=============*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t i, /*!< in: column number */ - ib_i8_t* ival); /*!< out: integer value */ - -/*************************************************************//** -Read an unsigned int 8 bit column from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_u8( -/*=============*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t i, /*!< in: column number */ - ib_u8_t* ival); /*!< out: integer value */ - -/*************************************************************//** -Read a signed int 16 bit column from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_i16( -/*==============*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t i, /*!< in: column number */ - ib_i16_t* ival); /*!< out: integer value */ - -/*************************************************************//** -Read an unsigned int 16 bit column from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_u16( -/*==============*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t i, /*!< in: column number */ - ib_u16_t* ival); /*!< out: integer value */ - -/*************************************************************//** -Read a signed int 32 bit column from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_i32( -/*==============*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t i, /*!< in: column number */ - ib_i32_t* ival); /*!< out: integer value */ - -/*************************************************************//** -Read an unsigned int 32 bit column from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_u32( -/*==============*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t i, /*!< in: column number */ - ib_u32_t* ival); /*!< out: integer value */ - -/*************************************************************//** -Read a signed int 64 bit column from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_i64( -/*==============*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t i, /*!< in: column number */ - ib_i64_t* ival); /*!< out: integer value */ - -/*************************************************************//** -Read an unsigned int 64 bit column from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_u64( -/*==============*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t i, /*!< in: column number */ - ib_u64_t* ival); /*!< out: integer value */ - -/*****************************************************************//** -Get a column value pointer from the tuple. -@return NULL or pointer to buffer */ -const void* -ib_col_get_value( -/*=============*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t i); /*!< in: column number */ - -/*****************************************************************//** -Get a column type, length and attributes from the tuple. -@return len of column data */ -ib_ulint_t -ib_col_get_meta( -/*============*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t i, /*!< in: column number */ - ib_col_meta_t* ib_col_meta); /*!< out: column meta data */ - -/*****************************************************************//** -"Clear" or reset an InnoDB tuple. We free the heap and recreate the tuple. -@return new tuple, or NULL */ -ib_tpl_t -ib_tuple_clear( -/*============*/ - ib_tpl_t ib_tpl); /*!< in: InnoDB tuple */ - -/*****************************************************************//** -Create a new cluster key search tuple and copy the contents of the -secondary index key tuple columns that refer to the cluster index record -to the cluster key. It does a deep copy of the column data. -@return DB_SUCCESS or error code */ -ib_err_t -ib_tuple_get_cluster_key( -/*=====================*/ - ib_crsr_t ib_crsr, /*!< in: secondary index cursor */ - ib_tpl_t* ib_dst_tpl, /*!< out,own: destination tuple */ - const ib_tpl_t ib_src_tpl); /*!< in: source tuple */ - -/*****************************************************************//** -Create an InnoDB tuple used for index/table search. -@return tuple for current index */ -ib_tpl_t -ib_sec_search_tuple_create( -/*=======================*/ - ib_crsr_t ib_crsr); /*!< in: Cursor instance */ - -/*****************************************************************//** -Create an InnoDB tuple used for index/table search. -@return tuple for current index */ -ib_tpl_t -ib_sec_read_tuple_create( -/*=====================*/ - ib_crsr_t ib_crsr); /*!< in: Cursor instance */ - -/*****************************************************************//** -Create an InnoDB tuple used for table key operations. -@return tuple for current table */ -ib_tpl_t -ib_clust_search_tuple_create( -/*=========================*/ - ib_crsr_t ib_crsr); /*!< in: Cursor instance */ - -/*****************************************************************//** -Create an InnoDB tuple for table row operations. -@return tuple for current table */ -ib_tpl_t -ib_clust_read_tuple_create( -/*=======================*/ - ib_crsr_t ib_crsr); /*!< in: Cursor instance */ - -/*****************************************************************//** -Return the number of user columns in the tuple definition. -@return number of user columns */ -ib_ulint_t -ib_tuple_get_n_user_cols( -/*=====================*/ - const ib_tpl_t ib_tpl); /*!< in: Tuple for current table */ - -/*****************************************************************//** -Return the number of columns in the tuple definition. -@return number of columns */ -ib_ulint_t -ib_tuple_get_n_cols( -/*================*/ - const ib_tpl_t ib_tpl); /*!< in: Tuple for current table */ - -/*****************************************************************//** -Destroy an InnoDB tuple. */ -void -ib_tuple_delete( -/*============*/ - ib_tpl_t ib_tpl); /*!< in,own: Tuple instance to delete */ - -/*****************************************************************//** -Truncate a table. The cursor handle will be closed and set to NULL -on success. -@return DB_SUCCESS or error code */ -ib_err_t -ib_cursor_truncate( -/*===============*/ - ib_crsr_t* ib_crsr, /*!< in/out: cursor for table - to truncate */ - ib_id_u64_t* table_id); /*!< out: new table id */ - -/*****************************************************************//** -Get a table id. -@return DB_SUCCESS if found */ -ib_err_t -ib_table_get_id( -/*============*/ - const char* table_name, /*!< in: table to find */ - ib_id_u64_t* table_id); /*!< out: table id if found */ - -/*****************************************************************//** -Check if cursor is positioned. -@return IB_TRUE if positioned */ -ib_bool_t -ib_cursor_is_positioned( -/*====================*/ - const ib_crsr_t ib_crsr); /*!< in: InnoDB cursor instance */ - -/*****************************************************************//** -Checks if the data dictionary is latched in exclusive mode by a -user transaction. -@return TRUE if exclusive latch */ -ib_bool_t -ib_schema_lock_is_exclusive( -/*========================*/ - const ib_trx_t ib_trx); /*!< in: transaction */ - -/*****************************************************************//** -Lock an InnoDB cursor/table. -@return DB_SUCCESS or error code */ -ib_err_t -ib_cursor_lock( -/*===========*/ - ib_crsr_t ib_crsr, /*!< in/out: InnoDB cursor */ - ib_lck_mode_t ib_lck_mode); /*!< in: InnoDB lock mode */ - -/*****************************************************************//** -Set the Lock an InnoDB table using the table id. -@return DB_SUCCESS or error code */ -ib_err_t -ib_table_lock( -/*===========*/ - ib_trx_t ib_trx, /*!< in/out: transaction */ - ib_id_u64_t table_id, /*!< in: table id */ - ib_lck_mode_t ib_lck_mode); /*!< in: InnoDB lock mode */ - -/*****************************************************************//** -Set the Lock mode of the cursor. -@return DB_SUCCESS or error code */ -ib_err_t -ib_cursor_set_lock_mode( -/*====================*/ - ib_crsr_t ib_crsr, /*!< in/out: InnoDB cursor */ - ib_lck_mode_t ib_lck_mode); /*!< in: InnoDB lock mode */ - -/*****************************************************************//** -Set need to access clustered index record flag. */ -void -ib_cursor_set_cluster_access( -/*=========================*/ - ib_crsr_t ib_crsr); /*!< in/out: InnoDB cursor */ - -/*****************************************************************//** -Inform the cursor that it's the start of an SQL statement. */ -void -ib_cursor_stmt_begin( -/*=================*/ - ib_crsr_t ib_crsr); /*!< in: cursor */ - -/*****************************************************************//** -Write a double value to a column. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_write_double( -/*==================*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - int col_no, /*!< in: column number */ - double val); /*!< in: value to write */ - -/*************************************************************//** -Read a double column value from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_double( -/*=================*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t col_no, /*!< in: column number */ - double* dval); /*!< out: double value */ - -/*****************************************************************//** -Write a float value to a column. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_write_float( -/*=================*/ - ib_tpl_t ib_tpl, /*!< in/out: tuple to write to */ - int col_no, /*!< in: column number */ - float val); /*!< in: value to write */ - -/*************************************************************//** -Read a float value from an InnoDB tuple. -@return DB_SUCCESS or error */ -ib_err_t -ib_tuple_read_float( -/*================*/ - ib_tpl_t ib_tpl, /*!< in: InnoDB tuple */ - ib_ulint_t col_no, /*!< in: column number */ - float* fval); /*!< out: float value */ - -/*****************************************************************//** -Get a column type, length and attributes from the tuple. -@return len of column data */ -const char* -ib_col_get_name( -/*============*/ - ib_crsr_t ib_crsr, /*!< in: InnoDB cursor instance */ - ib_ulint_t i); /*!< in: column index in tuple */ - -/*****************************************************************//** -Get an index field name from the cursor. -@return name of the field */ -const char* -ib_get_idx_field_name( -/*==================*/ - ib_crsr_t ib_crsr, /*!< in: InnoDB cursor instance */ - ib_ulint_t i); /*!< in: column index in tuple */ - -/*****************************************************************//** -Truncate a table. -@return DB_SUCCESS or error code */ -ib_err_t -ib_table_truncate( -/*==============*/ - const char* table_name, /*!< in: table name */ - ib_id_u64_t* table_id); /*!< out: new table id */ - -/*****************************************************************//** -Get generic configure status -@return configure status*/ -int -ib_cfg_get_cfg(); -/*============*/ - -/*****************************************************************//** -Increase/decrease the memcached sync count of table to sync memcached -DML with SQL DDLs. -@return DB_SUCCESS or error number */ -ib_err_t -ib_cursor_set_memcached_sync( -/*=========================*/ - ib_crsr_t ib_crsr, /*!< in: cursor */ - ib_bool_t flag); /*!< in: true for increasing */ - -/*****************************************************************//** -Return isolation configuration set by "innodb_api_trx_level" -@return trx isolation level*/ -ib_trx_level_t -ib_cfg_trx_level(); -/*==============*/ - -/*****************************************************************//** -Return configure value for background commit interval (in seconds) -@return background commit interval (in seconds) */ -ib_ulint_t -ib_cfg_bk_commit_interval(); -/*=======================*/ - -/*****************************************************************//** -Get a trx start time. -@return trx start_time */ -ib_u64_t -ib_trx_get_start_time( -/*==================*/ - ib_trx_t ib_trx); /*!< in: transaction */ - -/*****************************************************************//** -Wrapper of ut_strerr() which converts an InnoDB error number to a -human readable text message. -@return string, describing the error */ -const char* -ib_ut_strerr( -/*=========*/ - ib_err_t num); /*!< in: error number */ - -/** Check the table whether it contains virtual columns. -@param[in] crsr InnoDB Cursor -@return true if table contains virtual column else false. */ -ib_bool_t -ib_is_virtual_table( - ib_crsr_t crsr); - -#endif /* api0api_h */ diff --git a/storage/innobase/include/api0misc.h b/storage/innobase/include/api0misc.h deleted file mode 100644 index 84ac3d622a9..00000000000 --- a/storage/innobase/include/api0misc.h +++ /dev/null @@ -1,76 +0,0 @@ -/***************************************************************************** - -Copyright (c) 2008, 2014, Oracle and/or its affiliates. All Rights Reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/**************************************************//** -@file include/api0misc.h -InnoDB Native API - -3/20/2011 Jimmy Yang extracted from Embedded InnoDB -2008 Created by Sunny Bains -*******************************************************/ - -#ifndef api0misc_h -#define api0misc_h - -#include "univ.i" -#include "os0file.h" -#include "que0que.h" -#include "trx0trx.h" - -/** Whether binlog is enabled for applications using InnoDB APIs */ -extern my_bool ib_binlog_enabled; - -/** Whether MySQL MDL is enabled for applications using InnoDB APIs */ -extern my_bool ib_mdl_enabled; - -/** Whether InnoDB row lock is disabled for applications using InnoDB APIs */ -extern my_bool ib_disable_row_lock; - -/** configure value for transaction isolation level */ -extern ulong ib_trx_level_setting; - -/** configure value for background commit interval (in seconds) */ -extern ulong ib_bk_commit_interval; - -/******************************************************************** -Handles user errors and lock waits detected by the database engine. -@return TRUE if it was a lock wait and we should continue running -the query thread */ -ibool -ib_handle_errors( -/*=============*/ - dberr_t* new_err, /*!< out: possible new error - encountered in lock wait, or if - no new error, the value of - trx->error_state at the entry of this - function */ - trx_t* trx, /*!< in: transaction */ - que_thr_t* thr, /*!< in: query thread */ - trx_savept_t* savept); /*!< in: savepoint or NULL */ - -/************************************************************************* -Sets a lock on a table. -@return error code or DB_SUCCESS */ -dberr_t -ib_trx_lock_table_with_retry( -/*=========================*/ - trx_t* trx, /*!< in/out: transaction */ - dict_table_t* table, /*!< in: table to lock */ - enum lock_mode mode); /*!< in: lock mode */ - -#endif /* api0misc_h */ diff --git a/storage/innobase/include/btr0btr.h b/storage/innobase/include/btr0btr.h index c177f23824f..419c5db657d 100644 --- a/storage/innobase/include/btr0btr.h +++ b/storage/innobase/include/btr0btr.h @@ -2,7 +2,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2014, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2014, 2016, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -37,7 +37,6 @@ Created 6/2/1994 Heikki Tuuri #include "btr0types.h" #include "gis0type.h" -#ifndef UNIV_HOTBACKUP /** Maximum record size which can be stored on a page, without using the special big record storage structure */ #define BTR_PAGE_MAX_REC_SIZE (UNIV_PAGE_SIZE / 2 - 200) @@ -140,7 +139,6 @@ record is in spatial index */ ((latch_mode) & ~(BTR_LATCH_FOR_INSERT \ | BTR_LATCH_FOR_DELETE \ | BTR_MODIFY_EXTERNAL)) -#endif /* UNIV_HOTBACKUP */ /**************************************************************//** Report that an index page is corrupted. */ @@ -161,7 +159,6 @@ btr_corruption_report( ut_error; \ } -#ifndef UNIV_HOTBACKUP /**************************************************************//** Gets the root node of a tree and sx-latches it for segment access. @return root page, sx-latched */ @@ -242,10 +239,16 @@ btr_block_get_func( @param index index tree, may be NULL if not the insert buffer tree @param mtr mini-transaction handle @return the uncompressed page frame */ -# define btr_page_get(page_id, page_size, mode, index, mtr) \ - buf_block_get_frame(btr_block_get(page_id, page_size, \ - mode, index, mtr)) -#endif /* !UNIV_HOTBACKUP */ +UNIV_INLINE +page_t* +btr_page_get( +/*=========*/ + const page_id_t& page_id, + const page_size_t& page_size, + ulint mode, + dict_index_t* index, + mtr_t* mtr) + MY_ATTRIBUTE((warn_unused_result)); /**************************************************************//** Gets the index id field of a page. @return index id */ @@ -255,7 +258,6 @@ btr_page_get_index_id( /*==================*/ const page_t* page) /*!< in: index page */ MY_ATTRIBUTE((warn_unused_result)); -#ifndef UNIV_HOTBACKUP /********************************************************//** Gets the node level field in an index page. @return level, leaf level == 0 */ @@ -353,6 +355,34 @@ btr_free( const page_id_t& page_id, const page_size_t& page_size); +/** Read the last used AUTO_INCREMENT value from PAGE_ROOT_AUTO_INC. +@param[in,out] index clustered index +@return the last used AUTO_INCREMENT value +@retval 0 on error or if no AUTO_INCREMENT value was used yet */ +ib_uint64_t +btr_read_autoinc(dict_index_t* index) + MY_ATTRIBUTE((nonnull, warn_unused_result)); + +/** Read the last used AUTO_INCREMENT value from PAGE_ROOT_AUTO_INC, +or fall back to MAX(auto_increment_column). +@param[in] table table containing an AUTO_INCREMENT column +@param[in] col_no index of the AUTO_INCREMENT column +@return the AUTO_INCREMENT value +@retval 0 on error or if no AUTO_INCREMENT value was used yet */ +ib_uint64_t +btr_read_autoinc_with_fallback(const dict_table_t* table, unsigned col_no) + MY_ATTRIBUTE((nonnull, warn_unused_result)); + +/** Write the next available AUTO_INCREMENT value to PAGE_ROOT_AUTO_INC. +@param[in,out] index clustered index +@param[in] autoinc the AUTO_INCREMENT value +@param[in] reset whether to reset the AUTO_INCREMENT + to a possibly smaller value than currently + exists in the page */ +void +btr_write_autoinc(dict_index_t* index, ib_uint64_t autoinc, bool reset = false) + MY_ATTRIBUTE((nonnull)); + /*************************************************************//** Makes tree one level higher by splitting the root, and inserts the tuple. It is assumed that mtr contains an x-latch on the tree. @@ -479,9 +509,8 @@ btr_insert_on_non_leaf_level_func( const char* file, /*!< in: file name */ ulint line, /*!< in: line where called */ mtr_t* mtr); /*!< in: mtr */ -# define btr_insert_on_non_leaf_level(f,i,l,t,m) \ +#define btr_insert_on_non_leaf_level(f,i,l,t,m) \ btr_insert_on_non_leaf_level_func(f,i,l,t,__FILE__,__LINE__,m) -#endif /* !UNIV_HOTBACKUP */ /****************************************************************//** Sets a record as the predefined minimum record. */ void @@ -490,7 +519,6 @@ btr_set_min_rec_mark( rec_t* rec, /*!< in/out: record */ mtr_t* mtr) /*!< in: mtr */ MY_ATTRIBUTE((nonnull)); -#ifndef UNIV_HOTBACKUP /*************************************************************//** Deletes on the upper level the node pointer to a page. */ void @@ -543,7 +571,6 @@ btr_discard_page( btr_cur_t* cursor, /*!< in: cursor on the page to discard: not on the root page */ mtr_t* mtr); /*!< in: mtr */ -#endif /* !UNIV_HOTBACKUP */ /****************************************************************//** Parses the redo log record for setting an index record as the predefined minimum record. @@ -570,7 +597,6 @@ btr_parse_page_reorganize( buf_block_t* block, /*!< in: page to be reorganized, or NULL */ mtr_t* mtr) /*!< in: mtr or NULL */ MY_ATTRIBUTE((warn_unused_result)); -#ifndef UNIV_HOTBACKUP /**************************************************************//** Gets the number of pages in a B-tree. @return number of pages, or ULINT_UNDEFINED if the index is unavailable */ @@ -771,7 +797,6 @@ btr_lift_page_up( #define BTR_N_LEAF_PAGES 1 #define BTR_TOTAL_SIZE 2 -#endif /* !UNIV_HOTBACKUP */ #ifndef UNIV_NONINL #include "btr0btr.ic" diff --git a/storage/innobase/include/btr0btr.ic b/storage/innobase/include/btr0btr.ic index 58a0c6755b1..810ac8f8b67 100644 --- a/storage/innobase/include/btr0btr.ic +++ b/storage/innobase/include/btr0btr.ic @@ -25,7 +25,6 @@ Created 6/2/1994 Heikki Tuuri *******************************************************/ #include "mach0data.h" -#ifndef UNIV_HOTBACKUP #include "mtr0mtr.h" #include "mtr0log.h" #include "page0zip.h" @@ -63,7 +62,9 @@ btr_block_get_func( page_id, page_size, mode, NULL, BUF_GET, file, line, mtr, &err); if (err == DB_DECRYPTION_FAILED) { - index->table->is_encrypted = true; + if (index && index->table) { + index->table->is_encrypted = true; + } } if (block) { @@ -99,7 +100,36 @@ btr_page_set_index_id( mlog_write_ull(page + (PAGE_HEADER + PAGE_INDEX_ID), id, mtr); } } -#endif /* !UNIV_HOTBACKUP */ + +/** Gets a buffer page and declares its latching order level. +@param space tablespace identifier +@param zip_size compressed page size in bytes or 0 for uncompressed pages +@param page_no page number +@param mode latch mode +@param idx index tree, may be NULL if not the insert buffer tree +@param mtr mini-transaction handle +@return the uncompressed page frame */ +UNIV_INLINE +page_t* +btr_page_get( +/*=========*/ + const page_id_t& page_id, + const page_size_t& page_size, + ulint mode, + dict_index_t* index, + mtr_t* mtr) +{ + buf_block_t* block=NULL; + buf_frame_t* frame=NULL; + + block = btr_block_get(page_id, page_size, mode, index, mtr); + + if (block) { + frame = buf_block_get_frame(block); + } + + return ((page_t*)frame); +} /**************************************************************//** Gets the index id field of a page. @@ -113,7 +143,6 @@ btr_page_get_index_id( return(mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID)); } -#ifndef UNIV_HOTBACKUP /********************************************************//** Gets the node level field in an index page. @return level, leaf level == 0 */ @@ -306,4 +335,3 @@ btr_leaf_page_release( mtr->memo_release(block, mode); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index f582f04733c..a1e37ec46d0 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -60,7 +60,6 @@ struct btr_latch_leaves_t { ulint savepoints[3]; }; -#ifndef UNIV_HOTBACKUP #include "que0types.h" #include "row0types.h" #include "ha0ha.h" @@ -190,38 +189,10 @@ btr_cur_search_to_nth_level( RW_S_LATCH, or 0 */ const char* file, /*!< in: file name */ ulint line, /*!< in: line where called */ - mtr_t* mtr); /*!< in: mtr */ - -/** Searches an index tree and positions a tree cursor on a given level. -This function will avoid placing latches the travesal path and so -should be used only for cases where-in latching is not needed. - -@param[in] index index -@param[in] level the tree level of search -@param[in] tuple data tuple; Note: n_fields_cmp in compared - to the node ptr page node field -@param[in] mode PAGE_CUR_L, .... - Insert should always be made using PAGE_CUR_LE - to search the position. -@param[in,out] cursor tree cursor; points to record of interest. -@param[in] file file name -@param[in[ line line where called from -@param[in,out] mtr mtr -@param[in] mark_dirty - if true then mark the block as dirty -@return DB_SUCCESS or error code */ -dberr_t -btr_cur_search_to_nth_level_with_no_latch( - dict_index_t* index, - ulint level, - const dtuple_t* tuple, - page_cur_mode_t mode, - btr_cur_t* cursor, - const char* file, - ulint line, - mtr_t* mtr, - bool mark_dirty = true) - __attribute__((warn_unused_result)); + mtr_t* mtr, /*!< in/out: mini-transaction */ + ib_uint64_t autoinc = 0); + /*!< in: PAGE_ROOT_AUTO_INC to be written + (0 if none) */ /*****************************************************************//** Opens a cursor at either end of an index. @@ -244,35 +215,6 @@ btr_cur_open_at_index_side_func( #define btr_cur_open_at_index_side(f,i,l,c,lv,m) \ btr_cur_open_at_index_side_func(f,i,l,c,lv,__FILE__,__LINE__,m) -/** Opens a cursor at either end of an index. -Avoid taking latches on buffer, just pin (by incrementing fix_count) -to keep them in buffer pool. This mode is used by intrinsic table -as they are not shared and so there is no need of latching. -@param[in] from_left true if open to low end, false if open - to high end. -@param[in] index index -@param[in] latch_mode latch mode -@param[in,out] cursor cursor -@param[in] file file name -@param[in] line line where called -@param[in,out] mtr mini transaction -@return DB_SUCCESS or error code -*/ -dberr_t -btr_cur_open_at_index_side_with_no_latch_func( - bool from_left, - dict_index_t* index, - btr_cur_t* cursor, - ulint level, - const char* file, - ulint line, - mtr_t* mtr) - __attribute__((warn_unused_result)); - -#define btr_cur_open_at_index_side_with_no_latch(f,i,c,lv,m) \ - btr_cur_open_at_index_side_with_no_latch_func( \ - f,i,c,lv,__FILE__,__LINE__,m) - /**********************************************************************//** Positions a cursor at a randomly chosen position within a B-tree. @return true if the index is available and we have put the cursor, false @@ -587,7 +529,6 @@ btr_cur_pessimistic_delete( bool rollback,/*!< in: performing rollback? */ mtr_t* mtr) /*!< in: mtr */ MY_ATTRIBUTE((nonnull)); -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses a redo log record of updating a record in-place. @return end of log record or NULL */ @@ -622,7 +563,6 @@ btr_cur_parse_del_mark_set_sec_rec( byte* end_ptr,/*!< in: buffer end */ page_t* page, /*!< in/out: page or NULL */ page_zip_des_t* page_zip);/*!< in/out: compressed page, or NULL */ -#ifndef UNIV_HOTBACKUP /** Estimates the number of rows in a given index range. @param[in] index index @@ -998,7 +938,6 @@ btr_rec_set_deleted_flag( page_zip_des_t* page_zip,/*!< in/out: compressed page (or NULL) */ ulint flag); /*!< in: nonzero if delete marked */ - /** If pessimistic delete fails because of lack of file space, there is still a good change of success a little later. Try this many times. */ @@ -1052,7 +991,6 @@ extern ulint btr_cur_n_non_sea_old; srv_refresh_innodb_monitor_stats(). Referenced by srv_printf_innodb_monitor(). */ extern ulint btr_cur_n_sea_old; -#endif /* !UNIV_HOTBACKUP */ #ifdef UNIV_DEBUG /* Flag to limit optimistic insert records */ diff --git a/storage/innobase/include/btr0cur.ic b/storage/innobase/include/btr0cur.ic index 45c0d59a8aa..b1e59651a1d 100644 --- a/storage/innobase/include/btr0cur.ic +++ b/storage/innobase/include/btr0cur.ic @@ -23,7 +23,6 @@ The index tree cursor Created 10/16/1994 Heikki Tuuri *******************************************************/ -#ifndef UNIV_HOTBACKUP #include "btr0btr.h" #ifdef UNIV_DEBUG @@ -231,5 +230,3 @@ btr_rec_set_deleted_flag( rec_set_deleted_flag_old(rec, flag); } } - -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/btr0defragment.h b/storage/innobase/include/btr0defragment.h index 6257c4bc996..21ba6d9f426 100644 --- a/storage/innobase/include/btr0defragment.h +++ b/storage/innobase/include/btr0defragment.h @@ -20,10 +20,6 @@ this program; if not, write to the Free Software Foundation, Inc., #ifndef btr0defragment_h #define btr0defragment_h -#include "univ.i" - -#ifndef UNIV_HOTBACKUP - #include "btr0pcur.h" /* Max number of pages to consider at once during defragmentation. */ @@ -98,7 +94,4 @@ DECLARE_THREAD(btr_defragment_thread)( /*==========================================*/ void* arg); /*!< in: a dummy parameter required by os_thread_create */ - - -#endif /* !UNIV_HOTBACKUP */ #endif diff --git a/storage/innobase/include/btr0pcur.h b/storage/innobase/include/btr0pcur.h index 02f4faf24a5..947316f0e4d 100644 --- a/storage/innobase/include/btr0pcur.h +++ b/storage/innobase/include/btr0pcur.h @@ -114,9 +114,11 @@ btr_pcur_open_low( btr_pcur_t* cursor, /*!< in: memory buffer for persistent cursor */ const char* file, /*!< in: file name */ ulint line, /*!< in: line where called */ + ib_uint64_t autoinc,/*!< in: PAGE_ROOT_AUTO_INC to be written + (0 if none) */ mtr_t* mtr); /*!< in: mtr */ #define btr_pcur_open(i,t,md,l,c,m) \ - btr_pcur_open_low(i,0,t,md,l,c,__FILE__,__LINE__,m) + btr_pcur_open_low(i,0,t,md,l,c,__FILE__,__LINE__,0,m) /**************************************************************//** Opens an persistent cursor to an index tree without initializing the cursor. */ diff --git a/storage/innobase/include/btr0pcur.ic b/storage/innobase/include/btr0pcur.ic index 6cd968b4682..60790bc1316 100644 --- a/storage/innobase/include/btr0pcur.ic +++ b/storage/innobase/include/btr0pcur.ic @@ -434,6 +434,8 @@ btr_pcur_open_low( btr_pcur_t* cursor, /*!< in: memory buffer for persistent cursor */ const char* file, /*!< in: file name */ ulint line, /*!< in: line where called */ + ib_uint64_t autoinc,/*!< in: PAGE_ROOT_AUTO_INC to be written + (0 if none) */ mtr_t* mtr) /*!< in: mtr */ { btr_cur_t* btr_cursor; @@ -452,20 +454,9 @@ btr_pcur_open_low( ut_ad(!dict_index_is_spatial(index)); - if (dict_table_is_intrinsic(index->table)) { - ut_ad((latch_mode & BTR_MODIFY_LEAF) - || (latch_mode & BTR_SEARCH_LEAF) - || (latch_mode & BTR_MODIFY_TREE)); - err = btr_cur_search_to_nth_level_with_no_latch( - index, level, tuple, mode, btr_cursor, - file, line, mtr, - (((latch_mode & BTR_MODIFY_LEAF) - || (latch_mode & BTR_MODIFY_TREE)) ? true : false)); - } else { - err = btr_cur_search_to_nth_level( - index, level, tuple, mode, latch_mode, - btr_cursor, 0, file, line, mtr); - } + err = btr_cur_search_to_nth_level( + index, level, tuple, mode, latch_mode, + btr_cursor, 0, file, line, mtr, autoinc); if (err != DB_SUCCESS) { ib::warn() << " Error code: " << err @@ -521,18 +512,9 @@ btr_pcur_open_with_no_init_func( btr_cursor = btr_pcur_get_btr_cur(cursor); - if (dict_table_is_intrinsic(index->table)) { - ut_ad((latch_mode & BTR_MODIFY_LEAF) - || (latch_mode & BTR_SEARCH_LEAF)); - err = btr_cur_search_to_nth_level_with_no_latch( - index, 0, tuple, mode, btr_cursor, - file, line, mtr, - ((latch_mode & BTR_MODIFY_LEAF) ? true : false)); - } else { - err = btr_cur_search_to_nth_level( - index, 0, tuple, mode, latch_mode, btr_cursor, - has_search_latch, file, line, mtr); - } + err = btr_cur_search_to_nth_level( + index, 0, tuple, mode, latch_mode, btr_cursor, + has_search_latch, file, line, mtr); cursor->pos_state = BTR_PCUR_IS_POSITIONED; @@ -568,15 +550,9 @@ btr_pcur_open_at_index_side( btr_pcur_init(pcur); } - if (dict_table_is_intrinsic(index->table)) { - err = btr_cur_open_at_index_side_with_no_latch( - from_left, index, - btr_pcur_get_btr_cur(pcur), level, mtr); - } else { - err = btr_cur_open_at_index_side( - from_left, index, latch_mode, - btr_pcur_get_btr_cur(pcur), level, mtr); - } + err = btr_cur_open_at_index_side( + from_left, index, latch_mode, + btr_pcur_get_btr_cur(pcur), level, mtr); pcur->pos_state = BTR_PCUR_IS_POSITIONED; pcur->old_stored = false; diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index c4bc107044d..ce2a8e385fc 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -27,6 +27,9 @@ Created 11/5/1995 Heikki Tuuri #ifndef buf0buf_h #define buf0buf_h +/** Magic value to use instead of checksums when they are disabled */ +#define BUF_NO_CHECKSUM_MAGIC 0xDEADBEEFUL + #include "univ.i" #include "fil0fil.h" #include "mtr0types.h" @@ -35,7 +38,6 @@ Created 11/5/1995 Heikki Tuuri #include "hash0hash.h" #include "ut0byte.h" #include "page0types.h" -#ifndef UNIV_HOTBACKUP #include "ut0rbt.h" #include "os0proc.h" #include "log0log.h" @@ -99,20 +101,11 @@ extern volatile ulint buf_withdraw_clock; /*!< the clock is incremented every time a pointer to a page may become obsolete */ -#ifdef UNIV_DEBUG +# ifdef UNIV_DEBUG extern my_bool buf_disable_resize_buffer_pool_debug; /*!< if TRUE, resizing buffer pool is not allowed. */ -#endif /* UNIV_DEBUG */ -#else /* !UNIV_HOTBACKUP */ -extern buf_block_t* back_block1; /*!< first block, for --apply-log */ -extern buf_block_t* back_block2; /*!< second block, for page reorganize */ -#endif /* !UNIV_HOTBACKUP */ -#endif /* !UNIV_INNOCHECKSUM */ - -/** Magic value to use instead of checksums when they are disabled */ -#define BUF_NO_CHECKSUM_MAGIC 0xDEADBEEFUL +# endif /* UNIV_DEBUG */ -#ifndef UNIV_INNOCHECKSUM /** @brief States of a control block @see buf_page_t @@ -341,7 +334,6 @@ operator<<( std::ostream& out, const page_id_t& page_id); -#ifndef UNIV_HOTBACKUP /********************************************************************//** Acquire mutex on all buffer pool instances */ UNIV_INLINE @@ -397,15 +389,11 @@ buf_pool_resize(); /** This is the thread for resizing buffer pool. It waits for an event and when waked up either performs a resizing and sleeps again. -@param[in] arg a dummy parameter required by os_thread_create. @return this function does not return, calls os_thread_exit() */ extern "C" os_thread_ret_t -DECLARE_THREAD(buf_resize_thread)( -/*==============================*/ - void* arg); /*!< in: a dummy parameter - required by os_thread_create */ +DECLARE_THREAD(buf_resize_thread)(void*); /********************************************************************//** Clears the adaptive hash index on all pages in the buffer pool. */ @@ -468,7 +456,7 @@ void buf_block_free( /*===========*/ buf_block_t* block); /*!< in, own: block to be freed */ -#endif /* !UNIV_HOTBACKUP */ + /*********************************************************************//** Copies contents of a buffer frame to a given buffer. @return buf */ @@ -478,7 +466,7 @@ buf_frame_copy( /*===========*/ byte* buf, /*!< in: buffer to copy to */ const buf_frame_t* frame); /*!< in: buffer frame */ -#ifndef UNIV_HOTBACKUP + /**************************************************************//** NOTE! The following macros should be used instead of buf_page_get_gen, to improve debugging. Only values RW_S_LATCH and RW_X_LATCH are allowed @@ -570,9 +558,6 @@ BUF_PEEK_IF_IN_POOL, BUF_GET_NO_LATCH, or BUF_GET_IF_IN_POOL_OR_WATCH @param[in] line line where called @param[in] mtr mini-transaction @param[out] err DB_SUCCESS or error code -@param[in] dirty_with_no_latch - mark page as dirty even if page - is being pinned without any latch @return pointer to the block or NULL */ buf_block_t* buf_page_get_gen( @@ -584,8 +569,7 @@ buf_page_get_gen( const char* file, ulint line, mtr_t* mtr, - dberr_t* err, - bool dirty_with_no_latch = false); + dberr_t* err); /** Initializes a page to the buffer buf_pool. The page is usually not read from a file even if it cannot be found in the buffer buf_pool. This is one @@ -601,21 +585,6 @@ buf_page_create( const page_size_t& page_size, mtr_t* mtr); -#else /* !UNIV_HOTBACKUP */ - -/** Inits a page to the buffer buf_pool, for use in mysqlbackup --restore. -@param[in] page_id page id -@param[in] page_size page size -@param[in,out] block block to init */ -void -buf_page_init_for_backup_restore( - const page_id_t& page_id, - const page_size_t& page_size, - buf_block_t* block); - -#endif /* !UNIV_HOTBACKUP */ - -#ifndef UNIV_HOTBACKUP /********************************************************************//** Releases a compressed-only page acquired with buf_page_get_zip(). */ UNIV_INLINE @@ -792,17 +761,14 @@ buf_block_unfix( @param[in,out] b block to bufferfix @param[in] f file name where requested @param[in] l line number where requested */ -# define buf_block_buf_fix_inc(b,f,l) buf_block_buf_fix_inc_func(f,l,b) +# define buf_block_buf_fix_inc(b,f,l) buf_block_buf_fix_inc_func(f,l,b) # else /* UNIV_DEBUG */ /** Increments the bufferfix count. @param[in,out] b block to bufferfix @param[in] f file name where requested @param[in] l line number where requested */ -# define buf_block_buf_fix_inc(b,f,l) buf_block_buf_fix_inc_func(b) +# define buf_block_buf_fix_inc(b,f,l) buf_block_buf_fix_inc_func(b) # endif /* UNIV_DEBUG */ -#else /* !UNIV_HOTBACKUP */ -# define buf_block_modify_clock_inc(block) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ #endif /* !UNIV_INNOCHECKSUM */ /** Checks if a page contains only zeroes. @@ -839,7 +805,6 @@ buf_page_is_corrupted( #endif /* UNIV_INNOCHECKSUM */ ) MY_ATTRIBUTE((warn_unused_result)); #ifndef UNIV_INNOCHECKSUM -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Gets the space id, page offset, and byte offset within page of a pointer pointing to a buffer frame containing a file page. */ @@ -897,7 +862,7 @@ void buf_print(void); /*============*/ #endif /* UNIV_DEBUG_PRINT || UNIV_DEBUG || UNIV_BUF_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ + enum buf_page_print_flags { /** Do not crash at the end of buf_page_print(). */ BUF_PAGE_PRINT_NO_CRASH = 1, @@ -924,7 +889,7 @@ buf_zip_decompress( /*===============*/ buf_block_t* block, /*!< in/out: block */ ibool check); /*!< in: TRUE=verify the page checksum */ -#ifndef UNIV_HOTBACKUP + #ifdef UNIV_DEBUG /*********************************************************************//** Returns the number of latched pages in the buffer pool. @@ -994,7 +959,6 @@ this function is called: not latched and not modified. */ void buf_pool_invalidate(void); /*=====================*/ -#endif /* !UNIV_HOTBACKUP */ /*======================================================================== --------------------------- LOWER LEVEL ROUTINES ------------------------- @@ -1067,7 +1031,7 @@ buf_page_in_file( /*=============*/ const buf_page_t* bpage) /*!< in: pointer to control block */ MY_ATTRIBUTE((warn_unused_result)); -#ifndef UNIV_HOTBACKUP + /*********************************************************************//** Determines if a block should be on unzip_LRU list. @return TRUE if block belongs to unzip_LRU */ @@ -1225,7 +1189,7 @@ buf_page_get_block( /*===============*/ buf_page_t* bpage) /*!< in: control block, or NULL */ MY_ATTRIBUTE((warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ + #ifdef UNIV_DEBUG /*********************************************************************//** Gets a pointer to the memory frame of a block. @@ -1245,7 +1209,6 @@ Gets the compressed page descriptor corresponding to an uncompressed page if applicable. */ #define buf_block_get_page_zip(block) \ ((block)->page.zip.data ? &(block)->page.zip : NULL) -#ifndef UNIV_HOTBACKUP /** Get a buffer block from an adaptive hash index pointer. This function does not return if the block is not identified. @@ -1429,13 +1392,6 @@ buf_page_hash_get_low() function. #define buf_block_hash_get(b, page_id) \ buf_block_hash_get_locked(b, page_id, NULL, 0) -/*********************************************************************//** -Gets the current length of the free list of buffer blocks. -@return length of the free list */ -ulint -buf_get_free_list_len(void); -/*=======================*/ - /********************************************************************//** Determine if a block is a sentinel for a buffer pool watch. @return TRUE if a sentinel for a buffer pool watch, FALSE if not */ @@ -1535,8 +1491,6 @@ buf_flush_update_zip_checksum( ulint size, lsn_t lsn); -#endif /* !UNIV_HOTBACKUP */ - /********************************************************************//** The hook that is called just before a page is written to disk. The function encrypts the content of the page and returns a pointer @@ -1639,7 +1593,6 @@ public: /** Block state. @see buf_page_in_file */ buf_page_state state; -#ifndef UNIV_HOTBACKUP unsigned flush_type:2; /*!< if this block is currently being flushed to disk, this tells the flush_type. @@ -1650,7 +1603,6 @@ public: # error "MAX_BUFFER_POOLS > 64; redefine buf_pool_index:6" # endif /* @} */ -#endif /* !UNIV_HOTBACKUP */ page_zip_des_t zip; /*!< compressed page; zip.data (but not the data it points to) is also protected by buf_pool->mutex; @@ -1663,7 +1615,6 @@ public: if written again we check is TRIM operation needed. */ - ulint space; /*!< space id */ unsigned key_version; /*!< key version for this block */ bool page_encrypted; /*!< page is page encrypted */ bool page_compressed;/*!< page is page compressed */ @@ -1682,7 +1633,6 @@ public: buf_tmp_buffer_t* slot; /*!< Slot for temporary memory used for encryption/compression or NULL */ - #ifndef UNIV_HOTBACKUP buf_page_t* hash; /*!< node used in chaining to buf_pool->page_hash or buf_pool->zip_hash */ @@ -1796,7 +1746,6 @@ public: protected by buf_pool->zip_mutex or buf_block_t::mutex. */ # endif /* UNIV_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ }; /** The buffer control block structure */ @@ -1814,7 +1763,6 @@ struct buf_block_t{ is of size UNIV_PAGE_SIZE, and aligned to an address divisible by UNIV_PAGE_SIZE */ -#ifndef UNIV_HOTBACKUP BPageLock lock; /*!< read-write lock of the buffer frame */ UT_LIST_NODE_T(buf_block_t) unzip_LRU; @@ -1901,12 +1849,6 @@ struct buf_block_t{ complete, though: there may have been hash collisions, record deletions, etc. */ - bool made_dirty_with_no_latch; - /*!< true if block has been made dirty - without acquiring X/SX latch as the - block belongs to temporary tablespace - and block is always accessed by a - single thread. */ bool skip_flush_check; /*!< Skip check in buf_dblwr_check_block during bulk load, protected by lock.*/ @@ -1926,7 +1868,6 @@ struct buf_block_t{ and accessed; we introduce this new mutex in InnoDB-5.1 to relieve contention on the buffer pool mutex */ -#endif /* !UNIV_HOTBACKUP */ }; /** Check if a buf_block_t object is in a valid state @@ -1937,7 +1878,6 @@ struct buf_block_t{ && (buf_block_get_state(block) <= BUF_BLOCK_REMOVE_HASH)) -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Compute the hash fold value for blocks in buf_pool->zip_hash. */ /* @{ */ @@ -2243,7 +2183,9 @@ struct buf_pool_t{ os_event_t no_flush[BUF_FLUSH_N_TYPES]; /*!< this is in the set state when there is no flush batch - of the given type running */ + of the given type running; + os_event_set() and os_event_reset() + are protected by buf_pool_t::mutex */ ib_rbt_t* flush_rbt; /*!< a red-black tree is used exclusively during recovery to speed up insertions in the @@ -2472,7 +2414,6 @@ Use these instead of accessing buf_pool->mutex directly. */ /** Release the buffer pool mutex. */ # define buf_pool_mutex_exit(b) mutex_exit(&b->mutex) #endif -#endif /* !UNIV_HOTBACKUP */ /* @} */ /********************************************************************** diff --git a/storage/innobase/include/buf0buf.ic b/storage/innobase/include/buf0buf.ic index a912f3eb127..5e75c446bbd 100644 --- a/storage/innobase/include/buf0buf.ic +++ b/storage/innobase/include/buf0buf.ic @@ -32,7 +32,6 @@ Created 11/5/1995 Heikki Tuuri *******************************************************/ #include "mtr0mtr.h" -#ifndef UNIV_HOTBACKUP #include "buf0flu.h" #include "buf0lru.h" #include "buf0rea.h" @@ -208,7 +207,6 @@ buf_page_peek_if_too_old( return(!buf_page_peek_if_young(bpage)); } } -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Gets the state of a block. @@ -385,7 +383,6 @@ buf_page_in_file( return(FALSE); } -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Determines if a block should be on unzip_LRU list. @return TRUE if block belongs to unzip_LRU */ @@ -717,7 +714,6 @@ buf_page_get_block( return(NULL); } -#endif /* !UNIV_HOTBACKUP */ #ifdef UNIV_DEBUG /*********************************************************************//** @@ -738,15 +734,10 @@ buf_block_get_frame( case BUF_BLOCK_ZIP_PAGE: case BUF_BLOCK_ZIP_DIRTY: case BUF_BLOCK_NOT_USED: - if (block->page.encrypted) { - goto ok; - } ut_error; break; case BUF_BLOCK_FILE_PAGE: -# ifndef UNIV_HOTBACKUP ut_a(block->page.buf_fix_count > 0); -# endif /* !UNIV_HOTBACKUP */ /* fall through */ case BUF_BLOCK_READY_FOR_USE: case BUF_BLOCK_MEMORY: @@ -796,7 +787,6 @@ buf_ptr_get_fsp_addr( addr->boffset = ut_align_offset(ptr, UNIV_PAGE_SIZE); } -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Gets the hash value of the page the pointer is pointing to. This can be used in searches in the lock hash table. @@ -866,7 +856,6 @@ buf_block_free( buf_pool_mutex_exit(buf_pool); } -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Copies contents of a buffer frame to a given buffer. @@ -885,7 +874,6 @@ buf_frame_copy( return(buf); } -#ifndef UNIV_HOTBACKUP /********************************************************************//** Gets the youngest modification log sequence number for a frame. Returns zero if not file page or no modification occurred yet. @@ -926,7 +914,7 @@ buf_block_modify_clock_inc( #ifdef UNIV_DEBUG buf_pool_t* buf_pool = buf_pool_from_bpage((buf_page_t*) block); - /* No latch is acquired if block belongs to intrinsic table. */ + /* No latch is acquired for the shared temporary tablespace. */ if (!fsp_is_system_temporary(block->page.id.space())) { ut_ad((buf_pool_mutex_own(buf_pool) && (block->page.buf_fix_count == 0)) @@ -949,7 +937,7 @@ buf_block_get_modify_clock( buf_block_t* block) /*!< in: block */ { #ifdef UNIV_DEBUG - /* No latch is acquired if block belongs to intrinsic table. */ + /* No latch is acquired for the shared temporary tablespace. */ if (!fsp_is_system_temporary(block->page.id.space())) { ut_ad(rw_lock_own(&(block->lock), RW_LOCK_S) || rw_lock_own(&(block->lock), RW_LOCK_X) @@ -1451,14 +1439,12 @@ ulint buf_pool_size_align( ulint size) { - const ulint m = srv_buf_pool_instances * srv_buf_pool_chunk_unit; + const ib_uint64_t m = ((ib_uint64_t)srv_buf_pool_instances) * srv_buf_pool_chunk_unit; size = ut_max(size, srv_buf_pool_min_size); if (size % m == 0) { return(size); } else { - return((size / m + 1) * m); + return (ulint)((size / m + 1) * m); } } - -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/buf0dblwr.h b/storage/innobase/include/buf0dblwr.h index eb13c3b35e5..bf2bf6ab32b 100644 --- a/storage/innobase/include/buf0dblwr.h +++ b/storage/innobase/include/buf0dblwr.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -32,8 +33,6 @@ Created 2011/12/19 Inaam Rana #include "buf0types.h" #include "log0recv.h" -#ifndef UNIV_HOTBACKUP - /** Doublewrite system */ extern buf_dblwr_t* buf_dblwr; /** Set to TRUE when the doublewrite buffer is being created */ @@ -134,11 +133,13 @@ struct buf_dblwr_t{ ulint b_reserved;/*!< number of slots currently reserved for batch flush. */ os_event_t b_event;/*!< event where threads wait for a - batch flush to end. */ + batch flush to end; + os_event_set() and os_event_reset() + are protected by buf_dblwr_t::mutex */ ulint s_reserved;/*!< number of slots currently reserved for single page flushes. */ os_event_t s_event;/*!< event where threads wait for a - single page flush slot. */ + single page flush slot. Protected by mutex. */ bool* in_use; /*!< flag used to indicate if a slot is in use. Only used for single page flushes. */ @@ -156,7 +157,4 @@ struct buf_dblwr_t{ cached to write_buf */ }; - -#endif /* UNIV_HOTBACKUP */ - #endif diff --git a/storage/innobase/include/buf0flu.h b/storage/innobase/include/buf0flu.h index 40083798d48..273f9666419 100644 --- a/storage/innobase/include/buf0flu.h +++ b/storage/innobase/include/buf0flu.h @@ -30,7 +30,6 @@ Created 11/5/1995 Heikki Tuuri #include "univ.i" #include "ut0byte.h" #include "log0log.h" -#ifndef UNIV_HOTBACKUP #include "buf0types.h" /** Flag indicating if the page_cleaner is in active state. */ @@ -77,7 +76,6 @@ void buf_flush_write_complete( /*=====================*/ buf_page_t* bpage); /*!< in: pointer to the block in question */ -#endif /* !UNIV_HOTBACKUP */ /** Initialize a page for writing to the tablespace. @param[in] block buffer block; NULL if bypassing the buffer pool @param[in,out] page page frame @@ -92,7 +90,6 @@ buf_flush_init_for_writing( lsn_t newest_lsn, bool skip_checksum); -#ifndef UNIV_HOTBACKUP # if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG /********************************************************************//** Writes a flushable page asynchronously from the buffer pool to a file. @@ -128,7 +125,6 @@ buf_flush_do_batch( lsn_t lsn_limit, flush_counters_t* n); - /** This utility flushes dirty blocks from the end of the flush list of all buffer pool instances. NOTE: The calling thread is not allowed to own any latches on pages! @@ -438,8 +434,6 @@ private: bool m_interrupted; }; -#endif /* !UNIV_HOTBACKUP */ - /******************************************************************//** Start a buffer flush batch for LRU or flush list */ ibool diff --git a/storage/innobase/include/buf0flu.ic b/storage/innobase/include/buf0flu.ic index ecb98e32619..5a682ed121a 100644 --- a/storage/innobase/include/buf0flu.ic +++ b/storage/innobase/include/buf0flu.ic @@ -23,7 +23,6 @@ The database buffer pool flush algorithm Created 11/5/1995 Heikki Tuuri *******************************************************/ -#ifndef UNIV_HOTBACKUP #include "buf0buf.h" #include "mtr0mtr.h" #include "srv0srv.h" @@ -148,4 +147,3 @@ buf_flush_recv_note_modification( buf_page_mutex_exit(block); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/buf0lru.h b/storage/innobase/include/buf0lru.h index 0cbd77878ec..2ad1211a548 100644 --- a/storage/innobase/include/buf0lru.h +++ b/storage/innobase/include/buf0lru.h @@ -27,7 +27,6 @@ Created 11/5/1995 Heikki Tuuri #define buf0lru_h #include "univ.i" -#ifndef UNIV_HOTBACKUP #include "ut0byte.h" #include "buf0types.h" @@ -292,6 +291,4 @@ Increments the page_zip_decompress() counter in buf_LRU_stat_cur. */ #include "buf0lru.ic" #endif -#endif /* !UNIV_HOTBACKUP */ - #endif diff --git a/storage/innobase/include/data0data.h b/storage/innobase/include/data0data.h index 5537d70548a..8385fe3d0d3 100644 --- a/storage/innobase/include/data0data.h +++ b/storage/innobase/include/data0data.h @@ -196,7 +196,7 @@ dfield_dup( dfield_t* field, /*!< in/out: data field */ mem_heap_t* heap) /*!< in: memory heap where allocated */ MY_ATTRIBUTE((nonnull)); -#ifndef UNIV_HOTBACKUP + /*********************************************************************//** Tests if two data fields are equal. If len==0, tests the data length and content for equality. @@ -222,7 +222,6 @@ dfield_data_is_binary_equal( ulint len, /*!< in: data length or UNIV_SQL_NULL */ const byte* data) /*!< in: data */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Gets number of fields in a data tuple. @return number of fields */ @@ -338,7 +337,6 @@ dtuple_create( ulint n_fields)/*!< in: number of fields */ MY_ATTRIBUTE((nonnull, malloc)); - /** Initialize the virtual field data in a dtuple_t @param[in,out] vrow dtuple contains the virtual fields */ UNIV_INLINE @@ -631,7 +629,6 @@ struct dtuple_t { #endif /* UNIV_DEBUG */ }; - /** A slot for a field in a big rec vector */ struct big_rec_field_t { diff --git a/storage/innobase/include/data0data.ic b/storage/innobase/include/data0data.ic index dc51735d340..2fe5190591b 100644 --- a/storage/innobase/include/data0data.ic +++ b/storage/innobase/include/data0data.ic @@ -285,7 +285,6 @@ dfield_dup( } } -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Tests if two data fields are equal. If len==0, tests the data length and content for equality. @@ -330,7 +329,6 @@ dfield_data_is_binary_equal( && (len == UNIV_SQL_NULL || !memcmp(dfield_get_data(field), data, len))); } -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Gets info bits in a data tuple. @@ -469,7 +467,6 @@ dtuple_create_from_mem( dtuple_t* tuple; ulint n_t_fields = n_fields + n_v_fields; - ut_ad(buf != NULL); ut_a(buf_size >= DTUPLE_EST_ALLOC(n_t_fields)); tuple = (dtuple_t*) buf; diff --git a/storage/innobase/include/data0type.h b/storage/innobase/include/data0type.h index 00073dfca2c..d38aa7c1533 100644 --- a/storage/innobase/include/data0type.h +++ b/storage/innobase/include/data0type.h @@ -166,10 +166,6 @@ be less than 256 */ #define DATA_N_SYS_COLS 3 /* number of system columns defined above */ -#define DATA_ITT_N_SYS_COLS 2 - /* number of system columns for intrinsic - temporary table */ - #define DATA_FTS_DOC_ID 3 /* Used as FTS DOC ID column */ #define DATA_SYS_PRTYPE_MASK 0xF /* mask to extract the above from prtype */ @@ -196,6 +192,11 @@ be less than 256 */ for shorter VARCHARs MySQL uses only 1 byte */ #define DATA_VIRTUAL 8192 /* Virtual column */ +/** Get the number of system columns in a table. */ +#define dict_table_get_n_sys_cols(table) DATA_N_SYS_COLS +/** Check whether locking is disabled (never). */ +#define dict_table_is_locking_disabled(table) false + /*-------------------------------------------*/ /* This many bytes we need to store the type information affecting the @@ -259,7 +260,6 @@ the underling datatype of GEOMETRY(not DATA_POINT) data. */ /* Mask to get the Charset Collation number (0x7fff) */ #define CHAR_COLL_MASK MAX_CHAR_COLL_NUM -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Gets the MySQL type code from a dtype. @return MySQL type code; this is NOT an InnoDB type code! */ @@ -285,7 +285,6 @@ dtype_get_at_most_n_mbchars( ulint data_len, /*!< in: length of str (in bytes) */ const char* str); /*!< in: the string whose prefix length is being determined */ -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Checks if a data main type is a string type. Also a BLOB is considered a string type. @@ -349,7 +348,7 @@ ulint dtype_get_prtype( /*=============*/ const dtype_t* type); /*!< in: data type */ -#ifndef UNIV_HOTBACKUP + /*********************************************************************//** Compute the mbminlen and mbmaxlen members of a data type structure. */ UNIV_INLINE @@ -390,7 +389,6 @@ ibool dtype_is_utf8( /*==========*/ ulint prtype);/*!< in: precise data type */ -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Gets the type length. @return fixed length of the type, in bytes, or 0 if variable-length */ @@ -399,7 +397,7 @@ ulint dtype_get_len( /*==========*/ const dtype_t* type); /*!< in: data type */ -#ifndef UNIV_HOTBACKUP + /*********************************************************************//** Gets the minimum length of a character, in bytes. @return minimum length of a char, in bytes, or 0 if this is not a @@ -431,7 +429,6 @@ dtype_set_mbminmaxlen( ulint mbmaxlen); /*!< in: maximum length of a char, in bytes, or 0 if this is not a character type */ -#endif /* !UNIV_HOTBACKUP */ /***********************************************************************//** Returns the size of a fixed size data type, 0 if not a fixed size type. @return fixed size, or 0 */ @@ -445,7 +442,7 @@ dtype_get_fixed_size_low( ulint mbminmaxlen, /*!< in: minimum and maximum length of a multibyte character, in bytes */ ulint comp); /*!< in: nonzero=ROW_FORMAT=COMPACT */ -#ifndef UNIV_HOTBACKUP + /***********************************************************************//** Returns the minimum size of a data type. @return minimum size */ @@ -468,7 +465,6 @@ dtype_get_max_size_low( /*===================*/ ulint mtype, /*!< in: main type */ ulint len); /*!< in: length */ -#endif /* !UNIV_HOTBACKUP */ /***********************************************************************//** Returns the ROW_FORMAT=REDUNDANT stored SQL NULL size of a type. For fixed length types it is the fixed length of the type, otherwise 0. @@ -479,7 +475,7 @@ dtype_get_sql_null_size( /*====================*/ const dtype_t* type, /*!< in: type */ ulint comp); /*!< in: nonzero=ROW_FORMAT=COMPACT */ -#ifndef UNIV_HOTBACKUP + /**********************************************************************//** Reads to a type the stored information which determines its alphabetical ordering and the storage size of an SQL NULL value. */ @@ -527,8 +523,6 @@ dtype_sql_name( char* name, /*!< out: SQL name */ unsigned name_sz);/*!< in: size of the name buffer */ -#endif /* !UNIV_HOTBACKUP */ - /*********************************************************************//** Validates a data type structure. @return TRUE if ok */ @@ -571,13 +565,11 @@ struct dtype_t{ string data (in addition to the string, MySQL uses 1 or 2 bytes to store the string length) */ -#ifndef UNIV_HOTBACKUP unsigned mbminmaxlen:5; /*!< minimum and maximum length of a character, in bytes; DATA_MBMINMAXLEN(mbminlen,mbmaxlen); mbminlen=DATA_MBMINLEN(mbminmaxlen); mbmaxlen=DATA_MBMINLEN(mbminmaxlen) */ -#endif /* !UNIV_HOTBACKUP */ }; #ifndef UNIV_NONINL diff --git a/storage/innobase/include/data0type.ic b/storage/innobase/include/data0type.ic index 57770ec0e17..869d0ba8ed6 100644 --- a/storage/innobase/include/data0type.ic +++ b/storage/innobase/include/data0type.ic @@ -24,8 +24,7 @@ Created 1/16/1996 Heikki Tuuri *******************************************************/ #include "mach0data.h" -#ifndef UNIV_HOTBACKUP -# include "ha_prototypes.h" +#include "ha_prototypes.h" /*********************************************************************//** Gets the MySQL charset-collation code for MySQL string types. @@ -137,9 +136,6 @@ dtype_set_mblen( ut_ad(dtype_validate(type)); } -#else /* !UNIV_HOTBACKUP */ -# define dtype_set_mblen(type) (void) 0 -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Sets a data type structure. */ @@ -218,7 +214,6 @@ dtype_get_len( return(type->len); } -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Gets the minimum length of a character, in bytes. @return minimum length of a char, in bytes, or 0 if this is not a @@ -471,8 +466,6 @@ dtype_sql_name( return(name); } -#endif /* !UNIV_HOTBACKUP */ - /***********************************************************************//** Returns the size of a fixed size data type, 0 if not a fixed size type. @return fixed size, or 0 */ @@ -513,7 +506,6 @@ dtype_get_fixed_size_low( case DATA_POINT: return(len); case DATA_MYSQL: -#ifndef UNIV_HOTBACKUP if (prtype & DATA_BINARY_TYPE) { return(len); } else if (!comp) { @@ -534,9 +526,6 @@ dtype_get_fixed_size_low( return(len); } } -#else /* !UNIV_HOTBACKUP */ - return(len); -#endif /* !UNIV_HOTBACKUP */ /* fall through for variable-length charsets */ case DATA_VARCHAR: case DATA_BINARY: @@ -553,7 +542,6 @@ dtype_get_fixed_size_low( return(0); } -#ifndef UNIV_HOTBACKUP /***********************************************************************//** Returns the minimum size of a data type. @return minimum size */ @@ -659,7 +647,6 @@ dtype_get_max_size_low( return(ULINT_MAX); } -#endif /* !UNIV_HOTBACKUP */ /***********************************************************************//** Returns the ROW_FORMAT=REDUNDANT stored SQL NULL size of a type. @@ -672,11 +659,6 @@ dtype_get_sql_null_size( const dtype_t* type, /*!< in: type */ ulint comp) /*!< in: nonzero=ROW_FORMAT=COMPACT */ { -#ifndef UNIV_HOTBACKUP return(dtype_get_fixed_size_low(type->mtype, type->prtype, type->len, type->mbminmaxlen, comp)); -#else /* !UNIV_HOTBACKUP */ - return(dtype_get_fixed_size_low(type->mtype, type->prtype, type->len, - 0, 0)); -#endif /* !UNIV_HOTBACKUP */ } diff --git a/storage/innobase/include/db0err.h b/storage/innobase/include/db0err.h index 32f9117af84..b0609991f61 100644 --- a/storage/innobase/include/db0err.h +++ b/storage/innobase/include/db0err.h @@ -146,24 +146,6 @@ enum dberr_t { DB_IO_ERROR = 100, /*!< Generic IO error */ - DB_IO_DECOMPRESS_FAIL, /*!< Failure to decompress a page - after reading it from disk */ - - DB_IO_NO_PUNCH_HOLE, /*!< Punch hole not supported by - InnoDB */ - - DB_IO_NO_PUNCH_HOLE_FS, /*!< The file system doesn't support - punch hole */ - - DB_IO_NO_PUNCH_HOLE_TABLESPACE, /*!< The tablespace doesn't support - punch hole */ - - DB_IO_DECRYPT_FAIL, /*!< Failure to decrypt a page - after reading it from disk */ - - DB_IO_NO_ENCRYPT_TABLESPACE, /*!< The tablespace doesn't support - encrypt */ - DB_IO_PARTIAL_FAILED, /*!< Partial IO request failed */ DB_FORCED_ABORT, /*!< Transaction was forced to rollback @@ -172,14 +154,15 @@ enum dberr_t { DB_TABLE_CORRUPT, /*!< Table/clustered index is corrupted */ - DB_WRONG_FILE_NAME, /*!< Invalid Filename */ - DB_COMPUTE_VALUE_FAILED, /*!< Compute generated value failed */ DB_NO_FK_ON_S_BASE_COL, /*!< Cannot add foreign constrain placed on the base column of stored column */ + DB_IO_NO_PUNCH_HOLE, /*!< Punch hole not supported by + file system. */ + /* The following are partial failure codes */ DB_FAIL = 1000, DB_OVERFLOW, @@ -190,16 +173,6 @@ enum dberr_t { DB_END_OF_INDEX, DB_NOT_FOUND, /*!< Generic error code for "Not found" type of errors */ - - /* The following are API only error codes. */ - DB_DATA_MISMATCH = 2000, /*!< Column update or read failed - because the types mismatch */ - - DB_SCHEMA_NOT_LOCKED /*!< If an API function expects the - schema to be locked in exclusive mode - and if it's not then that API function - will return this error code */ - }; #endif diff --git a/storage/innobase/include/dict0crea.h b/storage/innobase/include/dict0crea.h index f9ef39fa8c6..9065f7ebf3d 100644 --- a/storage/innobase/include/dict0crea.h +++ b/storage/innobase/include/dict0crea.h @@ -67,13 +67,6 @@ dict_create_table_step( /*===================*/ que_thr_t* thr); /*!< in: query thread */ -/** Builds a tablespace to store various objects. -@param[in,out] tablespace Tablespace object describing what to build. -@return DB_SUCCESS or error code. */ -dberr_t -dict_build_tablespace( - Tablespace* tablespace); - /** Builds a tablespace to contain a table, using file-per-table=1. @param[in,out] table Table to build in its own tablespace. @param[in] node Table create node @@ -276,7 +269,6 @@ Add a foreign key definition to the data dictionary tables. dberr_t dict_create_add_foreign_to_dictionary( /*==================================*/ - dict_table_t* table, /*!< in: table */ const char* name, /*!< in: table name */ const dict_foreign_t* foreign,/*!< in: foreign key */ trx_t* trx) /*!< in/out: dictionary transaction */ diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index ccef08ff73f..036d905f0d8 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2015, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -51,8 +51,7 @@ Created 1/8/1996 Heikki Tuuri extern bool innodb_table_stats_not_found; extern bool innodb_index_stats_not_found; -#ifndef UNIV_HOTBACKUP -# include "sync0rw.h" +#include "sync0rw.h" /********************************************************************//** Get the database name length in a table name. @return database name length */ @@ -228,7 +227,6 @@ dict_max_v_field_len_store_undo( dict_table_t* table, ulint col_no); -#endif /* !UNIV_HOTBACKUP */ #ifdef UNIV_DEBUG /*********************************************************************//** Assert that a column and a data type match. @@ -241,7 +239,7 @@ dict_col_type_assert_equal( const dtype_t* type) /*!< in: data type */ MY_ATTRIBUTE((nonnull, warn_unused_result)); #endif /* UNIV_DEBUG */ -#ifndef UNIV_HOTBACKUP + /***********************************************************************//** Returns the minimum size of the column. @return minimum size */ @@ -327,46 +325,52 @@ dict_table_autoinc_lock( /*====================*/ dict_table_t* table) /*!< in/out: table */ MY_ATTRIBUTE((nonnull)); -/********************************************************************//** -Unconditionally set the autoinc counter. */ +/** Unconditionally set the AUTO_INCREMENT counter. +@param[in,out] table table or partition +@param[in] value next available AUTO_INCREMENT value */ +MY_ATTRIBUTE((nonnull)) +UNIV_INLINE void -dict_table_autoinc_initialize( -/*==========================*/ - dict_table_t* table, /*!< in/out: table */ - ib_uint64_t value) /*!< in: next value to assign to a row */ - MY_ATTRIBUTE((nonnull)); +dict_table_autoinc_initialize(dict_table_t* table, ib_uint64_t value) +{ + ut_ad(dict_table_autoinc_own(table)); + table->autoinc = value; +} -/** Store autoinc value when the table is evicted. -@param[in] table table evicted */ -void -dict_table_autoinc_store( - const dict_table_t* table); +/** +@param[in] table table or partition +@return the next AUTO_INCREMENT counter value +@retval 0 if AUTO_INCREMENT is not yet initialized */ +MY_ATTRIBUTE((nonnull, warn_unused_result)) +UNIV_INLINE +ib_uint64_t +dict_table_autoinc_read(const dict_table_t* table) +{ + ut_ad(dict_table_autoinc_own(table)); + return(table->autoinc); +} -/** Restore autoinc value when the table is loaded. -@param[in] table table loaded */ -void -dict_table_autoinc_restore( - dict_table_t* table); +/** Update the AUTO_INCREMENT sequence if the value supplied is greater +than the current value. +@param[in,out] table table or partition +@param[in] value AUTO_INCREMENT value that was assigned to a row +@return whether the AUTO_INCREMENT sequence was updated */ +MY_ATTRIBUTE((nonnull)) +UNIV_INLINE +bool +dict_table_autoinc_update_if_greater(dict_table_t* table, ib_uint64_t value) +{ + ut_ad(dict_table_autoinc_own(table)); -/********************************************************************//** -Reads the next autoinc value (== autoinc counter value), 0 if not yet -initialized. -@return value for a new row, or 0 */ -ib_uint64_t -dict_table_autoinc_read( -/*====================*/ - const dict_table_t* table) /*!< in: table */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); -/********************************************************************//** -Updates the autoinc counter if the value supplied is greater than the -current value. */ -void -dict_table_autoinc_update_if_greater( -/*=================================*/ + if (value > table->autoinc) { + + table->autoinc = value; + return(true); + } + + return(false); +} - dict_table_t* table, /*!< in/out: table */ - ib_uint64_t value) /*!< in: value which was assigned to a row */ - MY_ATTRIBUTE((nonnull)); /********************************************************************//** Release the autoinc lock. */ void @@ -374,7 +378,6 @@ dict_table_autoinc_unlock( /*======================*/ dict_table_t* table) /*!< in/out: table */ MY_ATTRIBUTE((nonnull)); -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Adds system columns to a table object. */ void @@ -383,7 +386,7 @@ dict_table_add_system_columns( dict_table_t* table, /*!< in/out: table */ mem_heap_t* heap) /*!< in: temporary heap */ MY_ATTRIBUTE((nonnull)); -#ifndef UNIV_HOTBACKUP + /** Mark if table has big rows. @param[in,out] table table handler */ void @@ -412,8 +415,9 @@ void dict_table_remove_from_cache_low( /*=============================*/ dict_table_t* table, /*!< in, own: table */ - ibool lru_evict); /*!< in: TRUE if table being evicted + ibool lru_evict) /*!< in: TRUE if table being evicted to make room in the table LRU list */ + MY_ATTRIBUTE((nonnull)); /**********************************************************************//** Renames a table object. @return TRUE if success */ @@ -613,18 +617,6 @@ dict_table_get_col_name( ulint col_nr) /*!< in: column number */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -/**********************************************************************//** -Returns a column's name. -@return column name. NOTE: not guaranteed to stay valid if table is -modified in any way (columns added, etc.). */ -UNIV_INTERN -const char* -dict_table_get_col_name_for_mysql( -/*==============================*/ - const dict_table_t* table, /*!< in: table */ - const char* col_name)/*!< in: MySQL table column name */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); - /** Returns a virtual column's name. @param[in] table table object @param[in] col_nr virtual column number(nth virtual column) @@ -736,7 +728,6 @@ dict_table_get_next_index( # define dict_table_get_last_index(table) UT_LIST_GET_LAST((table)->indexes) # define dict_table_get_next_index(index) UT_LIST_GET_NEXT(indexes, index) #endif /* UNIV_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ /* Skip corrupted index */ #define dict_table_skip_corrupt_index(index) \ @@ -843,17 +834,6 @@ ulint dict_table_get_n_tot_u_cols( const dict_table_t* table); /********************************************************************//** -Gets the number of system columns in a table. -For intrinsic table on ROW_ID column is added for all other -tables TRX_ID and ROLL_PTR are all also appeneded. -@return number of system (e.g., ROW_ID) columns of a table */ -UNIV_INLINE -ulint -dict_table_get_n_sys_cols( -/*======================*/ - const dict_table_t* table) /*!< in: table */ - MY_ATTRIBUTE((warn_unused_result)); -/********************************************************************//** Gets the number of all non-virtual columns (also system) in a table in the dictionary cache. @return number of columns of a table */ @@ -910,7 +890,6 @@ dict_table_n_rows_dec( dict_table_t* table) /*!< in/out: table */ MY_ATTRIBUTE((nonnull)); - /** Get nth virtual column @param[in] table target table @param[in] col_nr column number in MySQL Table definition @@ -969,7 +948,7 @@ dict_table_get_sys_col_no( const dict_table_t* table, /*!< in: table */ ulint sys) /*!< in: DATA_ROW_ID, ... */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -#ifndef UNIV_HOTBACKUP + /********************************************************************//** Returns the minimum data size of an index record. @return minimum data size in bytes */ @@ -979,7 +958,6 @@ dict_index_get_min_size( /*====================*/ const dict_index_t* index) /*!< in: index */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /********************************************************************//** Check whether the table uses the compact page format. @return TRUE if table uses the compact page format */ @@ -1014,7 +992,9 @@ dict_tf_get_format( @param[in] format, File Format @param[in] zip_ssize Zip Shift Size @param[in] use_data_dir Table uses DATA DIRECTORY -@param[in] shared_space Table uses a General Shared Tablespace */ +@param[in] page_compressed Table uses page compression +@param[in] page_compression_level Page compression level +@param[in] not_used For future */ UNIV_INLINE void dict_tf_set( @@ -1022,28 +1002,9 @@ dict_tf_set( rec_format_t format, ulint zip_ssize, bool use_data_dir, - bool shared_space, - bool page_compressed, - ulint page_compression_level, - ulint atomic_writes); - -/** Initialize a dict_table_t::flags pointer. -@param[in] compact, Table uses Compact or greater -@param[in] zip_ssize Zip Shift Size (log 2 minus 9) -@param[in] atomic_blobs Table uses Compressed or Dynamic -@param[in] data_dir Table uses DATA DIRECTORY -@param[in] shared_space Table uses a General Shared Tablespace */ -UNIV_INLINE -ulint -dict_tf_init( - bool compact, - ulint zip_ssize, - bool atomic_blobs, - bool data_dir, - bool shared_space, bool page_compressed, ulint page_compression_level, - ulint atomic_writes); + ulint not_used); /** Convert a 32 bit integer table flags to the 32 bit FSP Flags. Fsp Flags are written into the tablespace header at the offset @@ -1056,14 +1017,10 @@ dict_table_t::flags | 0 | 1 | 1 | 1 fil_space_t::flags | 0 | 0 | 1 | 1 ================================================================== @param[in] table_flags dict_table_t::flags -@param[in] is_temp whether the tablespace is temporary -@param[in] is_encrypted whether the tablespace is encrypted @return tablespace flags (fil_space_t::flags) */ +UNIV_INLINE ulint -dict_tf_to_fsp_flags( - ulint table_flags, - bool is_temp, - bool is_encrypted = false) +dict_tf_to_fsp_flags(ulint table_flags) MY_ATTRIBUTE((const)); /** Extract the page size from table flags. @@ -1092,7 +1049,6 @@ dict_table_page_size( const dict_table_t* table) MY_ATTRIBUTE((warn_unused_result)); -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Obtain exclusive locks on all index trees of the table. This is to prevent accessing index trees while InnoDB is updating internal metadata for @@ -1222,7 +1178,6 @@ dict_index_add_to_cache_w_vcol( ulint page_no, ibool strict) MY_ATTRIBUTE((warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /********************************************************************//** Gets the number of fields in the internal representation of an index, including fields added by the dictionary system. @@ -1413,7 +1368,7 @@ dict_index_add_col( dict_col_t* col, /*!< in: column */ ulint prefix_len) /*!< in: column prefix length */ MY_ATTRIBUTE((nonnull)); -#ifndef UNIV_HOTBACKUP + /*******************************************************************//** Copies types of fields contained in index to tuple. */ void @@ -1424,7 +1379,6 @@ dict_index_copy_types( ulint n_fields) /*!< in: number of field types to copy */ MY_ATTRIBUTE((nonnull)); -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Gets the field column. @return field->col, pointer to the table column */ @@ -1434,7 +1388,7 @@ dict_field_get_col( /*===============*/ const dict_field_t* field) /*!< in: index field */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -#ifndef UNIV_HOTBACKUP + /**********************************************************************//** Returns an index object if it is found in the dictionary cache. Assumes that dict_sys->mutex is already being held. @@ -1769,8 +1723,6 @@ extern dict_sys_t* dict_sys; /** the data dictionary rw-latch protecting dict_sys */ extern rw_lock_t* dict_operation_lock; -typedef std::map<table_id_t, ib_uint64_t> autoinc_map_t; - /* Dictionary system struct */ struct dict_sys_t{ DictSysMutex mutex; /*!< mutex protecting the data @@ -1806,19 +1758,18 @@ struct dict_sys_t{ UT_LIST_BASE_NODE_T(dict_table_t) table_non_LRU; /*!< List of tables that can't be evicted from the cache */ - autoinc_map_t* autoinc_map; /*!< Map to store table id and autoinc - when table is evicted */ }; -#endif /* !UNIV_HOTBACKUP */ /** dummy index for ROW_FORMAT=REDUNDANT supremum and infimum records */ extern dict_index_t* dict_ind_redundant; -/**********************************************************************//** -Inits dict_ind_redundant. */ +/** Initialize dict_ind_redundant. */ void -dict_ind_init(void); -/*===============*/ +dict_ind_init(); + +/** Free dict_ind_redundant. */ +void +dict_ind_free(); /* Auxiliary structs for checking a table definition @{ */ @@ -1901,7 +1852,7 @@ Closes the data dictionary module. */ void dict_close(void); /*============*/ -#ifndef UNIV_HOTBACKUP + /**********************************************************************//** Check whether the table is corrupted. @return nonzero for corrupted table, zero for valid tables */ @@ -1922,7 +1873,6 @@ dict_index_is_corrupted( const dict_index_t* index) /*!< in: index */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Flags an index and table corrupted both in the data dictionary cache and in the system table SYS_INDEXES. */ @@ -2007,55 +1957,6 @@ dict_table_is_temporary( MY_ATTRIBUTE((warn_unused_result)); /********************************************************************//** -Check if it is a encrypted table. -@return true if table encryption flag is set. */ -UNIV_INLINE -bool -dict_table_is_encrypted( -/*====================*/ - const dict_table_t* table) /*!< in: table to check */ - MY_ATTRIBUTE((warn_unused_result)); - -/** Check whether the table is intrinsic. -An intrinsic table is a special kind of temporary table that -is invisible to the end user. It is created internally by the MySQL server -layer or other module connected to InnoDB in order to gather and use data -as part of a larger task. Since access to it must be as fast as possible, -it does not need UNDO semantics, system fields DB_TRX_ID & DB_ROLL_PTR, -doublewrite, checksum, insert buffer, use of the shared data dictionary, -locking, or even a transaction. In short, these are not ACID tables at all, -just temporary - -@param[in] table table to check -@return true if intrinsic table flag is set. */ -UNIV_INLINE -bool -dict_table_is_intrinsic( - const dict_table_t* table) - MY_ATTRIBUTE((warn_unused_result)); - -/** Check if the table is in a shared tablespace (System or General). -@param[in] id Space ID to check -@return true if id is a shared tablespace, false if not. */ -UNIV_INLINE -bool -dict_table_in_shared_tablespace( - const dict_table_t* table) - MY_ATTRIBUTE((warn_unused_result)); - -/** Check whether locking is disabled for this table. -Currently this is done for intrinsic table as their visibility is limited -to the connection only. - -@param[in] table table to check -@return true if locking is disabled. */ -UNIV_INLINE -bool -dict_table_is_locking_disabled( - const dict_table_t* table) - MY_ATTRIBUTE((warn_unused_result)); - -/********************************************************************//** Turn-off redo-logging if temporary table. */ UNIV_INLINE void @@ -2064,31 +1965,6 @@ dict_disable_redo_if_temporary( const dict_table_t* table, /*!< in: table to check */ mtr_t* mtr); /*!< out: mini-transaction */ -/** Get table session row-id and increment the row-id counter for next use. -@param[in,out] table table handler -@return next table local row-id. */ -UNIV_INLINE -row_id_t -dict_table_get_next_table_sess_row_id( - dict_table_t* table); - -/** Get table session trx-id and increment the trx-id counter for next use. -@param[in,out] table table handler -@return next table local trx-id. */ -UNIV_INLINE -trx_id_t -dict_table_get_next_table_sess_trx_id( - dict_table_t* table); - -/** Get current session trx-id. -@param[in] table table handler -@return table local trx-id. */ -UNIV_INLINE -trx_id_t -dict_table_get_curr_table_sess_trx_id( - const dict_table_t* table); - -#ifndef UNIV_HOTBACKUP /*********************************************************************//** This function should be called whenever a page is successfully compressed. Updates the compression padding information. */ @@ -2129,18 +2005,6 @@ dict_index_node_ptr_max_size( /*=========================*/ const dict_index_t* index) /*!< in: index */ MY_ATTRIBUTE((warn_unused_result)); -/*****************************************************************//** -Get index by first field of the index -@return index which is having first field matches -with the field present in field_index position of table */ -UNIV_INLINE -dict_index_t* -dict_table_get_index_on_first_col( -/*==============================*/ - const dict_table_t* table, /*!< in: table */ - ulint col_index, /*!< in: position of column - in table */ - const char* field_name); /*!< in: field name */ /** Check if a column is a virtual column @param[in] col column @return true if it is a virtual column, false otherwise */ @@ -2201,9 +2065,6 @@ bool dict_table_have_virtual_index( dict_table_t* table); -#endif /* !UNIV_HOTBACKUP */ - - #ifndef UNIV_NONINL #include "dict0dict.ic" #endif diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index 8165263c95c..905a1b78782 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -25,7 +25,6 @@ Created 1/8/1996 Heikki Tuuri ***********************************************************************/ #include "data0type.h" -#ifndef UNIV_HOTBACKUP #include "dict0load.h" #include "rem0types.h" #include "fsp0fsp.h" @@ -101,8 +100,6 @@ dict_col_is_virtual( return(col->prtype & DATA_VIRTUAL); } -#endif /* !UNIV_HOTBACKUP */ - #ifdef UNIV_DEBUG /*********************************************************************//** Assert that a column and a data type match. @@ -120,15 +117,12 @@ dict_col_type_assert_equal( ut_ad(col->mtype == type->mtype); ut_ad(col->prtype == type->prtype); //ut_ad(col->len == type->len); -# ifndef UNIV_HOTBACKUP ut_ad(col->mbminmaxlen == type->mbminmaxlen); -# endif /* !UNIV_HOTBACKUP */ return(TRUE); } #endif /* UNIV_DEBUG */ -#ifndef UNIV_HOTBACKUP /***********************************************************************//** Returns the minimum size of the column. @return minimum size */ @@ -152,7 +146,6 @@ dict_col_get_max_size( { return(dtype_get_max_size_low(col->mtype, col->len)); } -#endif /* !UNIV_HOTBACKUP */ /***********************************************************************//** Returns the size of a fixed size column, 0 if not a fixed size column. @return fixed size, or 0 */ @@ -245,7 +238,6 @@ dict_col_get_index_pos( return(ULINT_UNDEFINED); } -#ifndef UNIV_HOTBACKUP #ifdef UNIV_DEBUG /********************************************************************//** Gets the first index on the table (the clustered index). @@ -293,7 +285,6 @@ dict_table_get_next_index( return(UT_LIST_GET_NEXT(indexes, (dict_index_t*) index)); } #endif /* UNIV_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ /********************************************************************//** Check whether the index is the clustered index. @@ -447,23 +438,6 @@ dict_table_get_n_tot_u_cols( return(dict_table_get_n_user_cols(table) + dict_table_get_n_v_cols(table)); } -/********************************************************************//** -Gets the number of system columns in a table. -For intrinsic table on ROW_ID column is added for all other -tables TRX_ID and ROLL_PTR are all also appeneded. -@return number of system (e.g., ROW_ID) columns of a table */ -UNIV_INLINE -ulint -dict_table_get_n_sys_cols( -/*======================*/ - const dict_table_t* table MY_ATTRIBUTE((unused))) /*!< in: table */ -{ - ut_ad(table); - ut_ad(table->magic_n == DICT_TABLE_MAGIC_N); - - return(dict_table_is_intrinsic(table) - ? DATA_ITT_N_SYS_COLS : DATA_N_SYS_COLS); -} /********************************************************************//** Gets the number of all non-virtual columns (also system) in a table @@ -686,11 +660,9 @@ dict_tf_is_valid( ulint zip_ssize = DICT_TF_GET_ZIP_SSIZE(flags); bool atomic_blobs = DICT_TF_HAS_ATOMIC_BLOBS(flags); bool data_dir = DICT_TF_HAS_DATA_DIR(flags); - bool shared_space = DICT_TF_HAS_SHARED_SPACE(flags); ulint unused = DICT_TF_GET_UNUSED(flags); bool page_compression = DICT_TF_GET_PAGE_COMPRESSION(flags); ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(flags); - ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(flags); bool flags_corrupt = false; /* Make sure there are no bits that we do not know about. */ @@ -737,17 +709,6 @@ dict_tf_is_valid( } } - if (atomic_writes) { - - if(atomic_writes > ATOMIC_WRITES_OFF) { - flags_corrupt = true; - } - } - - /* HAS_DATA_DIR and SHARED_SPACE are mutually exclusive. */ - if (data_dir && shared_space) { - flags_corrupt = true; - } if (flags_corrupt) { ib::error() @@ -759,9 +720,7 @@ dict_tf_is_valid( << " data_dir:" << data_dir << " zip_ssize:" << zip_ssize << " page_compression:" << page_compression - << " page_compression_level:" << page_compression_level - << " atomic_writes:" << atomic_writes - << " shared_space:" << shared_space; + << " page_compression_level:" << page_compression_level; return (false); } else { return(true); @@ -787,13 +746,6 @@ dict_tf2_is_valid( return(false); } - bool file_per_table = ((flags2 & DICT_TF2_USE_FILE_PER_TABLE) != 0); - bool shared_space = DICT_TF_HAS_SHARED_SPACE(flags); - - if (file_per_table && shared_space) { - return(false); - } - return(true); } @@ -815,9 +767,6 @@ dict_sys_tables_type_validate( ulint unused = DICT_TF_GET_UNUSED(type); bool page_compression = DICT_TF_GET_PAGE_COMPRESSION(type); ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(type); - ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(type); - - ut_a(atomic_writes <= ATOMIC_WRITES_OFF); /* The low order bit of SYS_TABLES.TYPE is always set to 1. If the format is UNIV_FORMAT_B or higher, this field is the same @@ -901,13 +850,6 @@ dict_sys_tables_type_validate( } } - /* Validate that the atomic writes number is within allowed range. */ - if (atomic_writes > ATOMIC_WRITES_OFF) { - ib::error() << "SYS_TABLES::TYPE=" << type - << " atomic_writes:" << atomic_writes; - return(ULINT_UNDEFINED); - } - /* Return the validated SYS_TABLES.TYPE. */ return(type); } @@ -975,11 +917,9 @@ dict_table_get_format( @param[in] format File Format @param[in] zip_ssize Zip Shift Size @param[in] use_data_dir Table uses DATA DIRECTORY -@param[in] atomic_writes Does table use atomic writes -@param[in] shared_space Table uses a General Shared Tablespace @param[in] page_compressed Table uses page compression @param[in] page_compression_level Page compression level -@param[in] atomic_writes Table uses atomic writes */ +@param[in] not_used For future */ UNIV_INLINE void dict_tf_set( @@ -988,10 +928,9 @@ dict_tf_set( rec_format_t format, ulint zip_ssize, bool use_data_dir, - bool shared_space, bool page_compressed, ulint page_compression_level, - ulint atomic_writes) + ulint not_used) { switch (format) { case REC_FORMAT_REDUNDANT: @@ -1018,10 +957,6 @@ dict_tf_set( *flags |= (1 << DICT_TF_POS_DATA_DIR); } - if (shared_space) { - *flags |= (1 << DICT_TF_POS_SHARED_SPACE); - } - if (page_compressed) { *flags |= (1 << DICT_TF_POS_ATOMIC_BLOBS) | (1 << DICT_TF_POS_PAGE_COMPRESSION) @@ -1031,72 +966,58 @@ dict_tf_set( ut_ad(dict_tf_get_page_compression(*flags) == TRUE); ut_ad(dict_tf_get_page_compression_level(*flags) == page_compression_level); } - - if (atomic_writes) { - *flags |= (atomic_writes << DICT_TF_POS_ATOMIC_WRITES); - ut_a(dict_tf_get_atomic_writes(*flags) == atomic_writes); - } } -/** Initialize a dict_table_t::flags pointer. -@param[in] compact, Table uses Compact or greater -@param[in] zip_ssize Zip Shift Size (log 2 minus 9) -@param[in] atomic_blobs Table uses Compressed or Dynamic -@param[in] data_dir Table uses DATA DIRECTORY -@param[in] shared_space Table uses a General Shared Tablespace -@param[in] page_compression Table uses page compression -@param[in] page_compression_level used compression level -@param[in] atomic_writes Table atomic writes option */ +/** Convert a 32 bit integer table flags to the 32 bit FSP Flags. +Fsp Flags are written into the tablespace header at the offset +FSP_SPACE_FLAGS and are also stored in the fil_space_t::flags field. +The following chart shows the translation of the low order bit. +Other bits are the same. +========================= Low order bit ========================== + | REDUNDANT | COMPACT | COMPRESSED | DYNAMIC +dict_table_t::flags | 0 | 1 | 1 | 1 +fil_space_t::flags | 0 | 0 | 1 | 1 +================================================================== +@param[in] table_flags dict_table_t::flags +@return tablespace flags (fil_space_t::flags) */ UNIV_INLINE ulint -dict_tf_init( - bool compact, - ulint zip_ssize, - bool atomic_blobs, - bool data_dir, - bool shared_space, - bool page_compressed, - ulint page_compression_level, - ulint atomic_writes) +dict_tf_to_fsp_flags(ulint table_flags) { - ulint flags = 0; + ulint fsp_flags; + ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL( + table_flags); + ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); - if (compact) { - flags |= DICT_TF_COMPACT; - } + ut_ad((DICT_TF_GET_PAGE_COMPRESSION(table_flags) == 0) + == (page_compression_level == 0)); - if (zip_ssize) { - flags |= (zip_ssize << DICT_TF_POS_ZIP_SSIZE); - } + DBUG_EXECUTE_IF("dict_tf_to_fsp_flags_failure", + return(ULINT_UNDEFINED);); - if (atomic_blobs) { - flags |= (1 << DICT_TF_POS_ATOMIC_BLOBS); - } + /* Adjust bit zero. */ + fsp_flags = DICT_TF_HAS_ATOMIC_BLOBS(table_flags) ? 1 : 0; - if (data_dir) { - flags |= (1 << DICT_TF_POS_DATA_DIR); - } + /* ZIP_SSIZE and ATOMIC_BLOBS are at the same position. */ + fsp_flags |= table_flags + & (DICT_TF_MASK_ZIP_SSIZE | DICT_TF_MASK_ATOMIC_BLOBS); - if (shared_space) { - flags |= (1 << DICT_TF_POS_SHARED_SPACE); + fsp_flags |= FSP_FLAGS_PAGE_SSIZE(); + + if (page_compression_level) { + fsp_flags |= FSP_FLAGS_MASK_PAGE_COMPRESSION; } - if (page_compressed) { - flags |= (1 << DICT_TF_POS_ATOMIC_BLOBS) - | (1 << DICT_TF_POS_PAGE_COMPRESSION) - | (page_compression_level << DICT_TF_POS_PAGE_COMPRESSION_LEVEL); + ut_a(fsp_flags_is_valid(fsp_flags)); - ut_ad(zip_ssize == 0); - ut_ad(dict_tf_get_page_compression(flags) == TRUE); - ut_ad(dict_tf_get_page_compression_level(flags) == page_compression_level); + if (DICT_TF_HAS_DATA_DIR(table_flags)) { + fsp_flags |= 1U << FSP_FLAGS_MEM_DATA_DIR; } - if (atomic_writes) { - flags |= (atomic_writes << DICT_TF_POS_ATOMIC_WRITES); - ut_a(dict_tf_get_atomic_writes(flags) == atomic_writes); - } + fsp_flags |= atomic_writes << FSP_FLAGS_MEM_ATOMIC_WRITES; + fsp_flags |= page_compression_level << FSP_FLAGS_MEM_COMPRESSION_LEVEL; - return(flags); + return(fsp_flags); } /********************************************************************//** @@ -1123,14 +1044,12 @@ dict_sys_tables_type_to_tf( flags = redundant ? 0 : 1; /* ZIP_SSIZE, ATOMIC_BLOBS, DATA_DIR, PAGE_COMPRESSION, - PAGE_COMPRESSION_LEVEL, ATOMIC_WRITES are the same. */ + PAGE_COMPRESSION_LEVEL are the same. */ flags |= type & (DICT_TF_MASK_ZIP_SSIZE | DICT_TF_MASK_ATOMIC_BLOBS | DICT_TF_MASK_DATA_DIR | DICT_TF_MASK_PAGE_COMPRESSION - | DICT_TF_MASK_PAGE_COMPRESSION_LEVEL - | DICT_TF_MASK_ATOMIC_WRITES - | DICT_TF_MASK_SHARED_SPACE); + | DICT_TF_MASK_PAGE_COMPRESSION_LEVEL); ut_ad(!DICT_TF_GET_ZIP_SSIZE(flags) || DICT_TF_HAS_ATOMIC_BLOBS(flags)); return(flags); @@ -1160,14 +1079,12 @@ dict_tf_to_sys_tables_type( type = 1; /* ZIP_SSIZE, ATOMIC_BLOBS, DATA_DIR, PAGE_COMPRESSION, - PAGE_COMPRESSION_LEVEL, ATOMIC_WRITES are the same. */ + PAGE_COMPRESSION_LEVEL are the same. */ type |= flags & (DICT_TF_MASK_ZIP_SSIZE | DICT_TF_MASK_ATOMIC_BLOBS | DICT_TF_MASK_DATA_DIR | DICT_TF_MASK_PAGE_COMPRESSION - | DICT_TF_MASK_PAGE_COMPRESSION_LEVEL - | DICT_TF_MASK_ATOMIC_WRITES - | DICT_TF_MASK_SHARED_SPACE); + | DICT_TF_MASK_PAGE_COMPRESSION_LEVEL); return(type); } @@ -1208,7 +1125,6 @@ dict_table_page_size( return(dict_tf_get_page_size(table->flags)); } -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Obtain exclusive locks on all index trees of the table. This is to prevent accessing index trees while InnoDB is updating internal metadata for @@ -1251,7 +1167,6 @@ dict_table_x_unlock_indexes( rw_lock_x_unlock(dict_index_get_lock(index)); } } -#endif /* !UNIV_HOTBACKUP */ /********************************************************************//** Gets the number of fields in the internal representation of an index, @@ -1327,7 +1242,6 @@ ulint dict_index_get_n_unique_in_tree_nonleaf( const dict_index_t* index) { - ut_ad(index != NULL); ut_ad(index->magic_n == DICT_INDEX_MAGIC_N); ut_ad(index->cached); @@ -1457,7 +1371,6 @@ dict_index_get_nth_col_pos( prefix_col_pos)); } -#ifndef UNIV_HOTBACKUP /********************************************************************//** Returns the minimum data size of an index record. @return minimum data size in bytes */ @@ -1805,64 +1718,6 @@ dict_table_is_temporary( } /********************************************************************//** -Check if it is a encrypted table. -@return true if table encrypted flag is set. */ -UNIV_INLINE -bool -dict_table_is_encrypted( -/*====================*/ - const dict_table_t* table) /*!< in: table to check */ -{ - return(DICT_TF2_FLAG_IS_SET(table, DICT_TF2_ENCRYPTION)); -} - -/** Check whether the table is intrinsic. -An intrinsic table is a special kind of temporary table that -is invisible to the end user. It can be created internally by InnoDB, the MySQL -server layer or other modules connected to InnoDB in order to gather and use -data as part of a larger task. Since access to it must be as fast as possible, -it does not need UNDO semantics, system fields DB_TRX_ID & DB_ROLL_PTR, -doublewrite, checksum, insert buffer, use of the shared data dictionary, -locking, or even a transaction. In short, these are not ACID tables at all, -just temporary data stored and manipulated during a larger process. - -@param[in] table table to check -@return true if intrinsic table flag is set. */ -UNIV_INLINE -bool -dict_table_is_intrinsic( - const dict_table_t* table) -{ - return(DICT_TF2_FLAG_IS_SET(table, DICT_TF2_INTRINSIC)); -} - -/** Check if the table is in a shared tablespace (System or General). -@param[in] id Space ID to check -@return true if id is a shared tablespace, false if not. */ -UNIV_INLINE -bool -dict_table_in_shared_tablespace( - const dict_table_t* table) -{ - return(is_system_tablespace(table->space) - || DICT_TF_HAS_SHARED_SPACE(table->flags)); -} - -/** Check whether locking is disabled for this table. -Currently this is done for intrinsic table as their visibility is limited -to the connection only. - -@param[in] table table to check -@return true if locking is disabled. */ -UNIV_INLINE -bool -dict_table_is_locking_disabled( - const dict_table_t* table) -{ - return(dict_table_is_intrinsic(table)); -} - -/********************************************************************//** Turn-off redo-logging if temporary table. */ UNIV_INLINE void @@ -1884,8 +1739,8 @@ A patch was added to v3.23.52 which initializes the MIX_LEN field to 0. Since file-per-table tablespaces were added in 4.1, any SYS_TABLES record with a non-zero space ID will have a reliable MIX_LEN field. However, this test does not use flags2 from SYS_TABLES.MIX_LEN. Instead, -assume that if the tablespace is not a predefined system tablespace and it -is not a general shared tablespace, then it must be file-per-table. +assume that if the tablespace is not a predefined system tablespace, + then it must be file-per-table. Also, during ALTER TABLE, the DICT_TF2_USE_FILE_PER_TABLE flag may not be set on one of the file-per-table tablespaces. This test cannot be done on a table in the process of being created @@ -1898,8 +1753,7 @@ dict_table_is_file_per_table( const dict_table_t* table) /*!< in: table to check */ { bool is_file_per_table = - !is_system_tablespace(table->space) - && !DICT_TF_HAS_SHARED_SPACE(table->flags); + !is_system_tablespace(table->space); /* If the table is file-per-table and it is not redundant, then it should have the flags2 bit for DICT_TF2_USE_FILE_PER_TABLE. */ @@ -1907,76 +1761,7 @@ dict_table_is_file_per_table( || !DICT_TF_GET_COMPACT(table->flags) || DICT_TF2_FLAG_IS_SET(table, DICT_TF2_USE_FILE_PER_TABLE)); - return(is_file_per_table ); -} - -/**********************************************************************//** -Get index by first field of the index -@return index which is having first field matches -with the field present in field_index position of table */ -UNIV_INLINE -dict_index_t* -dict_table_get_index_on_first_col( -/*==============================*/ - const dict_table_t* table, /*!< in: table */ - ulint col_index, /*!< in: position of column - in table */ - const char* field_name) /*!< in: field name */ -{ - ut_ad(col_index < table->n_cols); - - dict_col_t* column = dict_table_get_nth_col(table, col_index); - - for (dict_index_t* index = dict_table_get_first_index(table); - index != NULL; index = dict_table_get_next_index(index)) { - - if (index->fields[0].col == column) { - return(index); - } - } - - /* If not yet found use field_name */ - for (dict_index_t* index = dict_table_get_first_index(table); - index != NULL; index = dict_table_get_next_index(index)) { - if (!strcmp(index->fields[0].name, field_name)) { - return (index); - } - } - ut_error; - return(0); -} - -/** Get table session row-id and increment the row-id counter for next use. -@param[in,out] table table handler -@return next table session row-id. */ -UNIV_INLINE -row_id_t -dict_table_get_next_table_sess_row_id( - dict_table_t* table) -{ - return(++table->sess_row_id); -} - -/** Get table session trx-id and increment the trx-id counter for next use. -@param[in,out] table table handler -@return next table session trx-id. */ -UNIV_INLINE -trx_id_t -dict_table_get_next_table_sess_trx_id( - dict_table_t* table) -{ - return(++table->sess_trx_id); -} - -/** Get current session trx-id. -@param[in] table table handler -@return table session trx-id. */ -UNIV_INLINE -trx_id_t -dict_table_get_curr_table_sess_trx_id( - const dict_table_t* table) -{ - return(table->sess_trx_id); + return(is_file_per_table); } /** Get reference count. @@ -1985,7 +1770,7 @@ inline ulint dict_table_t::get_ref_count() const { - ut_ad(mutex_own(&dict_sys->mutex) || dict_table_is_intrinsic(this)); + ut_ad(mutex_own(&dict_sys->mutex)); return(n_ref_count); } @@ -1994,7 +1779,7 @@ inline void dict_table_t::acquire() { - ut_ad(mutex_own(&dict_sys->mutex) || dict_table_is_intrinsic(this)); + ut_ad(mutex_own(&dict_sys->mutex)); ++n_ref_count; } @@ -2003,25 +1788,11 @@ inline void dict_table_t::release() { - ut_ad(mutex_own(&dict_sys->mutex) || dict_table_is_intrinsic(this)); + ut_ad(mutex_own(&dict_sys->mutex)); ut_ad(n_ref_count > 0); --n_ref_count; } -/** Check if tablespace name is "innodb_general". -@param[in] tablespace_name tablespace name -@retval true if name is "innodb_general" -@retval false if name is not "innodb_general" */ -inline -bool -dict_table_has_temp_general_tablespace_name( - const char* tablespace_name) { - - return(tablespace_name != NULL - && strncmp(tablespace_name, general_space_name, - strlen(general_space_name)) == 0); -} - /** Encode the number of columns and number of virtual columns in a 4 bytes value. We could do this because the number of columns in InnoDB is limited to 1017 @@ -2068,7 +1839,6 @@ dict_free_vc_templ( ut_free(vc_templ->vtempl[i]); } } - ut_free(vc_templ->default_rec); ut_free(vc_templ->vtempl); vc_templ->vtempl = NULL; } @@ -2094,5 +1864,3 @@ dict_table_have_virtual_index( return(false); } - -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/dict0load.h b/storage/innobase/include/dict0load.h index 6d01c38c432..0a862bb17a9 100644 --- a/storage/innobase/include/dict0load.h +++ b/storage/innobase/include/dict0load.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -201,15 +202,6 @@ dict_get_and_save_data_dir_path( dict_table_t* table, bool dict_mutex_own); -/** Make sure the tablespace name is saved in dict_table_t if needed. -Try to read it from the file dictionary first, then from SYS_TABLESPACES. -@param[in] table Table object -@param[in] dict_mutex_own) true if dict_sys->mutex is owned already */ -void -dict_get_and_save_space_name( - dict_table_t* table, - bool dict_mutex_own); - /** Loads a table definition and also all its index definitions, and also the cluster definition if the table is a member in a cluster. Also loads all foreign key constraints where the foreign key is in the table or where diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index 4ee19f96008..4a084c347d6 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -35,11 +35,9 @@ Created 1/8/1996 Heikki Tuuri #include "row0types.h" #include "rem0types.h" #include "btr0types.h" -#ifndef UNIV_HOTBACKUP -# include "lock0types.h" -# include "que0types.h" -# include "sync0rw.h" -#endif /* !UNIV_HOTBACKUP */ +#include "lock0types.h" +#include "que0types.h" +#include "sync0rw.h" #include "ut0mem.h" #include "ut0rnd.h" #include "ut0byte.h" @@ -50,7 +48,6 @@ Created 1/8/1996 Heikki Tuuri #include "gis0type.h" #include "os0once.h" #include "ut0new.h" - #include "fil0fil.h" #include <my_crypt.h> #include "fil0crypt.h" @@ -140,12 +137,8 @@ This flag prevents older engines from attempting to open the table and allows InnoDB to update_create_info() accordingly. */ #define DICT_TF_WIDTH_DATA_DIR 1 -/** Width of the SHARED tablespace flag. -It is used to identify tables that exist inside a shared general tablespace. -If a table is created with the TABLESPACE=tsname option, an older engine will -not be able to find that table. This flag prevents older engines from attempting -to open the table and allows InnoDB to quickly find the tablespace. */ - +/** Width of the SHARED tablespace flag (Oracle MYSQL 5.7). +Not supported by MariaDB. */ #define DICT_TF_WIDTH_SHARED_SPACE 1 /** @@ -229,10 +222,6 @@ DEFAULT=0, ON = 1, OFF = 2 #define DICT_TF_MASK_DATA_DIR \ ((~(~0U << DICT_TF_WIDTH_DATA_DIR)) \ << DICT_TF_POS_DATA_DIR) -/** Bit mask of the SHARED_SPACE field */ -#define DICT_TF_MASK_SHARED_SPACE \ - ((~(~0U << DICT_TF_WIDTH_SHARED_SPACE)) \ - << DICT_TF_POS_SHARED_SPACE) /** Bit mask of the PAGE_COMPRESSION field */ #define DICT_TF_MASK_PAGE_COMPRESSION \ ((~(~0U << DICT_TF_WIDTH_PAGE_COMPRESSION)) \ @@ -270,10 +259,6 @@ DEFAULT=0, ON = 1, OFF = 2 #define DICT_TF_HAS_DATA_DIR(flags) \ ((flags & DICT_TF_MASK_DATA_DIR) \ >> DICT_TF_POS_DATA_DIR) -/** Return the value of the SHARED_SPACE field */ -#define DICT_TF_HAS_SHARED_SPACE(flags) \ - ((flags & DICT_TF_MASK_SHARED_SPACE) \ - >> DICT_TF_POS_SHARED_SPACE) /** Return the value of the PAGE_COMPRESSION field */ #define DICT_TF_GET_PAGE_COMPRESSION(flags) \ ((flags & DICT_TF_MASK_PAGE_COMPRESSION) \ @@ -310,7 +295,8 @@ for unknown bits in order to protect backward incompatibility. */ /* @{ */ /** Total number of bits in table->flags2. */ #define DICT_TF2_BITS 9 -#define DICT_TF2_UNUSED_BIT_MASK (~0U << DICT_TF2_BITS) +#define DICT_TF2_UNUSED_BIT_MASK (~0U << DICT_TF2_BITS | \ + 1U << DICT_TF_POS_SHARED_SPACE) #define DICT_TF2_BIT_MASK ~DICT_TF2_UNUSED_BIT_MASK /** TEMPORARY; TRUE for tables from CREATE TEMPORARY TABLE. */ @@ -337,15 +323,6 @@ use its own tablespace instead of the system tablespace. */ index tables) of a FTS table are in HEX format. */ #define DICT_TF2_FTS_AUX_HEX_NAME 64 -/** Intrinsic table bit -Intrinsic table is table created internally by MySQL modules viz. Optimizer, -FTS, etc.... Intrinsic table has all the properties of the normal table except -it is not created by user and so not visible to end-user. */ -#define DICT_TF2_INTRINSIC 128 - -/** Encryption table bit. */ -#define DICT_TF2_ENCRYPTION 256 - /* @} */ #define DICT_TF2_FLAG_SET(table, flag) \ @@ -859,95 +836,6 @@ struct zip_pad_info_t { a certain index.*/ #define STAT_DEFRAG_DATA_SIZE_N_SAMPLE 10 -/** If key is fixed length key then cache the record offsets on first -computation. This will help save computation cycle that generate same -redundant data. */ -class rec_cache_t -{ -public: - /** Constructor */ - rec_cache_t() - : - rec_size(), - offsets(), - sz_of_offsets(), - fixed_len_key(), - offsets_cached(), - key_has_null_cols() - { - /* Do Nothing. */ - } - -public: - /** Record size. (for fixed length key record size is constant) */ - ulint rec_size; - - /** Holds reference to cached offsets for record. */ - ulint* offsets; - - /** Size of offset array */ - uint32_t sz_of_offsets; - - /** If true, then key is fixed length key. */ - bool fixed_len_key; - - /** If true, then offset has been cached for re-use. */ - bool offsets_cached; - - /** If true, then key part can have columns that can take - NULL values. */ - bool key_has_null_cols; -}; - -/** Cache position of last inserted or selected record by caching record -and holding reference to the block where record resides. -Note: We don't commit mtr and hold it beyond a transaction lifetime as this is -a special case (intrinsic table) that are not shared accross connection. */ -class last_ops_cur_t -{ -public: - /** Constructor */ - last_ops_cur_t() - : - rec(), - block(), - mtr(), - disable_caching(), - invalid() - { - /* Do Nothing. */ - } - - /* Commit mtr and re-initialize cache record and block to NULL. */ - void release() - { - if (mtr.is_active()) { - mtr_commit(&mtr); - } - rec = NULL; - block = NULL; - invalid = false; - } - -public: - /** last inserted/selected record. */ - rec_t* rec; - - /** block where record reside. */ - buf_block_t* block; - - /** active mtr that will be re-used for next insert/select. */ - mtr_t mtr; - - /** disable caching. (disabled when table involves blob/text.) */ - bool disable_caching; - - /** If index structure is undergoing structural change viz. - split then invalidate the cached position as it would be no more - remain valid. Will be re-cached on post-split insert. */ - bool invalid; -}; - /** "GEN_CLUST_INDEX" is the name reserved for InnoDB default system clustered index when there is no primary key. */ const char innobase_index_reserve_name[] = "GEN_CLUST_INDEX"; @@ -964,7 +852,6 @@ struct dict_index_t{ id_name_t name; /*!< index name */ const char* table_name;/*!< table name */ dict_table_t* table; /*!< back pointer to table */ -#ifndef UNIV_HOTBACKUP unsigned space:32; /*!< space where the index tree is placed */ unsigned page:32;/*!< index tree root page number */ @@ -973,7 +860,6 @@ struct dict_index_t{ data size drops below this limit in percent, merging it to a neighbor is tried */ # define DICT_INDEX_MERGE_THRESHOLD_DEFAULT 50 -#endif /* !UNIV_HOTBACKUP */ unsigned type:DICT_IT_BITS; /*!< index type (DICT_CLUSTERED, DICT_UNIQUE, DICT_IBUF, DICT_CORRUPT) */ @@ -990,18 +876,15 @@ struct dict_index_t{ /*!< number of columns the user defined to be in the index: in the internal representation we add more columns */ - unsigned allow_duplicates:1; - /*!< if true, allow duplicate values - even if index is created with unique - constraint */ unsigned nulls_equal:1; /*!< if true, SQL NULL == SQL NULL */ - unsigned disable_ahi:1; - /*!< in true, then disable AHI. - Currently limited to intrinsic - temporary table as index id is not - unqiue for such table which is one of the - validation criterion for ahi. */ +#ifdef MYSQL_INDEX_DISABLE_AHI + unsigned disable_ahi:1; + /*!< whether to disable the + adaptive hash index. + Maybe this could be disabled for + temporary tables? */ +#endif unsigned n_uniq:10;/*!< number of fields from the beginning which are enough to determine an index entry uniquely */ @@ -1038,7 +921,6 @@ struct dict_index_t{ bool has_new_v_col; /*!< whether it has a newly added virtual column in ALTER */ -#ifndef UNIV_HOTBACKUP UT_LIST_NODE_T(dict_index_t) indexes;/*!< list of indexes of the table */ btr_search_t* search_info; @@ -1098,19 +980,6 @@ struct dict_index_t{ /* in which slot the next sample should be saved. */ /* @} */ - last_ops_cur_t* last_ins_cur; - /*!< cache the last insert position. - Currently limited to auto-generated - clustered index on intrinsic table only. */ - last_ops_cur_t* last_sel_cur; - /*!< cache the last selected position - Currently limited to intrinsic table only. */ - rec_cache_t rec_cache; - /*!< cache the field that needs to be - re-computed on each insert. - Limited to intrinsic table as this is common - share and can't be used without protection - if table is accessible to multiple-threads. */ rtr_ssn_t rtr_ssn;/*!< Node sequence number for RTree */ rtr_info_track_t* rtr_track;/*!< tracking all R-Tree search cursors */ @@ -1139,7 +1008,6 @@ struct dict_index_t{ ut_ad(committed || !(type & DICT_CLUSTERED)); uncommitted = !committed; } -#endif /* !UNIV_HOTBACKUP */ }; /** The status of online index creation */ @@ -1439,20 +1307,20 @@ struct dict_vcol_templ_t { /** table name */ std::string tb_name; - /** share->table_name */ - std::string share_name; - /** MySQL record length */ ulint rec_len; /** default column value if any */ byte* default_rec; -}; -/* This flag is for sync SQL DDL and memcached DML. -if table->memcached_sync_count == DICT_TABLE_IN_DDL means there's DDL running on -the table, DML from memcached will be blocked. */ -#define DICT_TABLE_IN_DDL -1 + /** cached MySQL TABLE object */ + TABLE* mysql_table; + + /** when mysql_table was cached */ + uint64_t mysql_table_query_id; + + dict_vcol_templ_t() : vtempl(0), mysql_table_query_id(-1) {} +}; /** These are used when MySQL FRM and InnoDB data dictionary are in inconsistent state. */ @@ -1478,6 +1346,8 @@ struct dict_table_t { inline void acquire(); void* thd; /*!< thd */ + bool page_0_read; /*!< true if page 0 has + been already read */ fil_space_crypt_t *crypt_data; /*!< crypt data if present */ /** Release the table handle. */ @@ -1500,19 +1370,9 @@ struct dict_table_t { /** Table name. */ table_name_t name; - /** NULL or the directory path where a TEMPORARY table that was - explicitly created by a user should be placed if innodb_file_per_table - is defined in my.cnf. In Unix this is usually "/tmp/...", - in Windows "temp\...". */ - const char* dir_path_of_temp_table; - /** NULL or the directory path specified by DATA DIRECTORY. */ char* data_dir_path; - /** NULL or the tablespace name that this table is assigned to, - specified by the TABLESPACE option.*/ - id_name_t tablespace; - /** Space where the clustered index of the table is placed. */ uint32_t space; @@ -1572,6 +1432,11 @@ struct dict_table_t { /** Number of virtual columns. */ unsigned n_v_cols:10; + /** 1 + the position of autoinc counter field in clustered + index, or 0 if there is no persistent AUTO_INCREMENT column in + the table. */ + unsigned persistent_autoinc:10; + /** TRUE if it's not an InnoDB system table or a table that has no FK relationships. */ unsigned can_be_evicted:1; @@ -1613,7 +1478,6 @@ struct dict_table_t { /*!< !DICT_FRM_CONSISTENT==0 if data dictionary information and MySQL FRM information mismatch. */ -#ifndef UNIV_HOTBACKUP /** Hash chain node. */ hash_node_t name_hash; @@ -1840,12 +1704,6 @@ struct dict_table_t { /* @} */ - /** Count of how many handles are opened to this table from memcached. - DDL on the table is NOT allowed until this count goes to zero. If - it is -1, then there's DDL on the table, DML from memcached will be - blocked. */ - lint memcached_sync_count; - /** FTS specific state variables. */ fts_t* fts; @@ -1874,20 +1732,6 @@ public: /** Timestamp of the last modification of this table. */ time_t update_time; - /** row-id counter for use by intrinsic table for getting row-id. - Given intrinsic table semantics, row-id can be locally maintained - instead of getting it from central generator which involves mutex - locking. */ - ib_uint64_t sess_row_id; - - /** trx_id counter for use by intrinsic table for getting trx-id. - Intrinsic table are not shared so don't need a central trx-id - but just need a increased counter to track consistent view while - proceeding SELECT as part of UPDATE. */ - ib_uint64_t sess_trx_id; - -#endif /* !UNIV_HOTBACKUP */ - bool is_encrypted; #ifdef UNIV_DEBUG @@ -1900,12 +1744,6 @@ public: /** mysql_row_templ_t for base columns used for compute the virtual columns */ dict_vcol_templ_t* vc_templ; - - /** encryption key, it's only for export/import */ - byte* encryption_key; - - /** encryption iv, it's only for export/import */ - byte* encryption_iv; }; /*******************************************************************//** diff --git a/storage/innobase/include/dict0mem.ic b/storage/innobase/include/dict0mem.ic index 3269596feb7..933f233aae6 100644 --- a/storage/innobase/include/dict0mem.ic +++ b/storage/innobase/include/dict0mem.ic @@ -60,44 +60,17 @@ dict_mem_fill_index_struct( /* Assign a ulint to a 4-bit-mapped field. Only the low-order 4 bits are assigned. */ index->type = type; -#ifndef UNIV_HOTBACKUP index->space = (unsigned int) space; index->page = FIL_NULL; index->merge_threshold = DICT_INDEX_MERGE_THRESHOLD_DEFAULT; -#endif /* !UNIV_HOTBACKUP */ index->table_name = table_name; index->n_fields = (unsigned int) n_fields; /* The '1 +' above prevents allocation of an empty mem block */ - index->allow_duplicates = false; index->nulls_equal = false; +#ifdef MYSQL_INDEX_DISABLE_AHI index->disable_ahi = false; - - new (&index->rec_cache) rec_cache_t(); - - if (heap != NULL) { - index->last_ins_cur = - static_cast<last_ops_cur_t*>(mem_heap_alloc( - heap, sizeof(last_ops_cur_t))); - - new (index->last_ins_cur) last_ops_cur_t(); - - index->last_sel_cur = - static_cast<last_ops_cur_t*>(mem_heap_alloc( - heap, sizeof(last_ops_cur_t))); - - new (index->last_sel_cur) last_ops_cur_t(); - - index->rec_cache.offsets = - static_cast<ulint*>(mem_heap_alloc( - heap, sizeof(ulint) * OFFS_IN_REC_NORMAL_SIZE)); - - index->rec_cache.sz_of_offsets = OFFS_IN_REC_NORMAL_SIZE; - } else { - index->last_ins_cur = NULL; - index->last_sel_cur = NULL; - index->rec_cache.offsets = NULL; - } +#endif #ifdef UNIV_DEBUG index->magic_n = DICT_INDEX_MAGIC_N; diff --git a/storage/innobase/include/dict0pagecompress.h b/storage/innobase/include/dict0pagecompress.h index 19a2a6c52f3..6641f6ba85f 100644 --- a/storage/innobase/include/dict0pagecompress.h +++ b/storage/innobase/include/dict0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013 SkySQL Ab. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -56,37 +56,6 @@ dict_table_page_compression_level( const dict_table_t* table) /*!< in: table */ __attribute__((const)); -/********************************************************************//** -Verify that dictionary flags match tablespace flags -@return true if flags match, false if not */ -UNIV_INLINE -ibool -dict_tf_verify_flags( -/*=================*/ - ulint table_flags, /*!< in: dict_table_t::flags */ - ulint fsp_flags) /*!< in: fil_space_t::flags */ - __attribute__((const)); - -/********************************************************************//** -Extract the atomic writes flag from table flags. -@return true if atomic writes are used, false if not used */ -UNIV_INLINE -atomic_writes_t -dict_tf_get_atomic_writes( -/*======================*/ - ulint flags) /*!< in: flags */ - __attribute__((const)); - -/********************************************************************//** -Check whether the table uses the atomic writes. -@return true if atomic writes is used, false if not */ -UNIV_INLINE -atomic_writes_t -dict_table_get_atomic_writes( -/*=========================*/ - const dict_table_t* table); /*!< in: table */ - - #ifndef UNIV_NONINL #include "dict0pagecompress.ic" #endif diff --git a/storage/innobase/include/dict0pagecompress.ic b/storage/innobase/include/dict0pagecompress.ic index 811976434a8..67f4a26723f 100644 --- a/storage/innobase/include/dict0pagecompress.ic +++ b/storage/innobase/include/dict0pagecompress.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013 SkySQL Ab. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -25,92 +25,6 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com ***********************************************************************/ /********************************************************************//** -Verify that dictionary flags match tablespace flags -@return true if flags match, false if not */ -UNIV_INLINE -ibool -dict_tf_verify_flags( -/*=================*/ - ulint table_flags, /*!< in: dict_table_t::flags */ - ulint fsp_flags) /*!< in: fil_space_t::flags */ -{ - ulint table_unused = DICT_TF_GET_UNUSED(table_flags); - ulint compact = DICT_TF_GET_COMPACT(table_flags); - ulint ssize = DICT_TF_GET_ZIP_SSIZE(table_flags); - ulint atomic_blobs = DICT_TF_HAS_ATOMIC_BLOBS(table_flags); - ulint data_dir = DICT_TF_HAS_DATA_DIR(table_flags); - ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); - ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); - ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); - ulint post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(fsp_flags); - ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(fsp_flags); - ulint fsp_atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(fsp_flags); - ulint page_ssize = FSP_FLAGS_GET_PAGE_SSIZE(fsp_flags); - ulint fsp_unused = FSP_FLAGS_GET_UNUSED(fsp_flags); - ulint fsp_page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(fsp_flags); - ulint fsp_page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(fsp_flags); - ulint fsp_atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(fsp_flags); - - DBUG_EXECUTE_IF("dict_tf_verify_flags_failure", - return(ULINT_UNDEFINED);); - - ut_a(!table_unused); - ut_a(!fsp_unused); - ut_a(page_ssize == 0 || page_ssize != 0); /* silence compiler */ - ut_a(compact == 0 || compact == 1); /* silence compiler */ - ut_a(data_dir == 0 || data_dir == 1); /* silence compiler */ - ut_a(post_antelope == 0 || post_antelope == 1); /* silence compiler */ - - if (ssize != zip_ssize) { - fprintf(stderr, - "InnoDB: Error: table flags has zip_ssize %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has zip_ssize %ld\n", - ssize, zip_ssize); - return (FALSE); - } - if (atomic_blobs != fsp_atomic_blobs) { - fprintf(stderr, - "InnoDB: Error: table flags has atomic_blobs %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has atomic_blobs %ld\n", - atomic_blobs, fsp_atomic_blobs); - - return (FALSE); - } - if (page_compression != fsp_page_compression) { - fprintf(stderr, - "InnoDB: Error: table flags has page_compression %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file ahas page_compression %ld\n", - page_compression, fsp_page_compression); - - return (FALSE); - } - if (page_compression_level != fsp_page_compression_level) { - fprintf(stderr, - "InnoDB: Error: table flags has page_compression_level %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has page_compression_level %ld\n", - page_compression_level, fsp_page_compression_level); - - return (FALSE); - } - - if (atomic_writes != fsp_atomic_writes) { - fprintf(stderr, - "InnoDB: Error: table flags has atomic writes %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has atomic_writes %ld\n", - atomic_writes, fsp_atomic_writes); - - return (FALSE); - } - - return(TRUE); -} - -/********************************************************************//** Extract the page compression level from dict_table_t::flags. These flags are in memory, so assert that they are valid. @return page compression level, or 0 if not compressed */ @@ -165,27 +79,3 @@ dict_table_is_page_compressed( { return (dict_tf_get_page_compression(table->flags)); } - -/********************************************************************//** -Extract the atomic writes flag from table flags. -@return enumerated value of atomic writes */ -UNIV_INLINE -atomic_writes_t -dict_tf_get_atomic_writes( -/*======================*/ - ulint flags) /*!< in: flags */ -{ - return((atomic_writes_t)DICT_TF_GET_ATOMIC_WRITES(flags)); -} - -/********************************************************************//** -Check whether the table uses the atomic writes. -@return enumerated value of atomic writes */ -UNIV_INLINE -atomic_writes_t -dict_table_get_atomic_writes( -/*=========================*/ - const dict_table_t* table) /*!< in: table */ -{ - return ((atomic_writes_t)dict_tf_get_atomic_writes(table->flags)); -} diff --git a/storage/innobase/include/dict0priv.ic b/storage/innobase/include/dict0priv.ic index fd10c566be6..fb7af2772fc 100644 --- a/storage/innobase/include/dict0priv.ic +++ b/storage/innobase/include/dict0priv.ic @@ -26,7 +26,6 @@ Created Wed 13 Oct 2010 16:10:14 EST Sunny Bains #include "dict0dict.h" #include "dict0load.h" #include "dict0priv.h" -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Gets a table; loads it to the dictionary cache if necessary. A low-level @@ -125,4 +124,3 @@ dict_table_check_if_in_cache_low( !strcmp(table->name.m_name, table_name)); DBUG_RETURN(table); } -#endif /*! UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/dict0stats.ic b/storage/innobase/include/dict0stats.ic index 80709091734..61c88773912 100644 --- a/storage/innobase/include/dict0stats.ic +++ b/storage/innobase/include/dict0stats.ic @@ -182,7 +182,7 @@ dict_stats_deinit( /*==============*/ dict_table_t* table) /*!< in/out: table */ { - ut_ad(mutex_own(&dict_sys->mutex) || dict_table_is_intrinsic(table)); + ut_ad(mutex_own(&dict_sys->mutex)); ut_a(table->get_ref_count() == 0); diff --git a/storage/innobase/include/dict0stats_bg.h b/storage/innobase/include/dict0stats_bg.h index 50c2591332e..e09716e3842 100644 --- a/storage/innobase/include/dict0stats_bg.h +++ b/storage/innobase/include/dict0stats_bg.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -32,7 +33,8 @@ Created Apr 26, 2012 Vasil Dimov #include "os0event.h" #include "os0thread.h" -/** Event to wake up the stats thread */ +/** Event to wake up dict_stats_thread on dict_stats_recalc_pool_add() +or shutdown. Not protected by any mutex. */ extern os_event_t dict_stats_event; #ifdef HAVE_PSI_INTERFACE diff --git a/storage/innobase/include/dict0types.h b/storage/innobase/include/dict0types.h index ae002dd9487..b0623f82bb7 100644 --- a/storage/innobase/include/dict0types.h +++ b/storage/innobase/include/dict0types.h @@ -80,13 +80,6 @@ enum ib_quiesce_t { QUIESCE_COMPLETE /*!< All done */ }; -/** Enum values for atomic_writes table option */ -typedef enum { - ATOMIC_WRITES_DEFAULT = 0, - ATOMIC_WRITES_ON = 1, - ATOMIC_WRITES_OFF = 2 -} atomic_writes_t; - #ifndef UNIV_INNOCHECKSUM typedef ib_mutex_t DictSysMutex; #endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index acac155ef3f..7185857c039 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -1,6 +1,6 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (c) 2015, 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -26,6 +26,9 @@ Created 04/01/2015 Jan Lindström #ifndef fil0crypt_h #define fil0crypt_h +#include "os0event.h" +#include "my_crypt.h" + /** * Magic pattern in start of crypt data on page 0 */ @@ -34,9 +37,6 @@ Created 04/01/2015 Jan Lindström static const unsigned char CRYPT_MAGIC[MAGIC_SZ] = { 's', 0xE, 0xC, 'R', 'E', 't' }; -static const unsigned char EMPTY_PATTERN[MAGIC_SZ] = { - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0 }; - /* This key will be used if nothing else is given */ #define FIL_DEFAULT_ENCRYPTION_KEY ENCRYPTION_KEY_SYSTEM_DATA @@ -48,6 +48,8 @@ typedef enum { FIL_SPACE_ENCRYPTION_OFF = 2 /* Tablespace is not encrypted */ } fil_encryption_t; +extern os_event_t fil_crypt_threads_event; + /** * CRYPT_SCHEME_UNENCRYPTED * @@ -75,6 +77,17 @@ struct key_struct (that is L in CRYPT_SCHEME_1) */ }; +/** is encryption enabled */ +extern ulong srv_encrypt_tables; + +/** Mutex helper for crypt_data->scheme +@param[in, out] schme encryption scheme +@param[in] exit should we exit or enter mutex ? */ +void +crypt_data_scheme_locker( + st_encryption_scheme* scheme, + int exit); + struct fil_space_rotate_state_t { time_t start_time; /*!< time when rotation started */ @@ -96,13 +109,109 @@ struct fil_space_rotate_state_t struct fil_space_crypt_struct : st_encryption_scheme { + public: + /** Constructor. Does not initialize the members! + The object is expected to be placed in a buffer that + has been zero-initialized. */ + fil_space_crypt_struct( + ulint new_type, + uint new_min_key_version, + uint new_key_id, + ulint offset, + fil_encryption_t new_encryption) + : st_encryption_scheme(), + min_key_version(new_min_key_version), + page0_offset(offset), + encryption(new_encryption), + closing(false), + key_found(), + rotate_state() + { + key_found = new_min_key_version; + key_id = new_key_id; + my_random_bytes(iv, sizeof(iv)); + mutex_create(LATCH_ID_FIL_CRYPT_DATA_MUTEX, &mutex); + locker = crypt_data_scheme_locker; + type = new_type; + + if (new_encryption == FIL_SPACE_ENCRYPTION_OFF || + (!srv_encrypt_tables && + new_encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { + type = CRYPT_SCHEME_UNENCRYPTED; + } else { + type = CRYPT_SCHEME_1; + min_key_version = key_get_latest_version(); + } + } + + /** Destructor */ + ~fil_space_crypt_struct() + { + closing = true; + mutex_free(&mutex); + } + + /** Get latest key version from encryption plugin + @retval key_version or + @retval ENCRYPTION_KEY_VERSION_INVALID if used key_id + is not found from encryption plugin. */ + uint key_get_latest_version(void); + + /** Returns true if key was found from encryption plugin + and false if not. */ + bool is_key_found() const { + return key_found != ENCRYPTION_KEY_VERSION_INVALID; + } + + /** Returns true if tablespace should be encrypted */ + bool should_encrypt() const { + return ((encryption == FIL_SPACE_ENCRYPTION_ON) || + (srv_encrypt_tables && + encryption == FIL_SPACE_ENCRYPTION_DEFAULT)); + } + + /** Return true if tablespace is encrypted. */ + bool is_encrypted() const { + return (encryption != FIL_SPACE_ENCRYPTION_OFF); + } + + /** Return true if default tablespace encryption is used, */ + bool is_default_encryption() const { + return (encryption == FIL_SPACE_ENCRYPTION_DEFAULT); + } + + /** Return true if tablespace is not encrypted. */ + bool not_encrypted() const { + return (encryption == FIL_SPACE_ENCRYPTION_OFF); + } + + /** Is this tablespace closing. */ + bool is_closing(bool is_fixed) { + bool closed; + if (!is_fixed) { + mutex_enter(&mutex); + } + closed = closing; + if (!is_fixed) { + mutex_exit(&mutex); + } + return closed; + } + uint min_key_version; // min key version for this space ulint page0_offset; // byte offset on page 0 for crypt data fil_encryption_t encryption; // Encryption setup ib_mutex_t mutex; // mutex protecting following variables bool closing; // is tablespace being closed - bool inited; + + /** Return code from encryption_key_get_latest_version. + If ENCRYPTION_KEY_VERSION_INVALID encryption plugin + could not find the key and there is no need to call + get_latest_key_version again as keys are read only + at startup. */ + uint key_found; + fil_space_rotate_state_t rotate_state; }; @@ -288,12 +397,6 @@ fil_crypt_set_thread_cnt( uint new_cnt); /*!< in: requested #threads */ /********************************************************************* -End threads for key rotation */ -UNIV_INTERN -void -fil_crypt_threads_end(); - -/********************************************************************* Cleanup resources for threads for key rotation */ UNIV_INTERN void @@ -321,7 +424,8 @@ UNIV_INTERN void fil_space_crypt_mark_space_closing( /*===============================*/ - ulint space); /*!< in: tablespace id */ + ulint space, /*!< in: tablespace id */ + fil_space_crypt_t* crypt_data); /*!< in: crypt_data or NULL */ /********************************************************************* Wait for crypt threads to stop accessing space */ diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index bc1121d1530..36fb5e51f1e 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -33,73 +33,15 @@ Created 10/25/1995 Heikki Tuuri #include "log0recv.h" #include "dict0types.h" #include "page0size.h" -#ifndef UNIV_HOTBACKUP #include "ibuf0types.h" -#else -#include "log0log.h" -#include "os0file.h" -#include "m_string.h" -#endif /* !UNIV_HOTBACKUP */ #include <list> #include <vector> -#ifdef UNIV_HOTBACKUP -#include <cstring> -/** determine if file is intermediate / temporary.These files are created during -reorganize partition, rename tables, add / drop columns etc. -@param[in] filepath asbosolute / relative or simply file name -@retvalue true if it is intermediate file -@retvalue false if it is normal file */ -inline -bool -is_intermediate_file(const std::string& filepath) -{ - std::string file_name = filepath; - - // extract file name from relative or absolute file name - std::size_t pos = file_name.rfind(OS_PATH_SEPARATOR); - if (pos != std::string::npos) - file_name = file_name.substr(++pos); - - transform(file_name.begin(), file_name.end(), - file_name.begin(), ::tolower); - - if (file_name[0] != '#') { - pos = file_name.rfind("#tmp#.ibd"); - if (pos != std::string::npos) - return true; - else - return false; /* normal file name */ - } - - std::vector<std::string> file_name_patterns = {"#sql-", "#sql2-", - "#tmp#", "#ren#"}; - - /* search for the unsupported patterns */ - for (auto itr = file_name_patterns.begin(); - itr != file_name_patterns.end(); - itr++) { - - if (0 == std::strncmp(file_name.c_str(), - itr->c_str(), itr->length())){ - return true; - } - } - - return false; -} -#endif /* UNIV_HOTBACKUP */ - -extern const char general_space_name[]; - // Forward declaration struct trx_t; class page_id_t; class truncate_t; -struct fil_node_t; -struct fil_space_t; -struct btr_create_t; /* structure containing encryption specification */ typedef struct fil_space_crypt_struct fil_space_crypt_t; @@ -184,7 +126,12 @@ struct fil_space_t { /*!< length of the FSP_FREE list */ ulint free_limit; /*!< contents of FSP_FREE_LIMIT */ - ulint flags; /*!< tablespace flags; see + ulint recv_size; + /*!< recovered tablespace size in pages; + 0 if no size change was read from the redo log, + or if the size change was implemented */ + ulint flags; /*!< FSP_SPACE_FLAGS and FSP_FLAGS_MEM_ flags; + see fsp0types.h, fsp_flags_is_valid(), page_size_t(ulint) (constructor) */ ulint n_reserved_extents; @@ -203,10 +150,8 @@ struct fil_space_t { Protected by fil_system->mutex. */ hash_node_t hash; /*!< hash chain node */ hash_node_t name_hash;/*!< hash chain the name_hash table */ -#ifndef UNIV_HOTBACKUP rw_lock_t latch; /*!< latch protecting the file space storage allocation */ -#endif /* !UNIV_HOTBACKUP */ UT_LIST_NODE_T(fil_space_t) unflushed_spaces; /*!< list of spaces with at least one unflushed file we have written to */ @@ -219,26 +164,11 @@ struct fil_space_t { UT_LIST_NODE_T(fil_space_t) space_list; /*!< list of all spaces */ - /** Compression algorithm */ - Compression::Type compression_type; - - /** Encryption algorithm */ - Encryption::Type encryption_type; - - /** Encrypt key */ - byte encryption_key[ENCRYPTION_KEY_LEN]; - - /** Encrypt key length*/ - ulint encryption_klen; - - /** Encrypt initial vector */ - byte encryption_iv[ENCRYPTION_KEY_LEN]; - /** MariaDB encryption data */ fil_space_crypt_t* crypt_data; - /** Space file block size */ - ulint file_block_size; + /** tablespace crypt data has been read */ + bool page_0_crypt_read; /** True if we have already printed compression failure */ bool printed_compression_failure; @@ -246,10 +176,19 @@ struct fil_space_t { /** True if page 0 of tablespace is read */ bool read_page0; + /** True if we have tested if this filespace supports atomic writes */ + bool atomic_write_tested; + /** True if the device this filespace is on supports atomic writes */ + bool atomic_write_supported; + /** Release the reserved free extents. @param[in] n_reserved number of reserved extents */ void release_free_extents(ulint n_reserved); + /** True if file system storing this tablespace supports + punch hole */ + bool punch_hole; + ulint magic_n;/*!< FIL_SPACE_MAGIC_N */ }; @@ -262,11 +201,11 @@ struct fil_node_t { fil_space_t* space; /** file name; protected by fil_system->mutex and log_sys->mutex. */ char* name; - /** whether this file is open */ - bool is_open; /** file handle (valid if is_open) */ os_file_t handle; - /** event that groups and serializes calls to fsync */ + /** event that groups and serializes calls to fsync; + os_event_set() and os_event_reset() are protected by + fil_system_t::mutex */ os_event_t sync_event; /** whether the file actually is a raw device or disk partition */ bool is_raw_disk; @@ -294,17 +233,20 @@ struct fil_node_t { /** link to the fil_system->LRU list (keeping track of open files) */ UT_LIST_NODE_T(fil_node_t) LRU; - /** whether the file system of this file supports PUNCH HOLE */ - bool punch_hole; + /** whether this file could use atomic write (data file) */ + bool atomic_write; - /** block size to use for punching holes */ + /** Filesystem block size */ ulint block_size; - /** whether atomic write is enabled for this file */ - bool atomic_write; - /** FIL_NODE_MAGIC_N */ ulint magic_n; + + /** @return whether this file is open */ + bool is_open() const + { + return(handle != OS_FILE_CLOSED); + } }; /** Value of fil_node_t::magic_n */ @@ -315,110 +257,17 @@ enum ib_extention { NO_EXT = 0, IBD = 1, ISL = 2, - CFG = 3, - CFP = 4 + CFG = 3 }; extern const char* dot_ext[]; #define DOT_IBD dot_ext[IBD] #define DOT_ISL dot_ext[ISL] #define DOT_CFG dot_ext[CFG] -#define DOT_CPF dot_ext[CFP] - -/** Wrapper for a path to a directory. -This folder may or may not yet esist. Since not all directory paths -end in "/", we should only use this for a directory path or a filepath -that has a ".ibd" extension. */ -class Folder -{ -public: - /** Default constructor */ - Folder() : m_folder(NULL) {} - - /** Constructor - @param[in] path pathname (not necessarily NUL-terminated) - @param[in] len length of the path, in bytes */ - Folder(const char* path, size_t len); - - /** Assignment operator - @param[in] folder folder string provided */ - class Folder& operator=(const char* path); - - /** Destructor */ - ~Folder() - { - ut_free(m_folder); - } - - /** Implicit type conversion - @return the wrapped object */ - operator const char*() const - { - return(m_folder); - } - - /** Explicit type conversion - @return the wrapped object */ - const char* operator()() const - { - return(m_folder); - } - - /** return the length of m_folder - @return the length of m_folder */ - size_t len() - { - return m_folder_len; - } - - /** Determine if two folders are equal - @param[in] other folder to compare to - @return whether the folders are equal */ - bool operator==(const Folder& other) const; - - /** Determine if the left folder is the same or an ancestor of - (contains) the right folder. - @param[in] other folder to compare to - @return whether this is the same or an ancestor or the other folder. */ - bool operator>=(const Folder& other) const; - - /** Determine if the left folder is an ancestor of (contains) - the right folder. - @param[in] other folder to compare to - @return whether this is an ancestor of the other folder */ - bool operator>(const Folder& other) const; - - /** Determine if the directory referenced by m_folder exists. - @return whether the directory exists */ - bool exists(); - -private: - /** Build the basic folder name from the path and length provided - @param[in] path pathname (not necessarily NUL-terminated) - @param[in] len length of the path, in bytes */ - void make_path(const char* path, size_t len); - - /** Resolve a relative path in m_folder to an absolute path - in m_abs_path setting m_abs_len. */ - void make_abs_path(); - - /** The wrapped folder string */ - char* m_folder; - - /** Length of m_folder */ - size_t m_folder_len; - - /** A full absolute path to the same file. */ - char m_abs_path[FN_REFLEN + 2]; - - /** Length of m_abs_path to the deepest folder */ - size_t m_abs_len; -}; /** When mysqld is run, the default directory "." is the mysqld datadir, but in the MySQL Embedded Server Library and mysqlbackup it is not the default directory, and we must set the base file path explicitly */ extern const char* fil_path_to_mysql_datadir; -extern Folder folder_mysql_datadir; /** Initial size of a single-table tablespace in pages */ #define FIL_IBD_FILE_INITIAL_SIZE 4 @@ -500,23 +349,6 @@ extern fil_addr_t fil_addr_null; used to encrypt the page + 32-bit checksum or 64 bits of zero if no encryption */ -/** If page type is FIL_PAGE_COMPRESSED then the 8 bytes starting at -FIL_PAGE_FILE_FLUSH_LSN are broken down as follows: */ - -/** Control information version format (u8) */ -static const ulint FIL_PAGE_VERSION = FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION; - -/** Compression algorithm (u8) */ -static const ulint FIL_PAGE_ALGORITHM_V1 = FIL_PAGE_VERSION + 1; - -/** Original page type (u16) */ -static const ulint FIL_PAGE_ORIGINAL_TYPE_V1 = FIL_PAGE_ALGORITHM_V1 + 1; - -/** Original data size in bytes (u16)*/ -static const ulint FIL_PAGE_ORIGINAL_SIZE_V1 = FIL_PAGE_ORIGINAL_TYPE_V1 + 2; - -/** Size after compression (u16) */ -static const ulint FIL_PAGE_COMPRESS_SIZE_V1 = FIL_PAGE_ORIGINAL_SIZE_V1 + 2; /** This overloads FIL_PAGE_FILE_FLUSH_LSN for RTREE Split Sequence Number */ #define FIL_RTREE_SPLIT_SEQ_NUM FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION @@ -566,11 +398,12 @@ static const ulint FIL_PAGE_COMPRESS_SIZE_V1 = FIL_PAGE_ORIGINAL_SIZE_V1 + 2; #define FIL_PAGE_TYPE_UNKNOWN 13 /*!< In old tablespaces, garbage in FIL_PAGE_TYPE is replaced with this value when flushing pages. */ -#define FIL_PAGE_COMPRESSED 14 /*!< Compressed page */ -#define FIL_PAGE_ENCRYPTED 15 /*!< Encrypted page */ -#define FIL_PAGE_COMPRESSED_AND_ENCRYPTED 16 - /*!< Compressed and Encrypted page */ -#define FIL_PAGE_ENCRYPTED_RTREE 17 /*!< Encrypted R-tree page */ + +/* File page types introduced in MySQL 5.7, not supported in MariaDB */ +//#define FIL_PAGE_COMPRESSED 14 +//#define FIL_PAGE_ENCRYPTED 15 +//#define FIL_PAGE_COMPRESSED_AND_ENCRYPTED 16 +//#define FIL_PAGE_ENCRYPTED_RTREE 17 /** Used by i_s.cc to index into the text description. */ #define FIL_PAGE_TYPE_LAST FIL_PAGE_TYPE_UNKNOWN @@ -617,9 +450,7 @@ fil_space_get( data space) is stored here; below we talk about tablespaces, but also the ib_logfiles form a 'space' and it is handled here */ struct fil_system_t { -#ifndef UNIV_HOTBACKUP ib_mutex_t mutex; /*!< The mutex protecting the cache */ -#endif /* !UNIV_HOTBACKUP */ hash_table_t* spaces; /*!< The hash table of spaces in the system; they are hashed on the space id */ @@ -681,7 +512,6 @@ extern fil_system_t* fil_system; #include "fil0crypt.h" -#ifndef UNIV_HOTBACKUP /** Returns the latch of a file space. @param[in] id space id @param[out] flags tablespace flags @@ -708,22 +538,21 @@ void fil_space_set_imported( ulint id); -# ifdef UNIV_DEBUG +#ifdef UNIV_DEBUG /** Determine if a tablespace is temporary. @param[in] id tablespace identifier @return whether it is a temporary tablespace */ bool fsp_is_temporary(ulint id) MY_ATTRIBUTE((warn_unused_result, pure)); -# endif /* UNIV_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ +#endif /* UNIV_DEBUG */ /** Append a file to the chain of files of a space. @param[in] name file name of a file that is not open @param[in] size file size in entire database blocks @param[in,out] space tablespace from fil_space_create() @param[in] is_raw whether this is a raw device or partition -@param[in] atomic_write true if atomic write enabled +@param[in] atomic_write true if atomic write could be enabled @param[in] max_pages maximum number of pages in file, ULINT_MAX means the file size is unlimited. @return pointer to the file name @@ -753,7 +582,8 @@ fil_space_create( ulint id, ulint flags, fil_type_t purpose, /*!< in: FIL_TABLESPACE, or FIL_LOG if log */ - fil_space_crypt_t* crypt_data) /*!< in: crypt data */ + fil_space_crypt_t* crypt_data, /*!< in: crypt data */ + bool create_table) /*!< in: true if create table */ MY_ATTRIBUTE((warn_unused_result)); /*******************************************************************//** @@ -787,6 +617,12 @@ char* fil_space_get_first_path( ulint id); +/** Set the recovered size of a tablespace in pages. +@param id tablespace ID +@param size recovered size in pages */ +UNIV_INTERN +void +fil_space_set_recv_size(ulint id, ulint size); /*******************************************************************//** Returns the size of the space in pages. The tablespace must be cached in the memory cache. @@ -804,13 +640,6 @@ fil_space_get_flags( /*================*/ ulint id); /*!< in: space id */ -/** Check if table is mark for truncate. -@param[in] id space id -@return true if tablespace is marked for truncate. */ -bool -fil_space_is_being_truncated( - ulint id); - /** Open each fil_node_t of a named fil_space_t if not already open. @param[in] name Tablespace name @return true if all file nodes are opened. */ @@ -875,7 +704,6 @@ void fil_set_max_space_id_if_bigger( /*===========================*/ ulint max_id);/*!< in: maximum known id */ -#ifndef UNIV_HOTBACKUP /** Write the flushed LSN to the page header of the first page in the system tablespace. @@ -967,8 +795,6 @@ private: fil_space_t* m_space; }; -#endif /* !UNIV_HOTBACKUP */ - /********************************************************//** Creates the database directory for a table if it does not exist yet. */ void @@ -1068,7 +894,7 @@ fil_close_tablespace( /*=================*/ trx_t* trx, /*!< in/out: Transaction covering the close */ ulint id); /*!< in: space id */ -#ifndef UNIV_HOTBACKUP + /*******************************************************************//** Discards a single-table tablespace. The tablespace must be cached in the memory cache. Discarding is like deleting a tablespace, but @@ -1088,7 +914,6 @@ fil_discard_tablespace( /*===================*/ ulint id) /*!< in: space id */ MY_ATTRIBUTE((warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /** Test if a tablespace file can be renamed to a new filepath by checking if that the old filepath exists and the new filepath does not exist. @@ -1134,14 +959,15 @@ fil_make_filepath( ib_extention suffix, bool strip_name); -/** Creates a new General or Single-Table tablespace +/** Create a tablespace file. @param[in] space_id Tablespace ID @param[in] name Tablespace name in dbname/tablename format. -For general tablespaces, the 'dbname/' part may be missing. @param[in] path Path and filename of the datafile to create. @param[in] flags Tablespace flags @param[in] size Initial size of the tablespace file in pages, must be >= FIL_IBD_FILE_INITIAL_SIZE +@param[in] mode MariaDB encryption mode +@param[in] key_id MariaDB encryption key_id @return DB_SUCCESS or error code */ dberr_t fil_ibd_create( @@ -1150,10 +976,18 @@ fil_ibd_create( const char* path, ulint flags, ulint size, - fil_encryption_t mode, /*!< in: encryption mode */ - ulint key_id) /*!< in: encryption key_id */ + fil_encryption_t mode, + ulint key_id) MY_ATTRIBUTE((warn_unused_result)); +/** Try to adjust FSP_SPACE_FLAGS if they differ from the expectations. +(Typically when upgrading from MariaDB 10.1.0..10.1.20.) +@param[in] space_id tablespace ID +@param[in] flags desired tablespace flags */ +UNIV_INTERN +void +fsp_flags_try_adjust(ulint space_id, ulint flags); + /********************************************************************//** Tries to open a single-table tablespace and optionally checks the space id is right in it. If does not succeed, prints an error message to the .err log. This @@ -1178,7 +1012,7 @@ statement to update the dictionary tables if they are incorrect. @param[in] fix_dict true if the dictionary is available to be fixed @param[in] purpose FIL_TYPE_TABLESPACE or FIL_TYPE_TEMPORARY @param[in] id tablespace ID -@param[in] flags tablespace flags +@param[in] flags expected FSP_SPACE_FLAGS @param[in] space_name tablespace name of the datafile If file-per-table, it is the table name in the databasename/tablename format @param[in] path_in expected filepath, usually read from dictionary @@ -1234,7 +1068,6 @@ fil_file_readdir_next_file( os_file_dir_t dir, /*!< in: directory stream */ os_file_stat_t* info); /*!< in/out: buffer where the info is returned */ -#ifndef UNIV_HOTBACKUP /*******************************************************************//** Returns true if a matching tablespace exists in the InnoDB tablespace memory cache. Note that if we have not done a crash recovery at the database startup, @@ -1255,17 +1088,9 @@ fil_space_for_table_exists_in_mem( when find table space mismatch */ mem_heap_t* heap, /*!< in: heap memory */ table_id_t table_id, /*!< in: table id */ - dict_table_t* table); /*!< in: table or NULL */ -#else /* !UNIV_HOTBACKUP */ -/********************************************************************//** -Extends all tablespaces to the size stored in the space header. During the -mysqlbackup --apply-log phase we extended the spaces on-demand so that log -records could be appllied, but that may have left spaces still too small -compared to the size stored in the space header. */ -void -fil_extend_tablespaces_to_stored_len(void); -/*======================================*/ -#endif /* !UNIV_HOTBACKUP */ + dict_table_t* table, /*!< in: table or NULL */ + ulint table_flags); /*!< in: table flags */ + /** Try to extend a tablespace if it is smaller than the specified size. @param[in,out] space tablespace @param[in] size desired size in pages @@ -1314,11 +1139,6 @@ fil_space_get_n_reserved_extents( aligned @param[in] message message for aio handler if non-sync aio used, else ignored -@param[in,out] write_size Actual write size initialized - after fist successfull trim - operation for this page and if - nitialized we do not trim again if - Actual page @return DB_SUCCESS, DB_TABLESPACE_DELETED or DB_TABLESPACE_TRUNCATED if we are trying to do i/o on a tablespace which does not exist */ @@ -1331,8 +1151,7 @@ fil_io( ulint byte_offset, ulint len, void* buf, - void* message, - ulint* write_size); + void* message); /**********************************************************************//** Waits for an aio operation to complete. This function is used to write the handler for completed requests. The aio array of pending requests is divided @@ -1462,14 +1281,6 @@ fil_space_inc_redo_skipped_count( void fil_space_dec_redo_skipped_count( ulint id); - -/*******************************************************************//** -Check whether a single-table tablespace is redo skipped. -@return true if redo skipped */ -bool -fil_space_is_redo_skipped( -/*======================*/ - ulint id); /*!< in: space id */ #endif /********************************************************************//** @@ -1528,10 +1339,6 @@ struct PageCallback { @retval the space flags of the tablespace being iterated over */ virtual ulint get_space_flags() const UNIV_NOTHROW = 0; - /** Set the tablespace table size. - @param[in] page a page belonging to the tablespace */ - void set_page_size(const buf_frame_t* page) UNIV_NOTHROW; - /** The compressed page size @return the compressed page size */ const page_size_t& get_page_size() const @@ -1695,43 +1502,6 @@ fil_names_dirty_and_write( fil_space_t* space, mtr_t* mtr); -/** Set the compression type for the tablespace of a table -@param[in] table Table that should be compressesed -@param[in] algorithm Text representation of the algorithm -@return DB_SUCCESS or error code */ -dberr_t -fil_set_compression( - dict_table_t* table, - const char* algorithm) - MY_ATTRIBUTE((warn_unused_result)); - -/** Get the compression type for the tablespace -@param[in] space_id Space ID to check -@return the compression algorithm */ -Compression::Type -fil_get_compression( - ulint space_id) - MY_ATTRIBUTE((warn_unused_result)); - -/** Set the encryption type for the tablespace -@param[in] space Space ID of tablespace for which to set -@param[in] algorithm Encryption algorithm -@param[in] key Encryption key -@param[in] iv Encryption iv -@return DB_SUCCESS or error code */ -dberr_t -fil_set_encryption( - ulint space_id, - Encryption::Type algorithm, - byte* key, - byte* iv) - MY_ATTRIBUTE((warn_unused_result)); - -/** -@return true if the re-encrypt success */ -bool -fil_encryption_rotate(); - /** Write MLOG_FILE_NAME records if a persistent tablespace was modified for the first time since the latest fil_names_clear(). @param[in,out] space tablespace @@ -1797,19 +1567,6 @@ fil_names_clear( lsn_t lsn, bool do_write); -#if !defined(NO_FALLOCATE) && defined(UNIV_LINUX) -/** -Try and enable FusionIO atomic writes. -@param[in] file OS file handle -@return true if successful */ -bool -fil_fusionio_enable_atomic_write(os_file_t file); -#endif /* !NO_FALLOCATE && UNIV_LINUX */ - -/** Note that the file system where the file resides doesn't support PUNCH HOLE -@param[in,out] node Node to set */ -void fil_no_punch_hole(fil_node_t* node); - #ifdef UNIV_ENABLE_UNIT_TEST_MAKE_FILEPATH void test_make_filepath(); #endif /* UNIV_ENABLE_UNIT_TEST_MAKE_FILEPATH */ @@ -1817,15 +1574,16 @@ void test_make_filepath(); /*******************************************************************//** Returns the block size of the file space +@param[in] space_id space id +@param[in] offset page offset +@param[in] len page len @return block size */ UNIV_INTERN ulint fil_space_get_block_size( -/*=====================*/ - ulint id, /*!< in: space id */ - ulint offset, /*!< in: page offset */ - ulint len); /*!< in: page len */ - + ulint id, + os_offset_t offset, + ulint len); /*******************************************************************//** Increments the count of pending operation, if space is not being deleted. @return TRUE if being deleted, and operation should be skipped */ diff --git a/storage/innobase/include/fil0fil.ic b/storage/innobase/include/fil0fil.ic index 8f8a4194c0d..01fa1093e5e 100644 --- a/storage/innobase/include/fil0fil.ic +++ b/storage/innobase/include/fil0fil.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2015, MariaDB Corporation. +Copyright (c) 2015, 2017 MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -58,49 +58,41 @@ fil_get_page_type_name( { switch(page_type) { case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED: - return (const char*)"PAGE_COMPRESSED_ENRYPTED"; + return "PAGE_COMPRESSED_ENRYPTED"; case FIL_PAGE_PAGE_COMPRESSED: - return (const char*)"PAGE_COMPRESSED"; + return "PAGE_COMPRESSED"; case FIL_PAGE_INDEX: - return (const char*)"INDEX"; + return "INDEX"; case FIL_PAGE_RTREE: - return (const char*)"RTREE"; + return "RTREE"; case FIL_PAGE_UNDO_LOG: - return (const char*)"UNDO LOG"; + return "UNDO LOG"; case FIL_PAGE_INODE: - return (const char*)"INODE"; + return "INODE"; case FIL_PAGE_IBUF_FREE_LIST: - return (const char*)"IBUF_FREE_LIST"; + return "IBUF_FREE_LIST"; case FIL_PAGE_TYPE_ALLOCATED: - return (const char*)"ALLOCATED"; + return "ALLOCATED"; case FIL_PAGE_IBUF_BITMAP: - return (const char*)"IBUF_BITMAP"; + return "IBUF_BITMAP"; case FIL_PAGE_TYPE_SYS: - return (const char*)"SYS"; + return "SYS"; case FIL_PAGE_TYPE_TRX_SYS: - return (const char*)"TRX_SYS"; + return "TRX_SYS"; case FIL_PAGE_TYPE_FSP_HDR: - return (const char*)"FSP_HDR"; + return "FSP_HDR"; case FIL_PAGE_TYPE_XDES: - return (const char*)"XDES"; + return "XDES"; case FIL_PAGE_TYPE_BLOB: - return (const char*)"BLOB"; + return "BLOB"; case FIL_PAGE_TYPE_ZBLOB: - return (const char*)"ZBLOB"; + return "ZBLOB"; case FIL_PAGE_TYPE_ZBLOB2: - return (const char*)"ZBLOB2"; - case FIL_PAGE_COMPRESSED: - return (const char*)"ORACLE PAGE COMPRESSED"; - case FIL_PAGE_ENCRYPTED: - return (const char*)"ORACLE PAGE ENCRYPTED"; - case FIL_PAGE_COMPRESSED_AND_ENCRYPTED: - return (const char*)"ORACLE PAGE COMPRESSED AND ENCRYPTED"; - case FIL_PAGE_ENCRYPTED_RTREE: - return (const char*)"ORACLE RTREE ENCRYPTED"; + return "ZBLOB2"; case FIL_PAGE_TYPE_UNKNOWN: - return (const char*)"OLD UNKOWN PAGE TYPE"; + return "OLD UNKOWN PAGE TYPE"; default: - return (const char*)"PAGE TYPE CORRUPTED"; + return "PAGE TYPE CORRUPTED"; } } @@ -145,52 +137,22 @@ fil_page_type_validate( page_type == FIL_PAGE_TYPE_BLOB || page_type == FIL_PAGE_TYPE_ZBLOB || page_type == FIL_PAGE_TYPE_ZBLOB2 || - page_type == FIL_PAGE_COMPRESSED || - page_type == FIL_PAGE_TYPE_UNKNOWN || - page_type == FIL_PAGE_ENCRYPTED || - page_type == FIL_PAGE_COMPRESSED_AND_ENCRYPTED || - page_type == FIL_PAGE_ENCRYPTED_RTREE))) { - - uint key_version = mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - bool page_compressed = (page_type == FIL_PAGE_PAGE_COMPRESSED); - bool page_compressed_encrypted = (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED); + page_type == FIL_PAGE_TYPE_UNKNOWN))) { + ulint space = mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); ulint offset = mach_read_from_4(page + FIL_PAGE_OFFSET); - ib_uint64_t lsn = mach_read_from_8(page + FIL_PAGE_LSN); - ulint compressed_len = mach_read_from_2(page + FIL_PAGE_DATA); fil_system_enter(); fil_space_t* rspace = fil_space_get_by_id(space); fil_system_exit(); /* Dump out the page info */ - fprintf(stderr, "InnoDB: Space %lu offset %lu name %s page_type %lu page_type_name %s\n" - "InnoDB: key_version %u page_compressed %d page_compressed_encrypted %d lsn %llu compressed_len %lu\n", - space, offset, rspace->name, page_type, fil_get_page_type_name(page_type), - key_version, page_compressed, page_compressed_encrypted, (ulonglong)lsn, compressed_len); - fflush(stderr); - - ut_ad(page_type == FIL_PAGE_PAGE_COMPRESSED || - page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED || - page_type == FIL_PAGE_INDEX || - page_type == FIL_PAGE_RTREE || - page_type == FIL_PAGE_UNDO_LOG || - page_type == FIL_PAGE_INODE || - page_type == FIL_PAGE_IBUF_FREE_LIST || - page_type == FIL_PAGE_TYPE_ALLOCATED || - page_type == FIL_PAGE_IBUF_BITMAP || - page_type == FIL_PAGE_TYPE_SYS || - page_type == FIL_PAGE_TYPE_TRX_SYS || - page_type == FIL_PAGE_TYPE_FSP_HDR || - page_type == FIL_PAGE_TYPE_XDES || - page_type == FIL_PAGE_TYPE_BLOB || - page_type == FIL_PAGE_TYPE_ZBLOB || - page_type == FIL_PAGE_TYPE_ZBLOB2 || - page_type == FIL_PAGE_COMPRESSED || - page_type == FIL_PAGE_TYPE_UNKNOWN || - page_type == FIL_PAGE_ENCRYPTED || - page_type == FIL_PAGE_COMPRESSED_AND_ENCRYPTED || - page_type == FIL_PAGE_ENCRYPTED_RTREE); - + ib::fatal() << "Page " << space << ":" << offset + << " name " << (rspace ? rspace->name : "???") + << " page_type " << page_type + << " key_version " + << mach_read_from_4(page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) + << " lsn " << mach_read_from_8(page + FIL_PAGE_LSN) + << " compressed_len " << mach_read_from_2(page + FIL_PAGE_DATA); return false; } diff --git a/storage/innobase/include/fil0pagecompress.h b/storage/innobase/include/fil0pagecompress.h index 2948207133b..0bf6aa75f19 100644 --- a/storage/innobase/include/fil0pagecompress.h +++ b/storage/innobase/include/fil0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2016 MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017 MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -49,20 +49,11 @@ fil_space_is_page_compressed( /*=========================*/ ulint id); /*!< in: space id */ /*******************************************************************//** -Returns the page compression flag of the space, or false if the space -is not compressed. The tablespace must be cached in the memory cache. -@return true if page compressed, false if not or space not found */ -UNIV_INTERN -bool -fil_space_get_page_compressed( -/*=========================*/ - fil_space_t* space); /*!< in: space id */ -/*******************************************************************//** Returns the atomic writes flag of the space, or false if the space is not using atomic writes. The tablespace must be cached in the memory cache. @return atomic write table option value */ UNIV_INLINE -atomic_writes_t +bool fil_space_get_atomic_writes( /*=========================*/ ulint id); /*!< in: space id */ @@ -124,5 +115,4 @@ fil_node_get_block_size( /*====================*/ fil_node_t* node); /*!< in: Node where to get block size */ - #endif diff --git a/storage/innobase/include/fsp0file.h b/storage/innobase/include/fsp0file.h index 83aa370abf0..1c4ac8152c9 100644 --- a/storage/innobase/include/fsp0file.h +++ b/storage/innobase/include/fsp0file.h @@ -65,11 +65,8 @@ public: m_is_valid(), m_first_page_buf(), m_first_page(), - m_atomic_write(), m_last_os_error(), m_file_info(), - m_encryption_key(NULL), - m_encryption_iv(NULL), m_crypt_info() { /* No op */ @@ -91,11 +88,8 @@ public: m_is_valid(), m_first_page_buf(), m_first_page(), - m_atomic_write(), m_last_os_error(), m_file_info(), - m_encryption_key(NULL), - m_encryption_iv(NULL), m_crypt_info() { ut_ad(m_name != NULL); @@ -115,11 +109,8 @@ public: m_is_valid(file.m_is_valid), m_first_page_buf(), m_first_page(), - m_atomic_write(file.m_atomic_write), m_last_os_error(), m_file_info(), - m_encryption_key(NULL), - m_encryption_iv(NULL), m_crypt_info() { m_name = mem_strdup(file.m_name); @@ -178,13 +169,9 @@ public: it should be reread if needed */ m_first_page_buf = NULL; m_first_page = NULL; - m_encryption_key = NULL; - m_encryption_iv = NULL; /* Do not copy crypt info it is read from first page */ m_crypt_info = NULL; - m_atomic_write = file.m_atomic_write; - return(*this); } @@ -233,9 +220,8 @@ public: void set_filepath(const char* filepath); /** Allocate and set the datafile or tablespace name in m_name. - If a name is provided, use it; else if the datafile is file-per-table, - extract a file-per-table tablespace name from m_filepath; else it is a - general tablespace, so just call it that for now. The value of m_name + If a name is provided, use it; else extract a file-per-table + tablespace name from m_filepath. The value of m_name will be freed in the destructor. @param[in] name Tablespace Name if known, NULL if not */ void set_name(const char* name); @@ -245,13 +231,9 @@ public: successfully opened in order for this function to validate it. @param[in] space_id The expected tablespace ID. @param[in] flags The expected tablespace flags. - @param[in] for_import is it for importing @retval DB_SUCCESS if tablespace is valid, DB_ERROR if not. m_is_valid is also set true on success, else false. */ - dberr_t validate_to_dd( - ulint space_id, - ulint flags, - bool for_import) + dberr_t validate_to_dd(ulint space_id, ulint flags) MY_ATTRIBUTE((warn_unused_result)); /** Validates this datafile for the purpose of recovery. @@ -270,13 +252,10 @@ public: so the Space ID found here must not already be open. m_is_valid is set true on success, else false. @param[out] flush_lsn contents of FIL_PAGE_FILE_FLUSH_LSN - @param[in] for_import if it is for importing - (only valid for the first file of the system tablespace) @retval DB_SUCCESS on if the datafile is valid @retval DB_CORRUPTION if the datafile is not readable @retval DB_TABLESPACE_EXISTS if there is a duplicate space_id */ - dberr_t validate_first_page(lsn_t* flush_lsn, - bool for_import) + dberr_t validate_first_page(lsn_t* flush_lsn) MY_ATTRIBUTE((warn_unused_result)); /** Get Datafile::m_name. @@ -428,12 +407,10 @@ private: else DB_ERROR. */ dberr_t find_space_id(); - /** Finds a given page of the given space id from the double write - buffer and copies it to the corresponding .ibd file. - @param[in] page_no Page number to restore - @return DB_SUCCESS if page was restored, else DB_ERROR */ - dberr_t restore_from_doublewrite( - ulint restore_page_no); + /** Restore the first page of the tablespace from + the double write buffer. + @return whether the operation failed */ + bool restore_from_doublewrite(); /** Points into m_filepath to the file name with extension */ char* m_filename; @@ -475,9 +452,6 @@ private: /** Pointer to the first page held in the buffer above */ byte* m_first_page; - /** true if atomic writes enabled for this file */ - bool m_atomic_write; - protected: /** Last OS error received so it can be reported if needed. */ ulint m_last_os_error; @@ -492,12 +466,6 @@ public: struct stat m_file_info; #endif /* WIN32 */ - /** Encryption key read from first page */ - byte* m_encryption_key; - - /** Encryption iv read from first page */ - byte* m_encryption_iv; - /** Encryption information */ fil_space_crypt_t* m_crypt_info; }; @@ -581,13 +549,10 @@ public: the path ".". @param[in] name tablespace name @param[in] filepath remote filepath of tablespace datafile - @param[in] is_shared true for general tablespace, - false for file-per-table @return DB_SUCCESS or error code */ static dberr_t create_link_file( const char* name, - const char* filepath, - bool is_shared = false); + const char* filepath); /** Delete an InnoDB Symbolic Link (ISL) file by name. @param[in] name tablespace name */ @@ -597,8 +562,6 @@ public: It is always created under the datadir of MySQL. For file-per-table tablespaces, the isl file is expected to be in a 'database' directory and called 'tablename.isl'. - For general tablespaces, there will be no 'database' directory. - The 'basename.isl' will be in the datadir. The caller must free the memory returned if it is not null. @param[in] link_filepath filepath of the ISL file @return Filepath of the IBD file read from the ISL file */ diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index 17ccb113e91..fa8d5b76960 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -43,17 +43,55 @@ Created 12/18/1995 Heikki Tuuri #endif /* !UNIV_INNOCHECKSUM */ #include "fsp0types.h" -#define FSP_FLAGS_POS_DATA_DIR_ORACLE (FSP_FLAGS_POS_ATOMIC_BLOBS \ - + FSP_FLAGS_WIDTH_ATOMIC_BLOBS \ - + FSP_FLAGS_WIDTH_PAGE_SSIZE) -/** Bit mask of the DATA_DIR field */ -#define FSP_FLAGS_MASK_DATA_DIR_ORACLE \ - ((~(~0U << FSP_FLAGS_WIDTH_DATA_DIR)) \ - << FSP_FLAGS_POS_DATA_DIR_ORACLE) +/** @return the PAGE_SSIZE flags for the current innodb_page_size */ +#define FSP_FLAGS_PAGE_SSIZE() \ + ((UNIV_PAGE_SIZE == UNIV_PAGE_SIZE_ORIG) ? \ + 0 : (UNIV_PAGE_SIZE_SHIFT - UNIV_ZIP_SIZE_SHIFT_MIN + 1) \ + << FSP_FLAGS_POS_PAGE_SSIZE) + +/* @defgroup Compatibility macros for MariaDB 10.1.0 through 10.1.20; +see the table in fsp0types.h @{ */ +/** Zero relative shift position of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101 \ + (FSP_FLAGS_POS_ATOMIC_BLOBS \ + + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) +/** Zero relative shift position of the PAGE_COMPRESSION_LEVEL field */ +#define FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101 \ + (FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101 + 1) +/** Zero relative shift position of the ATOMIC_WRITES field */ +#define FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101 \ + (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101 + 4) +/** Zero relative shift position of the PAGE_SSIZE field */ +#define FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101 \ + (FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101 + 2) + +/** Bit mask of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION_MARIADB101 \ + (1U << FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101) +/** Bit mask of the PAGE_COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL_MARIADB101 \ + (15U << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101) +/** Bit mask of the ATOMIC_WRITES field */ +#define FSP_FLAGS_MASK_ATOMIC_WRITES_MARIADB101 \ + (3U << FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101) +/** Bit mask of the PAGE_SSIZE field */ +#define FSP_FLAGS_MASK_PAGE_SSIZE_MARIADB101 \ + (15U << FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101) + +/** Return the value of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101) +/** Return the value of the PAGE_COMPRESSION_LEVEL field */ +#define FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101) +/** Return the value of the PAGE_SSIZE field */ +#define FSP_FLAGS_GET_PAGE_SSIZE_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_SSIZE_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101) -#define FSP_FLAGS_HAS_DATA_DIR_ORACLE(flags) \ - ((flags & FSP_FLAGS_MASK_DATA_DIR_ORACLE) \ - >> FSP_FLAGS_POS_DATA_DIR_ORACLE) +/* @} */ /* @defgroup Tablespace Header Constants (moved from fsp0fsp.c) @{ */ @@ -274,7 +312,7 @@ fsp_header_get_tablespace_size(void); /*================================*/ /** Calculate the number of pages to extend a datafile. -We extend single-table and general tablespaces first one extent at a time, +We extend single-table tablespaces first one extent at a time, but 4 at a time for bigger tablespaces. It is not enough to extend always by one extent, because we need to add at least one extent to FSP_FREE. A single extent descriptor page will track many extents. And the extent @@ -329,37 +367,15 @@ fsp_header_get_flags(const page_t* page) return(fsp_header_get_field(page, FSP_SPACE_FLAGS)); } -/** Reads the page size from the first page of a tablespace. -@param[in] page first page of a tablespace -@return page size */ -page_size_t -fsp_header_get_page_size( - const page_t* page); - -/** Decoding the encryption info -from the first page of a tablespace. -@param[in/out] key key -@param[in/out] iv iv -@param[in] encryption_info encrytion info. -@return true if success */ -bool -fsp_header_decode_encryption_info( - byte* key, - byte* iv, - byte* encryption_info); - -/** Reads the encryption key from the first page of a tablespace. -@param[in] fsp_flags tablespace flags -@param[in/out] key tablespace key -@param[in/out] iv tablespace iv -@param[in] page first page of a tablespace -@return true if success */ -bool -fsp_header_get_encryption_key( - ulint fsp_flags, - byte* key, - byte* iv, - page_t* page); +/** Get the byte offset of encryption information in page 0. +@param[in] ps page size +@return byte offset relative to FSP_HEADER_OFFSET */ +inline MY_ATTRIBUTE((pure, warn_unused_result)) +ulint +fsp_header_get_encryption_offset(const page_size_t& ps) +{ + return XDES_ARR_OFFSET + XDES_SIZE * ps.physical() / FSP_EXTENT_SIZE; +} /** Check the encryption key from the first page of a tablespace. @param[in] fsp_flags tablespace flags @@ -382,17 +398,6 @@ fsp_header_init_fields( ulint flags); /*!< in: tablespace flags (FSP_SPACE_FLAGS): 0, or table->flags if newer than COMPACT */ -/** Rotate the encryption info in the space header. -@param[in] space tablespace -@param[in] encrypt_info buffer for re-encrypt key. -@param[in,out] mtr mini-transaction -@return true if success. */ -bool -fsp_header_rotate_encryption( - fil_space_t* space, - byte* encrypt_info, - mtr_t* mtr); - /** Initializes the space header of a new created space and creates also the insert buffer tree root if space == 0. @param[in] space_id space id @@ -662,60 +667,135 @@ fseg_print( mtr_t* mtr); /*!< in/out: mini-transaction */ #endif /* UNIV_BTR_PRINT */ -/** Determine if the tablespace is compressed from tablespace flags. -@param[in] flags Tablespace flags -@return true if compressed, false if not compressed */ +/** Convert FSP_SPACE_FLAGS from the buggy MariaDB 10.1.0..10.1.20 format. +@param[in] flags the contents of FSP_SPACE_FLAGS +@return the flags corrected from the buggy MariaDB 10.1 format +@retval ULINT_UNDEFINED if the flags are not in the buggy 10.1 format */ +MY_ATTRIBUTE((warn_unused_result, const)) UNIV_INLINE -bool -fsp_flags_is_compressed( - ulint flags); +ulint +fsp_flags_convert_from_101(ulint flags) +{ + DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", + return(ULINT_UNDEFINED);); + if (flags == 0) { + return(flags); + } + + if (flags >> 18) { + /* The most significant FSP_SPACE_FLAGS bit that was ever set + by MariaDB 10.1.0 to 10.1.20 was bit 17 (misplaced DATA_DIR flag). + The flags must be less than 1<<18 in order to be valid. */ + return(ULINT_UNDEFINED); + } + + if ((flags & (FSP_FLAGS_MASK_POST_ANTELOPE | FSP_FLAGS_MASK_ATOMIC_BLOBS)) + == FSP_FLAGS_MASK_ATOMIC_BLOBS) { + /* If the "atomic blobs" flag (indicating + ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED) flag + is set, then the "post Antelope" (ROW_FORMAT!=REDUNDANT) flag + must also be set. */ + return(ULINT_UNDEFINED); + } + + /* Bits 6..10 denote compression in MariaDB 10.1.0 to 10.1.20. + They must be either 0b00000 or 0b00011 through 0b10011. + In correct versions, these bits would be + 0bd0sss where d is the DATA_DIR flag (garbage bit) and + sss is the PAGE_SSIZE (3, 4, 6, or 7). + + NOTE: MariaDB 10.1.0 to 10.1.20 can misinterpret + uncompressed data files with innodb_page_size=4k or 64k as + compressed innodb_page_size=16k files. Below is an exhaustive + state space analysis. + + -0by1zzz: impossible (the bit 4 must be clean; see above) + -0b101xx: DATA_DIR, innodb_page_size>4k: invalid (COMPRESSION_LEVEL>9) + +0bx0011: innodb_page_size=4k: + !!! Misinterpreted as COMPRESSION_LEVEL=9 or 1, COMPRESSION=1. + -0bx0010: impossible, because sss must be 0b011 or 0b1xx + -0bx0001: impossible, because sss must be 0b011 or 0b1xx + -0b10000: DATA_DIR, innodb_page_size=16: + invalid (COMPRESSION_LEVEL=8 but COMPRESSION=0) + +0b00111: no DATA_DIR, innodb_page_size=64k: + !!! Misinterpreted as COMPRESSION_LEVEL=3, COMPRESSION=1. + -0b00101: impossible, because sss must be 0 for 16k, not 0b101 + -0b001x0: no DATA_DIR, innodb_page_size=32k or 8k: + invalid (COMPRESSION_LEVEL=3 but COMPRESSION=0) + +0b00000: innodb_page_size=16k (looks like COMPRESSION=0) + ??? Could actually be compressed; see PAGE_SSIZE below */ + const ulint level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL_MARIADB101( + flags); + if (FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) != (level != 0) + || level > 9) { + /* The compression flags are not in the buggy MariaDB + 10.1 format. */ + return(ULINT_UNDEFINED); + } + if (!(~flags & FSP_FLAGS_MASK_ATOMIC_WRITES_MARIADB101)) { + /* The ATOMIC_WRITES flags cannot be 0b11. + (The bits 11..12 should actually never be 0b11, + because in MySQL they would be SHARED|TEMPORARY.) */ + return(ULINT_UNDEFINED); + } + + /* Bits 13..16 are the wrong position for PAGE_SSIZE, and they + should contain one of the values 3,4,6,7, that is, be of the form + 0b0011 or 0b01xx (except 0b0110). + In correct versions, these bits should be 0bc0se + where c is the MariaDB COMPRESSED flag + and e is the MySQL 5.7 ENCRYPTION flag + and s is the MySQL 8.0 SDI flag. MariaDB can only support s=0, e=0. + + Compressed innodb_page_size=16k tables with correct FSP_SPACE_FLAGS + will be properly rejected by older MariaDB 10.1.x because they + would read as PAGE_SSIZE>=8 which is not valid. */ + + const ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE_MARIADB101(flags); + if (ssize == 1 || ssize == 2 || ssize == 5 || ssize & 8) { + /* the page_size is not between 4k and 64k; + 16k should be encoded as 0, not 5 */ + return(ULINT_UNDEFINED); + } + const ulint zssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); + if (zssize == 0) { + /* not ROW_FORMAT=COMPRESSED */ + } else if (zssize > (ssize ? ssize : 5)) { + /* invalid KEY_BLOCK_SIZE */ + return(ULINT_UNDEFINED); + } else if (~flags & (FSP_FLAGS_MASK_POST_ANTELOPE + | FSP_FLAGS_MASK_ATOMIC_BLOBS)) { + /* both these flags should be set for + ROW_FORMAT=COMPRESSED */ + return(ULINT_UNDEFINED); + } + + flags = ((flags & 0x3f) | ssize << FSP_FLAGS_POS_PAGE_SSIZE + | FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) + << FSP_FLAGS_POS_PAGE_COMPRESSION); + ut_ad(fsp_flags_is_valid(flags)); + return(flags); +} -/** Determine if two tablespaces are equivalent or compatible. -@param[in] flags1 First tablespace flags -@param[in] flags2 Second tablespace flags -@return true the flags are compatible, false if not */ +/** Compare tablespace flags. +@param[in] expected expected flags from dict_tf_to_fsp_flags() +@param[in] actual flags read from FSP_SPACE_FLAGS +@return whether the flags match */ +MY_ATTRIBUTE((warn_unused_result)) UNIV_INLINE bool -fsp_flags_are_equal( - ulint flags1, - ulint flags2); - -/** Initialize an FSP flags integer. -@param[in] page_size page sizes in bytes and compression flag. -@param[in] atomic_blobs Used by Dynammic and Compressed. -@param[in] has_data_dir This tablespace is in a remote location. -@param[in] is_shared This tablespace can be shared by many tables. -@param[in] is_temporary This tablespace is temporary. -@param[in] is_encrypted This tablespace is encrypted. -@return tablespace flags after initialization */ -UNIV_INLINE -ulint -fsp_flags_init( - const page_size_t& page_size, - bool atomic_blobs, - bool has_data_dir, - bool is_shared, - bool is_temporary, - bool page_compression, - ulint page_compression_level, - ulint atomic_writes, - bool is_encrypted = false); - -/** Convert a 32 bit integer tablespace flags to the 32 bit table flags. -This can only be done for a tablespace that was built as a file-per-table -tablespace. Note that the fsp_flags cannot show the difference between a -Compact and Redundant table, so an extra Compact boolean must be supplied. - Low order bit - | REDUNDANT | COMPACT | COMPRESSED | DYNAMIC -fil_space_t::flags | 0 | 0 | 1 | 1 -dict_table_t::flags | 0 | 1 | 1 | 1 -@param[in] fsp_flags fil_space_t::flags -@param[in] compact true if not Redundant row format -@return tablespace flags (fil_space_t::flags) */ -ulint -fsp_flags_to_dict_tf( - ulint fsp_flags, - bool compact); +fsp_flags_match(ulint expected, ulint actual) +{ + expected &= ~FSP_FLAGS_MEM_MASK; + ut_ad(fsp_flags_is_valid(expected)); + + if (actual == expected) { + return(true); + } + + actual = fsp_flags_convert_from_101(actual); + return(actual == expected); +} /** Calculates the descriptor index within a descriptor page. @param[in] page_size page size @@ -727,27 +807,6 @@ xdes_calc_descriptor_index( const page_size_t& page_size, ulint offset); -/** Gets pointer to a the extent descriptor of a page. -The page where the extent descriptor resides is x-locked. If the page offset -is equal to the free limit of the space, adds new extents from above the free -limit to the space free list, if not free limit == space size. This adding -is necessary to make the descriptor defined, as they are uninitialized -above the free limit. -@param[in] space_id space id -@param[in] offset page offset; if equal to the free limit, we -try to add new extents to the space free list -@param[in] page_size page size -@param[in,out] mtr mini-transaction -@return pointer to the extent descriptor, NULL if the page does not -exist in the space or if the offset exceeds the free limit */ -xdes_t* -xdes_get_descriptor( - ulint space_id, - ulint offset, - const page_size_t& page_size, - mtr_t* mtr) -MY_ATTRIBUTE((warn_unused_result)); - /**********************************************************************//** Gets a descriptor bit of a page. @return TRUE if free */ @@ -770,20 +829,6 @@ xdes_calc_descriptor_page( const page_size_t& page_size, ulint offset); -#endif /* !UNIV_INNOCHECKSUM */ - -/*********************************************************************//** -@return offset into fsp header where crypt data is stored */ -UNIV_INTERN -ulint -fsp_header_get_crypt_offset( -/*========================*/ - const page_size_t& page_size,/*!< in: page size */ - ulint* max_size); /*!< out: free space after offset */ - - -#ifndef UNIV_INNOCHECKSUM - /**********************************************************************//** Checks if a single page is free. @return true if free */ diff --git a/storage/innobase/include/fsp0fsp.ic b/storage/innobase/include/fsp0fsp.ic index 475dd238728..6317a67a089 100644 --- a/storage/innobase/include/fsp0fsp.ic +++ b/storage/innobase/include/fsp0fsp.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -40,224 +40,6 @@ fsp_descr_page( == FSP_XDES_OFFSET); } -/** Determine if the tablespace is compressed from tablespace flags. -@param[in] flags Tablespace flags -@return true if compressed, false if not compressed */ -UNIV_INLINE -bool -fsp_flags_is_compressed( - ulint flags) -{ - return(FSP_FLAGS_GET_ZIP_SSIZE(flags) != 0); -} - -#define ACTUAL_SSIZE(ssize) (0 == ssize ? UNIV_PAGE_SSIZE_ORIG : ssize) - -/** Determine if two tablespaces are equivalent or compatible. -@param[in] flags1 First tablespace flags -@param[in] flags2 Second tablespace flags -@return true the flags are compatible, false if not */ -UNIV_INLINE -bool -fsp_flags_are_equal( - ulint flags1, - ulint flags2) -{ - /* If either one of these flags is ULINT_UNDEFINED, - then they are not equal */ - if (flags1 == ULINT_UNDEFINED || flags2 == ULINT_UNDEFINED) { - return(false); - } - - if (!fsp_is_shared_tablespace(flags1) || !fsp_is_shared_tablespace(flags2)) { - /* At least one of these is a single-table tablespaces so all - flags must match. */ - return(flags1 == flags2); - } - - /* Both are shared tablespaces which can contain all formats. - But they must have the same logical and physical page size. - Once InnoDB can support multiple page sizes together, - the logical page size will not matter. */ - ulint zip_ssize1 = ACTUAL_SSIZE(FSP_FLAGS_GET_ZIP_SSIZE(flags1)); - ulint zip_ssize2 = ACTUAL_SSIZE(FSP_FLAGS_GET_ZIP_SSIZE(flags2)); - ulint page_ssize1 = ACTUAL_SSIZE(FSP_FLAGS_GET_PAGE_SSIZE(flags1)); - ulint page_ssize2 = ACTUAL_SSIZE(FSP_FLAGS_GET_PAGE_SSIZE(flags2)); - - return(zip_ssize1 == zip_ssize2 && page_ssize1 == page_ssize2); -} - -/** Convert a page size, which is a power of 2, to an ssize, which is -the number of bit shifts from 512 to make that page size. -@param[in] page_size compressed page size in bytes -@return an ssize created from the page size provided. */ -UNIV_INLINE -ulint -page_size_to_ssize( - ulint page_size) -{ - ulint ssize; - - for (ssize = UNIV_ZIP_SIZE_SHIFT_MIN; - ((ulint) 1 << ssize) < page_size; - ssize++) {}; - - return(ssize - UNIV_ZIP_SIZE_SHIFT_MIN + 1); -} - -/** Add the compressed page size to the tablespace flags. -@param[in] flags Tablespace flags -@param[in] page_size page sizes in bytes and compression flag. -@return tablespace flags after zip size is added */ -UNIV_INLINE -ulint -fsp_flags_set_zip_size( - ulint flags, - const page_size_t& page_size) -{ - if (!page_size.is_compressed()) { - return(flags); - } - - /* Zip size should be a power of 2 between UNIV_ZIP_SIZE_MIN - and UNIV_ZIP_SIZE_MAX */ - ut_ad(page_size.physical() >= UNIV_ZIP_SIZE_MIN); - ut_ad(page_size.physical() <= UNIV_ZIP_SIZE_MAX); - ut_ad(ut_is_2pow(page_size.physical())); - - ulint ssize = page_size_to_ssize(page_size.physical()); - - ut_ad(ssize > 0); - ut_ad(ssize <= UNIV_PAGE_SSIZE_MAX); - - flags |= (ssize << FSP_FLAGS_POS_ZIP_SSIZE); - - ut_ad(fsp_flags_is_valid(flags)); - - return(flags); -} - -/** Add the page size to the tablespace flags. -@param[in] flags Tablespace flags -@param[in] page_size page sizes in bytes and compression flag. -@return tablespace flags after page size is added */ -UNIV_INLINE -ulint -fsp_flags_set_page_size( - ulint flags, - const page_size_t& page_size) -{ - /* Page size should be a power of two between UNIV_PAGE_SIZE_MIN - and UNIV_PAGE_SIZE */ - ut_ad(page_size.logical() >= UNIV_PAGE_SIZE_MIN); - ut_ad(page_size.logical() <= UNIV_PAGE_SIZE_MAX); - ut_ad(ut_is_2pow(page_size.logical())); - - /* Remove this assert once we add support for different - page size per tablespace. Currently all tablespaces must - have a page size that is equal to innodb-page-size */ - ut_ad(page_size.logical() == UNIV_PAGE_SIZE); - - if (page_size.logical() == UNIV_PAGE_SIZE_ORIG) { - ut_ad(0 == FSP_FLAGS_GET_PAGE_SSIZE(flags)); - - } else { - ulint ssize = page_size_to_ssize(page_size.logical()); - - ut_ad(ssize); - ut_ad(ssize <= UNIV_PAGE_SSIZE_MAX); - - flags |= (ssize << FSP_FLAGS_POS_PAGE_SSIZE); - } - - ut_ad(fsp_flags_is_valid(flags)); - - return(flags); -} - -/** Initialize an FSP flags integer. -@param[in] page_size page sizes in bytes and compression flag. -@param[in] atomic_blobs Used by Dynammic and Compressed. -@param[in] has_data_dir This tablespace is in a remote location. -@param[in] is_shared This tablespace can be shared by many tables. -@param[in] is_temporary This tablespace is temporary. -@param[in] is_encrypted This tablespace is encrypted. -@param[in] page_compressed Table uses page compression -@param[in] page_compression_level Page compression level -@param[in] atomic_writes Table uses atomic writes -@@return tablespace flags after initialization */ -UNIV_INLINE -ulint -fsp_flags_init( - const page_size_t& page_size, - bool atomic_blobs, - bool has_data_dir, - bool is_shared, - bool is_temporary, - bool page_compression, - ulint page_compression_level, - ulint atomic_writes, - bool is_encrypted) -{ - ut_ad(page_size.physical() <= page_size.logical()); - ut_ad(!page_size.is_compressed() || atomic_blobs); - - /* Page size should be a power of two between UNIV_PAGE_SIZE_MIN - and UNIV_PAGE_SIZE, but zip_size may be 0 if not compressed. */ - ulint flags = fsp_flags_set_page_size(0, page_size); - - if (atomic_blobs) { - flags |= FSP_FLAGS_MASK_POST_ANTELOPE - | FSP_FLAGS_MASK_ATOMIC_BLOBS; - } - - /* If the zip_size is explicit and different from the default, - compressed row format is implied. */ - flags = fsp_flags_set_zip_size(flags, page_size); - - if (has_data_dir) { - flags |= FSP_FLAGS_MASK_DATA_DIR; - } - - /* Shared tablespaces can hold all row formats, so we only mark the - POST_ANTELOPE and ATOMIC_BLOB bits if it is compressed. */ - if (is_shared) { - ut_ad(!has_data_dir); - flags |= FSP_FLAGS_MASK_SHARED; - } - - if (is_temporary) { - ut_ad(!has_data_dir); - flags |= FSP_FLAGS_MASK_TEMPORARY; - } - - if (is_encrypted) { - flags |= FSP_FLAGS_MASK_ENCRYPTION; - } - - /* In addition, tablespace flags also contain if the page - compression is used for this table. */ - if (page_compression) { - flags |= FSP_FLAGS_SET_PAGE_COMPRESSION(flags, page_compression); - } - - /* In addition, tablespace flags also contain page compression level - if page compression is used for this table. */ - if (page_compression && page_compression_level) { - flags |= FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(flags, page_compression_level); - } - - /* In addition, tablespace flags also contain flag if atomic writes - is used for this table */ - if (atomic_writes) { - flags |= FSP_FLAGS_SET_ATOMIC_WRITES(flags, atomic_writes); - } - - ut_ad(fsp_flags_is_valid(flags)); - - return(flags); -} - /** Calculates the descriptor index within a descriptor page. @param[in] page_size page size @param[in] offset page offset @@ -337,15 +119,4 @@ xdes_calc_descriptor_page( return(ut_2pow_round(offset, page_size.physical())); } - -/** Calculates the descriptor array size. -@param[in] page_size page size -@return size of descriptor array */ -UNIV_INLINE -ulint -xdes_arr_size( - const page_size_t& page_size) -{ - return(page_size.physical()/FSP_EXTENT_SIZE); -} #endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/include/fsp0pagecompress.h b/storage/innobase/include/fsp0pagecompress.h index 44bdddfa3bf..b45a4b87890 100644 --- a/storage/innobase/include/fsp0pagecompress.h +++ b/storage/innobase/include/fsp0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -48,15 +48,6 @@ fsp_header_get_compression_level( const page_t* page); /*!< in: first page of a tablespace */ /********************************************************************//** -Determine if the tablespace is page compressed from dict_table_t::flags. -@return TRUE if page compressed, FALSE if not compressed */ -UNIV_INLINE -bool -fsp_flags_is_page_compressed( -/*=========================*/ - ulint flags); /*!< in: tablespace flags */ - -/********************************************************************//** Extract the page compression level from tablespace flags. A tablespace has only one physical page compression level whether that page is compressed or not. @@ -68,15 +59,6 @@ fsp_flags_get_page_compression_level( /*=================================*/ ulint flags); /*!< in: tablespace flags */ -/********************************************************************//** -Determine the tablespace is using atomic writes from dict_table_t::flags. -@return true if atomic writes is used, false if not */ -UNIV_INLINE -atomic_writes_t -fsp_flags_get_atomic_writes( -/*========================*/ - ulint flags); /*!< in: tablespace flags */ - #ifndef UNIV_NONINL #include "fsp0pagecompress.ic" #endif diff --git a/storage/innobase/include/fsp0pagecompress.ic b/storage/innobase/include/fsp0pagecompress.ic index a3971da6772..bb11371d712 100644 --- a/storage/innobase/include/fsp0pagecompress.ic +++ b/storage/innobase/include/fsp0pagecompress.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -26,18 +26,6 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@mariadb.com ***********************************************************************/ /********************************************************************//** -Determine if the tablespace is page compressed from dict_table_t::flags. -@return TRUE if page compressed, FALSE if not page compressed */ -UNIV_INLINE -bool -fsp_flags_is_page_compressed( -/*=========================*/ - ulint flags) /*!< in: tablespace flags */ -{ - return(FSP_FLAGS_GET_PAGE_COMPRESSION(flags)); -} - -/********************************************************************//** Determine the tablespace is page compression level from dict_table_t::flags. @return page compression level or 0 if not compressed*/ UNIV_INLINE @@ -49,17 +37,6 @@ fsp_flags_get_page_compression_level( return(FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags)); } -/********************************************************************//** -Determine the tablespace is using atomic writes from dict_table_t::flags. -@return true if atomic writes is used, false if not */ -UNIV_INLINE -atomic_writes_t -fsp_flags_get_atomic_writes( -/*========================*/ - ulint flags) /*!< in: tablespace flags */ -{ - return((atomic_writes_t)FSP_FLAGS_GET_ATOMIC_WRITES(flags)); -} /*******************************************************************//** Find out wheather the page is index page or not @@ -130,16 +107,10 @@ fil_space_is_page_compressed( /*=========================*/ ulint id) /*!< in: space id */ { - ulint flags; - - flags = fil_space_get_flags(id); - - if (flags && flags != ULINT_UNDEFINED) { - - return(fsp_flags_is_page_compressed(flags)); - } + ulint flags = fil_space_get_flags(id); - return(0); + return(flags != ULINT_UNDEFINED + && FSP_FLAGS_HAS_PAGE_COMPRESSION(flags)); } #endif /* UNIV_INNOCHECKSUM */ @@ -186,25 +157,28 @@ fil_get_compression_alg_name( /*******************************************************************//** Returns the atomic writes flag of the space, or false if the space is not using atomic writes. The tablespace must be cached in the memory cache. -@return atomic writes table option value */ +@return 1 if atomic writes can be used for the file */ UNIV_INLINE -atomic_writes_t +bool fil_space_get_atomic_writes( /*========================*/ ulint id) /*!< in: space id */ { - ulint flags; + struct fil_space_t* space; + bool ret= 0; - flags = fil_space_get_flags(id); + ut_ad(fil_system); - if (flags && flags != ULINT_UNDEFINED) { + mutex_enter(&fil_system->mutex); - return((atomic_writes_t)fsp_flags_get_atomic_writes(flags)); - } + if ((space = fil_space_get_by_id(id))) + ret= space->atomic_write_supported; - return((atomic_writes_t)0); + mutex_exit(&fil_system->mutex); + return(ret); } + /*******************************************************************//** Find out wheather the page is page compressed with lzo method @return true if page is page compressed with lzo method, false if not */ diff --git a/storage/innobase/include/fsp0space.h b/storage/innobase/include/fsp0space.h index 603c71b4aa6..6ecca6211ed 100644 --- a/storage/innobase/include/fsp0space.h +++ b/storage/innobase/include/fsp0space.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2013, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -18,7 +19,7 @@ this program; if not, write to the Free Software Foundation, Inc., /**************************************************//** @file include/fsp0space.h -General shared tablespace implementation. +Shared tablespace interface Created 2013-7-26 by Kevin Lewis *******************************************************/ @@ -61,10 +62,6 @@ public: shutdown(); ut_ad(m_files.empty()); ut_ad(m_space_id == ULINT_UNDEFINED); - if (m_name != NULL) { - ut_free(m_name); - m_name = NULL; - } if (m_path != NULL) { ut_free(m_path); m_path = NULL; @@ -75,21 +72,8 @@ public: Tablespace(const Tablespace&); Tablespace& operator=(const Tablespace&); - /** Set tablespace name - @param[in] name tablespace name */ - void set_name(const char* name) - { - ut_ad(m_name == NULL); - m_name = mem_strdup(name); - ut_ad(m_name != NULL); - } - - /** Get tablespace name - @return tablespace name */ - const char* name() const - { - return(m_name); - } + void set_name(const char* name) { m_name = name; } + const char* name() const { return m_name; } /** Set tablespace path and filename members. @param[in] path where tablespace file(s) resides @@ -219,7 +203,10 @@ public: /** Check if undo tablespace. @return true if undo tablespace */ - static bool is_undo_tablespace(ulint id); + static bool is_undo_tablespace(ulint id) + { + return(id <= srv_undo_tablespaces_open); + } private: /** @param[in] filename Name to lookup in the data files. @@ -233,7 +220,7 @@ private: /* DATA MEMBERS */ /** Name of the tablespace. */ - char* m_name; + const char* m_name; /** Tablespace ID */ ulint m_space_id; diff --git a/storage/innobase/include/fsp0sysspace.h b/storage/innobase/include/fsp0sysspace.h index 226d53ebd50..4c88b268f34 100644 --- a/storage/innobase/include/fsp0sysspace.h +++ b/storage/innobase/include/fsp0sysspace.h @@ -152,15 +152,6 @@ public: * ((1024 * 1024) / UNIV_PAGE_SIZE)); } - /** Roundoff to MegaBytes is similar as done in - SysTablespace::parse_units() function. - @return the pages when given size of file (bytes). */ - ulint get_pages_from_size(os_offset_t size) - { - return (ulint)((size / (1024 * 1024)) - * ((1024 * 1024) / UNIV_PAGE_SIZE)); - } - /** @return next increment size */ ulint get_increment() const; @@ -291,11 +282,9 @@ extern SysTablespace srv_tmp_space; @return true if id is a system tablespace, false if not. */ UNIV_INLINE bool -is_system_tablespace( - ulint id) +is_system_tablespace(ulint id) { - return(id == srv_sys_space.space_id() - || id == srv_tmp_space.space_id()); + return(id == TRX_SYS_SPACE || id == SRV_TMP_SPACE_ID); } /** Check if shared-system or undo tablespace. @@ -305,8 +294,7 @@ bool is_system_or_undo_tablespace( ulint id) { - return(id == srv_sys_space.space_id() - || id <= srv_undo_tablespaces_open); + return(id <= srv_undo_tablespaces_open); } /** Check if predefined shared tablespace. @@ -319,6 +307,6 @@ is_predefined_tablespace( ut_ad(srv_sys_space.space_id() == TRX_SYS_SPACE); ut_ad(TRX_SYS_SPACE == 0); return(id <= srv_undo_tablespaces_open - || id == srv_tmp_space.space_id()); + || id == SRV_TMP_SPACE_ID); } #endif /* fsp0sysspace_h */ diff --git a/storage/innobase/include/fsp0types.h b/storage/innobase/include/fsp0types.h index 21951acf72b..429af4210b7 100644 --- a/storage/innobase/include/fsp0types.h +++ b/storage/innobase/include/fsp0types.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2016, MariaDB Corporation. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -29,6 +29,12 @@ Created May 26, 2009 Vasil Dimov #ifndef UNIV_INNOCHECKSUM +/** The fil_space_t::id of the redo log. All persistent tablespaces +have a smaller fil_space_t::id. */ +#define SRV_LOG_SPACE_FIRST_ID 0xFFFFFFF0U +/** The fil_space_t::id of the innodb_temporary tablespace. */ +#define SRV_TMP_SPACE_ID 0xFFFFFFFEU + #include "univ.i" #include "ut0byte.h" @@ -181,24 +187,15 @@ every XDES_DESCRIBED_PER_PAGE pages in every tablespace. */ /*--------------------------------------*/ /* @} */ -/** Validate the tablespace flags. -These flags are stored in the tablespace header at offset FSP_SPACE_FLAGS. -They should be 0 for ROW_FORMAT=COMPACT and ROW_FORMAT=REDUNDANT. -The newer row formats, COMPRESSED and DYNAMIC, use a file format > Antelope -so they should have a file format number plus the DICT_TF_COMPACT bit set. -@param[in] flags Tablespace flags -@return true if valid, false if not */ -bool -fsp_flags_is_valid( - ulint flags) - MY_ATTRIBUTE((warn_unused_result, const)); - /** Check if tablespace is system temporary. @param[in] space_id verify is checksum is enabled for given space. @return true if tablespace is system temporary. */ +inline bool -fsp_is_system_temporary( - ulint space_id); +fsp_is_system_temporary(ulint space_id) +{ + return(space_id == SRV_TMP_SPACE_ID); +} /** Check if checksum is disabled for the given space. @param[in] space_id verify is checksum is enabled for given space. @@ -207,15 +204,6 @@ bool fsp_is_checksum_disabled( ulint space_id); -/** Check if tablespace is file-per-table. -@param[in] space_id Tablespace ID -@param[in] fsp_flags Tablespace Flags -@return true if tablespace is file-per-table. */ -bool -fsp_is_file_per_table( - ulint space_id, - ulint fsp_flags); - #ifdef UNIV_DEBUG /** Skip some of the sanity checks that are time consuming even in debug mode and can affect frequent verification runs that are done to ensure stability of @@ -240,43 +228,68 @@ to the two Barracuda row formats COMPRESSED and DYNAMIC. */ #define FSP_FLAGS_WIDTH_ATOMIC_BLOBS 1 /** Number of flag bits used to indicate the tablespace page size */ #define FSP_FLAGS_WIDTH_PAGE_SSIZE 4 -/** Width of the DATA_DIR flag. This flag indicates that the tablespace -is found in a remote location, not the default data directory. */ -#define FSP_FLAGS_WIDTH_DATA_DIR 1 -/** Width of the SHARED flag. This flag indicates that the tablespace -was created with CREATE TABLESPACE and can be shared by multiple tables. */ -#define FSP_FLAGS_WIDTH_SHARED 1 -/** Width of the TEMPORARY flag. This flag indicates that the tablespace -is a temporary tablespace and everything in it is temporary, meaning that -it is for a single client and should be deleted upon startup if it exists. */ -#define FSP_FLAGS_WIDTH_TEMPORARY 1 -/** Width of the encryption flag. This flag indicates that the tablespace -is a tablespace with encryption. */ -#define FSP_FLAGS_WIDTH_ENCRYPTION 1 - -/** Number of flag bits used to indicate the page compression and compression level */ -#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION 1 -#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL 4 - -/** Number of flag bits used to indicate atomic writes for this tablespace */ -#define FSP_FLAGS_WIDTH_ATOMIC_WRITES 2 - -/** Width of all the currently known tablespace flags */ +/** Number of reserved bits */ +#define FSP_FLAGS_WIDTH_RESERVED 6 +/** Number of flag bits used to indicate the page compression */ +#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION 1 + +/** Width of all the currently known persistent tablespace flags */ #define FSP_FLAGS_WIDTH (FSP_FLAGS_WIDTH_POST_ANTELOPE \ + FSP_FLAGS_WIDTH_ZIP_SSIZE \ + FSP_FLAGS_WIDTH_ATOMIC_BLOBS \ + FSP_FLAGS_WIDTH_PAGE_SSIZE \ - + FSP_FLAGS_WIDTH_DATA_DIR \ - + FSP_FLAGS_WIDTH_SHARED \ - + FSP_FLAGS_WIDTH_TEMPORARY \ - + FSP_FLAGS_WIDTH_ENCRYPTION \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES ) - -/** A mask of all the known/used bits in tablespace flags */ + + FSP_FLAGS_WIDTH_RESERVED \ + + FSP_FLAGS_WIDTH_PAGE_COMPRESSION) + +/** A mask of all the known/used bits in FSP_SPACE_FLAGS */ #define FSP_FLAGS_MASK (~(~0U << FSP_FLAGS_WIDTH)) +/* FSP_SPACE_FLAGS position and name in MySQL 5.6/MariaDB 10.0 or older +and MariaDB 10.1.20 or older MariaDB 10.1 and in MariaDB 10.1.21 +or newer. +MySQL 5.6 MariaDB 10.1.x MariaDB 10.1.21 +==================================================================== +Below flags in same offset +==================================================================== +0: POST_ANTELOPE 0:POST_ANTELOPE 0: POST_ANTELOPE +1..4: ZIP_SSIZE(0..5) 1..4:ZIP_SSIZE(0..5) 1..4: ZIP_SSIZE(0..5) +(NOTE: bit 4 is always 0) +5: ATOMIC_BLOBS 5:ATOMIC_BLOBS 5: ATOMIC_BLOBS +===================================================================== +Below note the order difference: +===================================================================== +6..9: PAGE_SSIZE(3..7) 6: COMPRESSION 6..9: PAGE_SSIZE(3..7) +10: DATA_DIR 7..10: COMP_LEVEL(0..9) 10: RESERVED (5.6 DATA_DIR) +===================================================================== +The flags below were in incorrect position in MariaDB 10.1, +or have been introduced in MySQL 5.7 or 8.0: +===================================================================== +11: UNUSED 11..12:ATOMIC_WRITES 11: RESERVED (5.7 SHARED) + 12: RESERVED (5.7 TEMPORARY) + 13..15:PAGE_SSIZE(3..7) 13: RESERVED (5.7 ENCRYPTION) + 14: RESERVED (8.0 SDI) + 15: RESERVED + 16: PAGE_SSIZE_msb(0) 16: COMPRESSION + 17: DATA_DIR 17: UNUSED + 18: UNUSED +===================================================================== +The flags below only exist in fil_space_t::flags, not in FSP_SPACE_FLAGS: +===================================================================== + 25: DATA_DIR + 26..27: ATOMIC_WRITES + 28..31: COMPRESSION_LEVEL +*/ + +/** A mask of the memory-only flags in fil_space_t::flags */ +#define FSP_FLAGS_MEM_MASK (~0U << FSP_FLAGS_MEM_DATA_DIR) + +/** Zero relative shift position of the DATA_DIR flag */ +#define FSP_FLAGS_MEM_DATA_DIR 25 +/** Zero relative shift position of the ATOMIC_WRITES field */ +#define FSP_FLAGS_MEM_ATOMIC_WRITES 26 +/** Zero relative shift position of the COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MEM_COMPRESSION_LEVEL 28 + /** Zero relative shift position of the POST_ANTELOPE field */ #define FSP_FLAGS_POS_POST_ANTELOPE 0 /** Zero relative shift position of the ZIP_SSIZE field */ @@ -285,34 +298,16 @@ is a tablespace with encryption. */ /** Zero relative shift position of the ATOMIC_BLOBS field */ #define FSP_FLAGS_POS_ATOMIC_BLOBS (FSP_FLAGS_POS_ZIP_SSIZE \ + FSP_FLAGS_WIDTH_ZIP_SSIZE) -/** Zero relative shift position of the PAGE_SSIZE field */ +/** Zero relative shift position of the start of the PAGE_SSIZE bits */ #define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_ATOMIC_BLOBS \ - + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) -/** Zero relative shift position of the start of the DATA_DIR bit */ -#define FSP_FLAGS_POS_DATA_DIR (FSP_FLAGS_POS_PAGE_SSIZE \ + + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) +/** Zero relative shift position of the start of the RESERVED bits +these are only used in MySQL 5.7 and used for compatibility. */ +#define FSP_FLAGS_POS_RESERVED (FSP_FLAGS_POS_PAGE_SSIZE \ + FSP_FLAGS_WIDTH_PAGE_SSIZE) -/** Zero relative shift position of the start of the SHARED bit */ -#define FSP_FLAGS_POS_SHARED (FSP_FLAGS_POS_DATA_DIR \ - + FSP_FLAGS_WIDTH_DATA_DIR) -/** Zero relative shift position of the start of the TEMPORARY bit */ -#define FSP_FLAGS_POS_TEMPORARY (FSP_FLAGS_POS_SHARED \ - + FSP_FLAGS_WIDTH_SHARED) -/** Zero relative shift position of the start of the ENCRYPTION bit */ -#define FSP_FLAGS_POS_ENCRYPTION (FSP_FLAGS_POS_TEMPORARY \ - + FSP_FLAGS_WIDTH_TEMPORARY) /** Zero relative shift position of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_POS_PAGE_COMPRESSION (FSP_FLAGS_POS_ENCRYPTION \ - + FSP_FLAGS_WIDTH_ENCRYPTION) -/** Zero relative shift position of the PAGE_COMPRESSION_LEVEL field */ -#define FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL (FSP_FLAGS_POS_PAGE_COMPRESSION \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION) -/** Zero relative shift position of the ATOMIC_WRITES field */ -#define FSP_FLAGS_POS_ATOMIC_WRITES (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL) -/** Zero relative shift position of the start of the UNUSED bits */ -#define FSP_FLAGS_POS_UNUSED (FSP_FLAGS_POS_ATOMIC_WRITES \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES) - +#define FSP_FLAGS_POS_PAGE_COMPRESSION (FSP_FLAGS_POS_RESERVED \ + + FSP_FLAGS_WIDTH_RESERVED) /** Bit mask of the POST_ANTELOPE field */ #define FSP_FLAGS_MASK_POST_ANTELOPE \ @@ -330,34 +325,22 @@ is a tablespace with encryption. */ #define FSP_FLAGS_MASK_PAGE_SSIZE \ ((~(~0U << FSP_FLAGS_WIDTH_PAGE_SSIZE)) \ << FSP_FLAGS_POS_PAGE_SSIZE) -/** Bit mask of the DATA_DIR field */ -#define FSP_FLAGS_MASK_DATA_DIR \ - ((~(~0U << FSP_FLAGS_WIDTH_DATA_DIR)) \ - << FSP_FLAGS_POS_DATA_DIR) -/** Bit mask of the SHARED field */ -#define FSP_FLAGS_MASK_SHARED \ - ((~(~0U << FSP_FLAGS_WIDTH_SHARED)) \ - << FSP_FLAGS_POS_SHARED) -/** Bit mask of the TEMPORARY field */ -#define FSP_FLAGS_MASK_TEMPORARY \ - ((~(~0U << FSP_FLAGS_WIDTH_TEMPORARY)) \ - << FSP_FLAGS_POS_TEMPORARY) -/** Bit mask of the ENCRYPTION field */ -#define FSP_FLAGS_MASK_ENCRYPTION \ - ((~(~0U << FSP_FLAGS_WIDTH_ENCRYPTION)) \ - << FSP_FLAGS_POS_ENCRYPTION) +/** Bit mask of the RESERVED1 field */ +#define FSP_FLAGS_MASK_RESERVED \ + ((~(~0U << FSP_FLAGS_WIDTH_RESERVED)) \ + << FSP_FLAGS_POS_RESERVED) /** Bit mask of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_MASK_PAGE_COMPRESSION \ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION \ ((~(~0U << FSP_FLAGS_WIDTH_PAGE_COMPRESSION)) \ << FSP_FLAGS_POS_PAGE_COMPRESSION) -/** Bit mask of the PAGE_COMPRESSION_LEVEL field */ -#define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL \ - ((~(~0U << FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL)) \ - << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) -/** Bit mask of the ATOMIC_WRITES field */ -#define FSP_FLAGS_MASK_ATOMIC_WRITES \ - ((~(~0U << FSP_FLAGS_WIDTH_ATOMIC_WRITES)) \ - << FSP_FLAGS_POS_ATOMIC_WRITES) + +/** Bit mask of the in-memory ATOMIC_WRITES field */ +#define FSP_FLAGS_MASK_MEM_ATOMIC_WRITES \ + (3U << FSP_FLAGS_MEM_ATOMIC_WRITES) + +/** Bit mask of the in-memory COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MASK_MEM_COMPRESSION_LEVEL \ + (15U << FSP_FLAGS_MEM_COMPRESSION_LEVEL) /** Return the value of the POST_ANTELOPE field */ #define FSP_FLAGS_GET_POST_ANTELOPE(flags) \ @@ -375,53 +358,88 @@ is a tablespace with encryption. */ #define FSP_FLAGS_GET_PAGE_SSIZE(flags) \ ((flags & FSP_FLAGS_MASK_PAGE_SSIZE) \ >> FSP_FLAGS_POS_PAGE_SSIZE) -/** Return the value of the DATA_DIR field */ -#define FSP_FLAGS_HAS_DATA_DIR(flags) \ - ((flags & FSP_FLAGS_MASK_DATA_DIR) \ - >> FSP_FLAGS_POS_DATA_DIR) -/** Return the contents of the SHARED field */ -#define FSP_FLAGS_GET_SHARED(flags) \ - ((flags & FSP_FLAGS_MASK_SHARED) \ - >> FSP_FLAGS_POS_SHARED) -/** Return the contents of the TEMPORARY field */ -#define FSP_FLAGS_GET_TEMPORARY(flags) \ - ((flags & FSP_FLAGS_MASK_TEMPORARY) \ - >> FSP_FLAGS_POS_TEMPORARY) -/** Return the contents of the ENCRYPTION field */ -#define FSP_FLAGS_GET_ENCRYPTION(flags) \ - ((flags & FSP_FLAGS_MASK_ENCRYPTION) \ - >> FSP_FLAGS_POS_ENCRYPTION) +/** @return the RESERVED flags */ +#define FSP_FLAGS_GET_RESERVED(flags) \ + ((flags & FSP_FLAGS_MASK_RESERVED) \ + >> FSP_FLAGS_POS_RESERVED) +/** @return the PAGE_COMPRESSION flag */ +#define FSP_FLAGS_HAS_PAGE_COMPRESSION(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION) + /** Return the contents of the UNUSED bits */ #define FSP_FLAGS_GET_UNUSED(flags) \ (flags >> FSP_FLAGS_POS_UNUSED) -/** Return the value of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_GET_PAGE_COMPRESSION(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION) \ - >> FSP_FLAGS_POS_PAGE_COMPRESSION) -/** Return the value of the PAGE_COMPRESSION_LEVEL field */ + +/** @return the value of the DATA_DIR field */ +#define FSP_FLAGS_HAS_DATA_DIR(flags) \ + (flags & 1U << FSP_FLAGS_MEM_DATA_DIR) +/** @return the COMPRESSION_LEVEL field */ #define FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL) \ - >> FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) -/** Return the value of the ATOMIC_WRITES field */ + ((flags & FSP_FLAGS_MASK_MEM_COMPRESSION_LEVEL) \ + >> FSP_FLAGS_MEM_COMPRESSION_LEVEL) +/** @return the ATOMIC_WRITES field */ #define FSP_FLAGS_GET_ATOMIC_WRITES(flags) \ - ((flags & FSP_FLAGS_MASK_ATOMIC_WRITES) \ - >> FSP_FLAGS_POS_ATOMIC_WRITES) -/** Use an alias in the code for FSP_FLAGS_GET_SHARED() */ -#define fsp_is_shared_tablespace FSP_FLAGS_GET_SHARED -/* @} */ + ((flags & FSP_FLAGS_MASK_MEM_ATOMIC_WRITES) \ + >> FSP_FLAGS_MEM_ATOMIC_WRITES) -/** Set a PAGE_COMPRESSION into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_COMPRESSION(flags, compression) \ - (flags | (compression << FSP_FLAGS_POS_PAGE_COMPRESSION)) +/* @} */ -/** Set a PAGE_COMPRESSION_LEVEL into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(flags, level) \ - (flags | (level << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL)) +/** Validate the tablespace flags, which are stored in the +tablespace header at offset FSP_SPACE_FLAGS. +@param[in] flags the contents of FSP_SPACE_FLAGS +@return whether the flags are correct (not in the buggy 10.1) format */ +MY_ATTRIBUTE((warn_unused_result, const)) +UNIV_INLINE +bool +fsp_flags_is_valid(ulint flags) +{ + DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", + return(false);); + if (flags == 0) { + return(true); + } + if (flags & ~FSP_FLAGS_MASK) { + return(false); + } + if ((flags & (FSP_FLAGS_MASK_POST_ANTELOPE | FSP_FLAGS_MASK_ATOMIC_BLOBS)) + == FSP_FLAGS_MASK_ATOMIC_BLOBS) { + /* If the "atomic blobs" flag (indicating + ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED) flag + is set, then the "post Antelope" (ROW_FORMAT!=REDUNDANT) flag + must also be set. */ + return(false); + } + /* Bits 10..14 should be 0b0000d where d is the DATA_DIR flag + of MySQL 5.6 and MariaDB 10.0, which we ignore. + In the buggy FSP_SPACE_FLAGS written by MariaDB 10.1.0 to 10.1.20, + bits 10..14 would be nonzero 0bsssaa where sss is + nonzero PAGE_SSIZE (3, 4, 6, or 7) + and aa is ATOMIC_WRITES (not 0b11). */ + if (FSP_FLAGS_GET_RESERVED(flags) & ~1) { + return(false); + } + + const ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); + if (ssize == 1 || ssize == 2 || ssize == 5 || ssize & 8) { + /* the page_size is not between 4k and 64k; + 16k should be encoded as 0, not 5 */ + return(false); + } + const ulint zssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); + if (zssize == 0) { + /* not ROW_FORMAT=COMPRESSED */ + } else if (zssize > (ssize ? ssize : 5)) { + /* invalid KEY_BLOCK_SIZE */ + return(false); + } else if (~flags & (FSP_FLAGS_MASK_POST_ANTELOPE + | FSP_FLAGS_MASK_ATOMIC_BLOBS)) { + /* both these flags should be set for + ROW_FORMAT=COMPRESSED */ + return(false); + } + + return(true); +} -/** Set a ATOMIC_WRITES into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_ATOMIC_WRITES(flags, atomics) \ - (flags | (atomics << FSP_FLAGS_POS_ATOMIC_WRITES)) #endif /* fsp0types_h */ diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 2d256472ef6..c6b39bce286 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -103,8 +104,8 @@ those defined in mysql file ft_global.h */ should not exceed FTS_DOC_ID_MAX_STEP */ #define FTS_DOC_ID_MAX_STEP 65535 -/** Maximum possible Fulltext word length */ -#define FTS_MAX_WORD_LEN HA_FT_MAXBYTELEN +/** Maximum possible Fulltext word length in bytes (assuming mbmaxlen=4) */ +#define FTS_MAX_WORD_LEN (HA_FT_MAXCHARLEN * 4) /** Maximum possible Fulltext word length (in characters) */ #define FTS_MAX_WORD_LEN_IN_CHAR HA_FT_MAXCHARLEN @@ -120,7 +121,6 @@ should not exceed FTS_DOC_ID_MAX_STEP */ #define FTS_CONFIG_TABLE_KEY_COL_LEN 50 #define FTS_CONFIG_TABLE_VALUE_COL_LEN 200 -#define FTS_INDEX_WORD_LEN FTS_MAX_WORD_LEN #define FTS_INDEX_FIRST_DOC_ID_LEN 8 #define FTS_INDEX_LAST_DOC_ID_LEN 8 #define FTS_INDEX_DOC_COUNT_LEN 4 diff --git a/storage/innobase/include/fts0types.h b/storage/innobase/include/fts0types.h index 039006265f6..5c6cd3ee56d 100644 --- a/storage/innobase/include/fts0types.h +++ b/storage/innobase/include/fts0types.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -127,7 +128,9 @@ struct fts_sync_t { bool in_progress; /*!< flag whether sync is in progress.*/ bool unlock_cache; /*!< flag whether unlock cache when write fts node */ - os_event_t event; /*!< sync finish event */ + os_event_t event; /*!< sync finish event; + only os_event_set() and os_event_wait() + are used */ }; /** The cache for the FTS system. It is a memory-based inverted index diff --git a/storage/innobase/include/fut0lst.h b/storage/innobase/include/fut0lst.h index 9c980d1358d..450def96227 100644 --- a/storage/innobase/include/fut0lst.h +++ b/storage/innobase/include/fut0lst.h @@ -48,7 +48,7 @@ typedef byte flst_node_t; /* The physical size of a list node in bytes */ #define FLST_NODE_SIZE (2 * FIL_ADDR_SIZE) -#if !defined UNIV_HOTBACKUP && !defined UNIV_INNOCHECKSUM +#ifndef UNIV_INNOCHECKSUM /********************************************************************//** Initializes a list base node. */ UNIV_INLINE @@ -99,29 +99,6 @@ flst_remove( flst_base_node_t* base, /*!< in: pointer to base node of list */ flst_node_t* node2, /*!< in: node to remove */ mtr_t* mtr); /*!< in: mini-transaction handle */ -/********************************************************************//** -Cuts off the tail of the list, including the node given. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -void -flst_cut_end( -/*=========*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node to remove */ - ulint n_nodes,/*!< in: number of nodes to remove, - must be >= 1 */ - mtr_t* mtr); /*!< in: mini-transaction handle */ -/********************************************************************//** -Cuts off the tail of the list, not including the given node. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -void -flst_truncate_end( -/*==============*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node not to remove */ - ulint n_nodes,/*!< in: number of nodes to remove */ - mtr_t* mtr); /*!< in: mini-transaction handle */ /** Get the length of a list. @param[in] base base node @return length */ @@ -204,6 +181,6 @@ flst_print( #include "fut0lst.ic" #endif -#endif /* !UNIV_HOTBACKUP && !UNIV_INNOCHECKSUM*/ +#endif /* !UNIV_INNOCHECKSUM */ #endif diff --git a/storage/innobase/include/ha0ha.h b/storage/innobase/include/ha0ha.h index 15a99ddf683..6857670a01b 100644 --- a/storage/innobase/include/ha0ha.h +++ b/storage/innobase/include/ha0ha.h @@ -174,7 +174,7 @@ ha_search_and_delete_if_found( hash_table_t* table, /*!< in: hash table */ ulint fold, /*!< in: folded value of the searched data */ const rec_t* data); /*!< in: pointer to the data */ -#ifndef UNIV_HOTBACKUP + /*****************************************************************//** Removes from the chain determined by fold all nodes whose data pointer points to the page given. */ @@ -202,7 +202,6 @@ ha_print_info( /*==========*/ FILE* file, /*!< in: file where to print */ hash_table_t* table); /*!< in: hash table */ -#endif /* !UNIV_HOTBACKUP */ /** The hash table external chain node */ struct ha_node_t { diff --git a/storage/innobase/include/ha_prototypes.h b/storage/innobase/include/ha_prototypes.h index f3641f93681..efcecf35117 100644 --- a/storage/innobase/include/ha_prototypes.h +++ b/storage/innobase/include/ha_prototypes.h @@ -31,7 +31,7 @@ simple headers. #include "univ.i" -#if !defined UNIV_HOTBACKUP && !defined UNIV_INNOCHECKSUM +#ifndef UNIV_INNOCHECKSUM /* Forward declarations */ class THD; @@ -41,18 +41,13 @@ struct fts_string_t; // JAN: TODO missing features: #undef MYSQL_57_SELECT_COUNT_OPTIMIZATION -#undef MYSQL_COMPRESSION -#undef MYSQL_ENCRYPTION #undef MYSQL_FT_INIT_EXT -#undef MYSQL_INNODB_API_CB #undef MYSQL_INNODB_PARTITIONING #undef MYSQL_PFS #undef MYSQL_RENAME_INDEX #undef MYSQL_REPLACE_TRX_IN_THD #undef MYSQL_SPATIAL_INDEX #undef MYSQL_STORE_FTS_DOC_ID -#undef MYSQL_TABLESPACES -#undef MYSQL_VIRTUAL_COLUMNS /*********************************************************************//** Wrapper around MySQL's copy_and_convert function. @@ -654,5 +649,20 @@ buffer pool size. void innodb_set_buf_pool_size(ulonglong buf_pool_size); -#endif /* !UNIV_HOTBACKUP && !UNIV_INNOCHECKSUM */ +/** Create a MYSQL_THD for background purge threads and mark it as such. +@returns new MYSQL_THD */ +MYSQL_THD +innobase_create_background_thd(); + +/** Destroy a background purge thread THD. +@param[in] thd MYSQL_THD to destroy */ +void +innobase_destroy_background_thd(MYSQL_THD); + +/** Close opened tables, free memory, delete items for a MYSQL_THD. +@param[in] thd MYSQL_THD to reset */ +void +innobase_reset_background_thd(MYSQL_THD); + +#endif /* !UNIV_INNOCHECKSUM */ #endif /* HA_INNODB_PROTOTYPES_H */ diff --git a/storage/innobase/include/hash0hash.h b/storage/innobase/include/hash0hash.h index a7bcee1185b..2922d424d37 100644 --- a/storage/innobase/include/hash0hash.h +++ b/storage/innobase/include/hash0hash.h @@ -28,9 +28,7 @@ Created 5/20/1997 Heikki Tuuri #include "univ.i" #include "mem0mem.h" -#ifndef UNIV_HOTBACKUP -# include "sync0rw.h" -#endif /* !UNIV_HOTBACKUP */ +#include "sync0rw.h" struct hash_table_t; struct hash_cell_t; @@ -60,7 +58,7 @@ hash_table_t* hash_create( /*========*/ ulint n); /*!< in: number of array cells */ -#ifndef UNIV_HOTBACKUP + /*************************************************************//** Creates a sync object array array to protect a hash table. ::sync_obj can be mutexes or rw_locks depening on the type of @@ -74,7 +72,6 @@ hash_create_sync_obj( latch_id_t id, /*!< in: mutex/rw_lock ID */ ulint n_sync_obj);/*!< in: number of sync objects, must be a power of 2 */ -#endif /* !UNIV_HOTBACKUP */ /*************************************************************//** Frees a hash table. */ @@ -91,15 +88,11 @@ hash_calc_hash( /*===========*/ ulint fold, /*!< in: folded value */ hash_table_t* table); /*!< in: hash table */ -#ifndef UNIV_HOTBACKUP /********************************************************************//** Assert that the mutex for the table is held */ -# define HASH_ASSERT_OWN(TABLE, FOLD) \ +#define HASH_ASSERT_OWN(TABLE, FOLD) \ ut_ad((TABLE)->type != HASH_TABLE_SYNC_MUTEX \ || (mutex_own(hash_get_mutex((TABLE), FOLD)))); -#else /* !UNIV_HOTBACKUP */ -# define HASH_ASSERT_OWN(TABLE, FOLD) -#endif /* !UNIV_HOTBACKUP */ /*******************************************************************//** Inserts a struct to a hash table. */ @@ -337,7 +330,6 @@ do {\ mem_heap_free_top(hash_get_heap(TABLE, fold111), sizeof(TYPE));\ } while (0) -#ifndef UNIV_HOTBACKUP /****************************************************************//** Move all hash table entries from OLD_TABLE to NEW_TABLE. */ @@ -537,22 +529,6 @@ hash_unlock_x_all_but( hash_table_t* table, /*!< in: hash table */ rw_lock_t* keep_lock); /*!< in: lock to keep */ -#else /* !UNIV_HOTBACKUP */ -# define hash_get_heap(table, fold) ((table)->heap) -# define hash_mutex_enter(table, fold) ((void) 0) -# define hash_mutex_exit(table, fold) ((void) 0) -# define hash_mutex_enter_all(table) ((void) 0) -# define hash_mutex_exit_all(table) ((void) 0) -# define hash_mutex_exit_all_but(t, m) ((void) 0) -# define hash_lock_s(t, f) ((void) 0) -# define hash_lock_x(t, f) ((void) 0) -# define hash_unlock_s(t, f) ((void) 0) -# define hash_unlock_x(t, f) ((void) 0) -# define hash_lock_x_all(t) ((void) 0) -# define hash_unlock_x_all(t) ((void) 0) -# define hash_unlock_x_all_but(t, l) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ - struct hash_cell_t{ void* node; /*!< hash chain node, NULL if none */ }; @@ -561,15 +537,13 @@ struct hash_cell_t{ struct hash_table_t { enum hash_table_sync_t type; /*<! type of hash_table. */ #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG -# ifndef UNIV_HOTBACKUP ibool adaptive;/* TRUE if this is the hash table of the adaptive hash index */ -# endif /* !UNIV_HOTBACKUP */ #endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ ulint n_cells;/* number of cells in the hash table */ hash_cell_t* array; /*!< pointer to cell array */ -#ifndef UNIV_HOTBACKUP + ulint n_sync_obj;/* if sync_objs != NULL, then the number of either the number of mutexes or the number of @@ -589,7 +563,6 @@ struct hash_table_t { can be allocated from these memory heaps; there are then n_mutexes many of these heaps */ -#endif /* !UNIV_HOTBACKUP */ mem_heap_t* heap; #ifdef UNIV_DEBUG ulint magic_n; diff --git a/storage/innobase/include/hash0hash.ic b/storage/innobase/include/hash0hash.ic index b99ac1eb501..8a2e4949694 100644 --- a/storage/innobase/include/hash0hash.ic +++ b/storage/innobase/include/hash0hash.ic @@ -85,7 +85,6 @@ hash_calc_hash( return(ut_hash_ulint(fold, table->n_cells)); } -#ifndef UNIV_HOTBACKUP /************************************************************//** Gets the sync object index for a fold value in a hash table. @return index */ @@ -276,4 +275,3 @@ hash_lock_x_confirm( return(hash_lock); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/ibuf0ibuf.h b/storage/innobase/include/ibuf0ibuf.h index f3fd5e9a364..e646f678964 100644 --- a/storage/innobase/include/ibuf0ibuf.h +++ b/storage/innobase/include/ibuf0ibuf.h @@ -32,9 +32,7 @@ Created 7/19/1997 Heikki Tuuri #include "mtr0mtr.h" #include "dict0mem.h" #include "fsp0fsp.h" - -#ifndef UNIV_HOTBACKUP -# include "ibuf0types.h" +#include "ibuf0types.h" /** Default value for maximum on-disk size of change buffer in terms of percentage of the buffer pool. */ @@ -377,7 +375,6 @@ ibuf_merge_space( /*=============*/ ulint space); /*!< in: space id */ -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Parses a redo log record of an ibuf bitmap page init. @return end of log record or NULL */ @@ -388,9 +385,8 @@ ibuf_parse_bitmap_init( byte* end_ptr,/*!< in: buffer end */ buf_block_t* block, /*!< in: block or NULL */ mtr_t* mtr); /*!< in: mtr or NULL */ -#ifndef UNIV_HOTBACKUP -#ifdef UNIV_IBUF_COUNT_DEBUG +#ifdef UNIV_IBUF_COUNT_DEBUG /** Gets the ibuf count for a given page. @param[in] page_id page id @return number of entries in the insert buffer currently buffered for @@ -398,7 +394,6 @@ this page */ ulint ibuf_count_get( const page_id_t& page_id); - #endif /******************************************************************//** Looks if the insert buffer is empty. @@ -447,8 +442,6 @@ ibuf_set_bitmap_for_bulk_load( #define IBUF_HEADER_PAGE_NO FSP_IBUF_HEADER_PAGE_NO #define IBUF_TREE_ROOT_PAGE_NO FSP_IBUF_TREE_ROOT_PAGE_NO -#endif /* !UNIV_HOTBACKUP */ - /* The ibuf header page currently contains only the file segment header for the file segment from which the pages for the ibuf tree are allocated */ #define IBUF_HEADER PAGE_DATA diff --git a/storage/innobase/include/ibuf0ibuf.ic b/storage/innobase/include/ibuf0ibuf.ic index de39592ae6b..09070c14059 100644 --- a/storage/innobase/include/ibuf0ibuf.ic +++ b/storage/innobase/include/ibuf0ibuf.ic @@ -26,7 +26,6 @@ Created 7/19/1997 Heikki Tuuri #include "page0page.h" #include "page0zip.h" #include "fsp0types.h" -#ifndef UNIV_HOTBACKUP #include "buf0lru.h" /** An index page must contain at least UNIV_PAGE_SIZE / @@ -340,4 +339,3 @@ ibuf_update_free_bits_if_full( ibuf_set_free_bits(block, after, before); } } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/lock0lock.h b/storage/innobase/include/lock0lock.h index f6bbd571907..d96ba0d9c7d 100644 --- a/storage/innobase/include/lock0lock.h +++ b/storage/innobase/include/lock0lock.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -892,8 +893,7 @@ lock_trx_handle_wait( /*=================*/ trx_t* trx, /*!< in/out: trx lock state */ bool lock_mutex_taken, - bool trx_mutex_taken) - MY_ATTRIBUTE((nonnull(1), warn_unused_result)); + bool trx_mutex_taken); /*********************************************************************//** Get the number of locks on a table. @return number of locks */ @@ -1040,7 +1040,12 @@ struct lock_sys_t{ srv_slot_t* waiting_threads; /*!< Array of user threads suspended while waiting for locks within InnoDB, protected - by the lock_sys->wait_mutex */ + by the lock_sys->wait_mutex; + os_event_set() and + os_event_reset() on + waiting_threads[]->event + are protected by + trx_t::mutex */ srv_slot_t* last_slot; /*!< highest slot ever used in the waiting_threads array, protected by @@ -1053,10 +1058,11 @@ struct lock_sys_t{ ulint n_lock_max_wait_time; /*!< Max wait time */ - os_event_t timeout_event; /*!< Set to the event that is - created in the lock wait monitor - thread. A value of 0 means the - thread is not active */ + os_event_t timeout_event; /*!< An event waited for by + lock_wait_timeout_thread. + Not protected by a mutex, + but the waits are timed. + Signaled on shutdown only. */ bool timeout_thread_active; /*!< True if the timeout thread is running */ diff --git a/storage/innobase/include/log0crypt.h b/storage/innobase/include/log0crypt.h index 6b164e90d6e..d1282043665 100644 --- a/storage/innobase/include/log0crypt.h +++ b/storage/innobase/include/log0crypt.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -22,27 +22,21 @@ Innodb log encrypt/decrypt Created 11/25/2013 Minli Zhu Modified Jan Lindström jan.lindstrom@mariadb.com +MDEV-11782: Rewritten for MariaDB 10.2 by Marko Mäkelä, MariaDB Corporation. *******************************************************/ #ifndef log0crypt_h #define log0crypt_h -#include "univ.i" -#include "ut0byte.h" -#include "my_crypt.h" +#include "log0log.h" -typedef int Crypt_result; - -/* If true, enable redo log encryption. */ +/** innodb_encrypt_log: whether to encrypt the redo log */ extern my_bool srv_encrypt_log; -/*********************************************************************** -Set next checkpoint's key version to latest one, and generate new key */ +/** Initialize the redo log encryption key. +@return whether the operation succeeded */ UNIV_INTERN -void -log_crypt_set_ver_and_key( -/*======================*/ - ib_uint64_t next_checkpoint_no);/*!< in: next checkpoint no */ - +bool +log_crypt_init(); /*********************************************************************//** Writes the crypto (version, msg and iv) info, which has been used for @@ -54,75 +48,33 @@ log_crypt_write_checkpoint_buf( /*===========================*/ byte* buf); /*!< in/out: checkpoint buffer */ -/*********************************************************************//** -Read the crypto (version, msg and iv) info, which has been used for -log blocks with lsn <= this checkpoint's lsn, from a log header's -checkpoint buf. */ +/** Read the MariaDB 10.1 checkpoint crypto (version, msg and iv) info. +@param[in] buf checkpoint buffer +@return whether the operation was successful */ UNIV_INTERN bool -log_crypt_read_checkpoint_buf( -/*===========================*/ - const byte* buf); /*!< in: checkpoint buffer */ +log_crypt_101_read_checkpoint(const byte* buf); -/******************************************************** -Encrypt one or more log block before it is flushed to disk */ +/** Decrypt a MariaDB 10.1 redo log block. +@param[in,out] buf log block +@return whether the decryption was successful */ UNIV_INTERN -void -log_encrypt_before_write( -/*=====================*/ - ib_uint64_t next_checkpoint_no, /*!< in: log group to be flushed */ - byte* block, /*!< in/out: pointer to a log block */ - const ulint size); /*!< in: size of log blocks */ - -/******************************************************** -Decrypt a specified log segment after they are read from a log file to a buffer. -*/ -UNIV_INTERN -void -log_decrypt_after_read( -/*===================*/ - byte* frame, /*!< in/out: log segment */ - const ulint size); /*!< in: log segment size */ - -/* Error codes for crypt info */ -typedef enum { - LOG_UNENCRYPTED = 0, - LOG_CRYPT_KEY_NOT_FOUND = 1, - LOG_DECRYPT_MAYBE_FAILED = 2 -} log_crypt_err_t; +bool +log_crypt_101_read_block(byte* buf); -/******************************************************** -Check is the checkpoint information encrypted. This check -is based on fact has log group crypt info and based -on this crypt info was the key version different from -unencrypted key version. There is no realible way to -distinguish encrypted log block from corrupted log block, -but if log block corruption is found this function is -used to find out if log block is maybe encrypted but -encryption key, key management plugin or encryption -algorithm does not match. -@return TRUE, if log block may be encrypted */ +/** Read the checkpoint crypto (version, msg and iv) info. +@param[in] buf checkpoint buffer +@return whether the operation was successful */ UNIV_INTERN -ibool -log_crypt_block_maybe_encrypted( -/*============================*/ - const byte* log_block, /*!< in: log block */ - log_crypt_err_t* err_info); /*!< out: error info */ +bool +log_crypt_read_checkpoint_buf(const byte* buf); -/******************************************************** -Print crypt error message to error log */ +/** Encrypt or decrypt log blocks. +@param[in,out] buf log blocks to encrypt or decrypt +@param[in] size size of the buffer, in bytes +@param[in] decrypt whether to decrypt instead of encrypting */ UNIV_INTERN void -log_crypt_print_error( -/*==================*/ - log_crypt_err_t err_info); /*!< out: error info */ - -/*********************************************************************//** -Print checkpoint no from log block and all encryption keys from -checkpoints if they are present. Used for problem analysis. */ -void -log_crypt_print_checkpoint_keys( -/*============================*/ - const byte* log_block); +log_crypt(byte* buf, ulint size, bool decrypt = false); #endif // log0crypt.h diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index caa067cd4ba..956cfffbad8 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -2,6 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2009, Google Inc. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -35,14 +36,9 @@ Created 12/9/1995 Heikki Tuuri #include "univ.i" #include "dyn0buf.h" -#ifndef UNIV_HOTBACKUP #include "sync0rw.h" -#endif /* !UNIV_HOTBACKUP */ -#include "log0crypt.h" #include "log0types.h" - -/** Redo log buffer */ -struct log_t; +#include "os0event.h" /** Redo log group */ struct log_group_t; @@ -72,7 +68,6 @@ log_calc_where_lsn_is( files */ int64_t log_file_size); /*!< in: log file size (including the header) */ -#ifndef UNIV_HOTBACKUP /** Append a string to the log. @param[in] str string @param[in] len string length @@ -268,19 +263,6 @@ log_write_checkpoint_info( mtr_buf_t* log_append_on_checkpoint( mtr_buf_t* buf); -#else /* !UNIV_HOTBACKUP */ -/******************************************************//** -Writes info to a buffer of a log group when log files are created in -backup restoration. */ -void -log_reset_first_header_and_checkpoint( -/*==================================*/ - byte* hdr_buf,/*!< in: buffer which will be written to the - start of the first log file */ - ib_uint64_t start); /*!< in: lsn of the start of the first log file; - we pretend that there is a checkpoint at - start + LOG_BLOCK_HDR_SIZE */ -#endif /* !UNIV_HOTBACKUP */ /** Checks that there is enough free space in the log to start a new query step. Flushes the log buffer or makes a new checkpoint if necessary. NOTE: this @@ -288,16 +270,7 @@ function may only be called if the calling thread owns no synchronization objects! */ void log_check_margins(void); -#ifndef UNIV_HOTBACKUP -/******************************************************//** -Reads a specified log segment to a buffer. */ -void -log_group_read_log_seg( -/*===================*/ - byte* buf, /*!< in: buffer where to read */ - log_group_t* group, /*!< in: log group */ - lsn_t start_lsn, /*!< in: read area start */ - lsn_t end_lsn); /*!< in: read area end */ + /********************************************************//** Sets the field values in group to correspond to a given lsn. For this function to work, the values must already be correctly initialized to correspond to @@ -316,7 +289,6 @@ lsn_t log_group_get_capacity( /*===================*/ const log_group_t* group); /*!< in: log group */ -#endif /* !UNIV_HOTBACKUP */ /************************************************************//** Gets a log block flush bit. @return TRUE if this block was the first to be written in a log flush */ @@ -422,17 +394,6 @@ log_block_init( /*===========*/ byte* log_block, /*!< in: pointer to the log buffer */ lsn_t lsn); /*!< in: lsn within the log block */ -#ifdef UNIV_HOTBACKUP -/************************************************************//** -Initializes a log block in the log buffer in the old, < 3.23.52 format, where -there was no checksum yet. */ -UNIV_INLINE -void -log_block_init_in_old_format( -/*=========================*/ - byte* log_block, /*!< in: pointer to the log buffer */ - lsn_t lsn); /*!< in: lsn within the log block */ -#endif /* UNIV_HOTBACKUP */ /************************************************************//** Converts a lsn to a log block number. @return log block number, it is > 0 and <= 1G */ @@ -475,9 +436,6 @@ void log_mem_free(void); /*==============*/ -/** Redo log system */ -extern log_t* log_sys; - /** Whether to generate and require checksums on the redo log pages */ extern my_bool innodb_log_checksums; @@ -534,6 +492,12 @@ extern my_bool innodb_log_checksums; #define LOG_CHECKPOINT_LSN 8 #define LOG_CHECKPOINT_OFFSET 16 #define LOG_CHECKPOINT_LOG_BUF_SIZE 24 +/** MariaDB 10.2.5 encrypted redo log encryption key version (32 bits)*/ +#define LOG_CHECKPOINT_CRYPT_KEY 32 +/** MariaDB 10.2.5 encrypted redo log random nonce (32 bits) */ +#define LOG_CHECKPOINT_CRYPT_NONCE 36 +/** MariaDB 10.2.5 encrypted redo log random message (MY_AES_BLOCK_SIZE) */ +#define LOG_CHECKPOINT_CRYPT_MESSAGE 40 /** Offsets of a log file header */ /* @{ */ @@ -564,19 +528,9 @@ or the MySQL version that created the redo log file. */ /** The redo log format identifier corresponding to the current format version. Stored in LOG_HEADER_FORMAT. */ #define LOG_HEADER_FORMAT_CURRENT 1 +/** Encrypted MariaDB redo log */ +#define LOG_HEADER_FORMAT_ENCRYPTED (1U<<31) -// JAN: TODO: Shoud 32 here be LOG_HEADER_CREATOR_END ? -// Problem: Log format 5.6 == 5.7 ? -#define LOG_CHECKPOINT_ARRAY_END (32 + 32 * 8) -#define LOG_CRYPT_VER (20 + LOG_CHECKPOINT_ARRAY_END) -#define LOG_CRYPT_MAX_ENTRIES (5) -#define LOG_CRYPT_ENTRY_SIZE (4 + 4 + 2 * MY_AES_BLOCK_SIZE) -#define LOG_CRYPT_SIZE (1 + 1 + \ - (LOG_CRYPT_MAX_ENTRIES * \ - LOG_CRYPT_ENTRY_SIZE)) - -#define LOG_CHECKPOINT_SIZE (20 + LOG_CHECKPOINT_ARRAY_END + \ - LOG_CRYPT_SIZE) /* @} */ #define LOG_CHECKPOINT_1 OS_FILE_LOG_BLOCK_SIZE @@ -637,6 +591,12 @@ struct log_group_t{ byte* checkpoint_buf; /** list of log groups */ UT_LIST_NODE_T(log_group_t) log_groups; + + /** @return whether the redo log is encrypted */ + bool is_encrypted() const + { + return((format & LOG_HEADER_FORMAT_ENCRYPTED) != 0); + } }; /** Redo log buffer */ @@ -648,7 +608,7 @@ struct log_t{ lsn_t lsn; /*!< log sequence number */ ulint buf_free; /*!< first free offset within the log buffer in use */ -#ifndef UNIV_HOTBACKUP + char pad2[CACHE_LINE_SIZE];/*!< Padding */ LogSysMutex mutex; /*!< mutex protecting the log */ char pad3[CACHE_LINE_SIZE]; /*!< Padding */ @@ -663,7 +623,6 @@ struct log_t{ mtr_commit and still ensure that insertions in the flush_list happen in the LSN order. */ -#endif /* !UNIV_HOTBACKUP */ byte* buf_ptr; /*!< unaligned log buffer, which should be of double of buf_size */ byte* buf; /*!< log buffer currently in use; @@ -692,7 +651,6 @@ struct log_t{ UT_LIST_BASE_NODE_T(log_group_t) log_groups; /*!< log groups */ -#ifndef UNIV_HOTBACKUP /** The fields involved in the log buffer flush @{ */ ulint buf_next_to_write;/*!< first offset in the log buffer @@ -711,16 +669,12 @@ struct log_t{ /*!< how far we have written the log AND flushed to disk */ ulint n_pending_flushes;/*!< number of currently - pending flushes; incrementing is - protected by the log mutex; - may be decremented between - resetting and setting flush_event */ + pending flushes; protected by + log_sys_t::mutex */ os_event_t flush_event; /*!< this event is in the reset state - when a flush is running; a thread - should wait for this without - owning the log mutex, but NOTE that - to set this event, the - thread MUST own the log mutex! */ + when a flush is running; + os_event_set() and os_event_reset() + are protected by log_sys_t::mutex */ ulint n_log_ios; /*!< number of log i/os initiated thus far */ ulint n_log_ios_old; /*!< number of log i/o's at the @@ -776,13 +730,21 @@ struct log_t{ checkpoint write is running; a thread should wait for this without owning the log mutex */ -#endif /* !UNIV_HOTBACKUP */ byte* checkpoint_buf_ptr;/* unaligned checkpoint header */ byte* checkpoint_buf; /*!< checkpoint header is read to this buffer */ /* @} */ + + /** @return whether the redo log is encrypted */ + bool is_encrypted() const + { + return(UT_LIST_GET_FIRST(log_groups)->is_encrypted()); + } }; +/** Redo log system */ +extern log_t* log_sys; + /** Test if flush order mutex is owned. */ #define log_flush_order_mutex_own() \ mutex_own(&log_sys->log_flush_order_mutex) @@ -835,21 +797,13 @@ log_group_calc_lsn_offset( lsn_t lsn, const log_group_t* group); -extern os_event_t log_scrub_event; /* log scrubbing speed, in bytes/sec */ extern ulonglong innodb_scrub_log_speed; -/*****************************************************************//** -This is the main thread for log scrub. It waits for an event and -when waked up fills current log block with dummy records and -sleeps again. -@return this function does not return, it calls os_thread_exit() */ -extern "C" UNIV_INTERN -os_thread_ret_t -DECLARE_THREAD(log_scrub_thread)( -/*===============================*/ - void* arg); /*!< in: a dummy parameter - required by os_thread_create */ +/** Event to wake up log_scrub_thread */ +extern os_event_t log_scrub_event; +/** Whether log_scrub_thread is active */ +extern bool log_scrub_thread_active; #ifndef UNIV_NONINL #include "log0log.ic" diff --git a/storage/innobase/include/log0log.ic b/storage/innobase/include/log0log.ic index a53f8770cea..82a94265776 100644 --- a/storage/innobase/include/log0log.ic +++ b/storage/innobase/include/log0log.ic @@ -297,32 +297,6 @@ log_block_init( log_block_set_first_rec_group(log_block, 0); } -#ifdef UNIV_HOTBACKUP -/************************************************************//** -Initializes a log block in the log buffer in the old format, where there -was no checksum yet. */ -UNIV_INLINE -void -log_block_init_in_old_format( -/*=========================*/ - byte* log_block, /*!< in: pointer to the log buffer */ - lsn_t lsn) /*!< in: lsn within the log block */ -{ - ulint no; - - ut_ad(log_mutex_own()); - - no = log_block_convert_lsn_to_no(lsn); - - log_block_set_hdr_no(log_block, no); - mach_write_to_4(log_block + OS_FILE_LOG_BLOCK_SIZE - - LOG_BLOCK_CHECKSUM, no); - log_block_set_data_len(log_block, LOG_BLOCK_HDR_SIZE); - log_block_set_first_rec_group(log_block, 0); -} -#endif /* UNIV_HOTBACKUP */ - -#ifndef UNIV_HOTBACKUP /** Append a string to the log. @param[in] str string @param[in] len string length @@ -529,4 +503,3 @@ log_free_check(void) log_check_margins(); } } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h index bd7118654f3..24a83ec2ab1 100644 --- a/storage/innobase/include/log0recv.h +++ b/storage/innobase/include/log0recv.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -37,83 +38,16 @@ Created 9/20/1997 Heikki Tuuri #include <list> #include <vector> -#ifdef UNIV_HOTBACKUP -extern bool recv_replay_file_ops; +/** @return whether recovery is currently running. */ +#define recv_recovery_is_on() recv_recovery_on -/*******************************************************************//** -Reads the checkpoint info needed in hot backup. -@return TRUE if success */ -ibool -recv_read_checkpoint_info_for_backup( -/*=================================*/ - const byte* hdr, /*!< in: buffer containing the log group - header */ - lsn_t* lsn, /*!< out: checkpoint lsn */ - lsn_t* offset, /*!< out: checkpoint offset in the log group */ - lsn_t* cp_no, /*!< out: checkpoint number */ - lsn_t* first_header_lsn) - /*!< out: lsn of of the start of the - first log file */ - MY_ATTRIBUTE((nonnull)); -/*******************************************************************//** -Scans the log segment and n_bytes_scanned is set to the length of valid -log scanned. */ -void -recv_scan_log_seg_for_backup( -/*=========================*/ - byte* buf, /*!< in: buffer containing log data */ - ulint buf_len, /*!< in: data length in that buffer */ - lsn_t* scanned_lsn, /*!< in/out: lsn of buffer start, - we return scanned lsn */ - ulint* scanned_checkpoint_no, - /*!< in/out: 4 lowest bytes of the - highest scanned checkpoint number so - far */ - ulint* n_bytes_scanned);/*!< out: how much we were able to - scan, smaller than buf_len if log - data ended here */ -#endif /* UNIV_HOTBACKUP */ -/*******************************************************************//** -Returns TRUE if recovery is currently running. -@return recv_recovery_on */ -UNIV_INLINE -bool -recv_recovery_is_on(void); -/*=====================*/ -/************************************************************************//** -Applies the hashed log records to the page, if the page lsn is less than the -lsn of a log record. This can be called when a buffer page has just been -read in, or also for a page already in the buffer pool. */ +/** Apply the hashed log records to the page, if the page lsn is less than the +lsn of a log record. +@param just_read_in whether the page recently arrived to the I/O handler +@param block the page in the buffer pool */ void -recv_recover_page_func( -/*===================*/ -#ifndef UNIV_HOTBACKUP - ibool just_read_in, - /*!< in: TRUE if the i/o handler calls - this for a freshly read page */ -#endif /* !UNIV_HOTBACKUP */ - buf_block_t* block); /*!< in/out: buffer block */ -#ifndef UNIV_HOTBACKUP -/** Wrapper for recv_recover_page_func(). -Applies the hashed log records to the page, if the page lsn is less than the -lsn of a log record. This can be called when a buffer page has just been -read in, or also for a page already in the buffer pool. -@param jri in: TRUE if just read in (the i/o handler calls this for -a freshly read page) -@param block in/out: the buffer block -*/ -# define recv_recover_page(jri, block) recv_recover_page_func(jri, block) -#else /* !UNIV_HOTBACKUP */ -/** Wrapper for recv_recover_page_func(). -Applies the hashed log records to the page, if the page lsn is less than the -lsn of a log record. This can be called when a buffer page has just been -read in, or also for a page already in the buffer pool. -@param jri in: TRUE if just read in (the i/o handler calls this for -a freshly read page) -@param block in/out: the buffer block -*/ -# define recv_recover_page(jri, block) recv_recover_page_func(block) -#endif /* !UNIV_HOTBACKUP */ +recv_recover_page(bool just_read_in, buf_block_t* block); + /** Start recovering from a redo log checkpoint. @see recv_recovery_from_checkpoint_finish @param[in] flush_lsn FIL_PAGE_FILE_FLUSH_LSN @@ -140,18 +74,6 @@ recv_reset_logs( OS_FILE_LOG_BLOCK_SIZE, after which we add LOG_BLOCK_HDR_SIZE */ -#ifdef UNIV_HOTBACKUP -/******************************************************//** -Creates new log files after a backup has been restored. */ -void -recv_reset_log_files_for_backup( -/*============================*/ - const char* log_dir, /*!< in: log file directory path */ - ulint n_log_files, /*!< in: number of log files */ - lsn_t log_file_size, /*!< in: log file size */ - lsn_t lsn); /*!< in: new start lsn, must be - divisible by OS_FILE_LOG_BLOCK_SIZE */ -#endif /* UNIV_HOTBACKUP */ /********************************************************//** Creates the recovery system. */ void @@ -173,7 +95,6 @@ void recv_sys_init( /*==========*/ ulint available_memory); /*!< in: available memory in bytes */ -#ifndef UNIV_HOTBACKUP /********************************************************//** Frees the recovery system. */ void @@ -184,7 +105,6 @@ Reset the state of the recovery system variables. */ void recv_sys_var_init(void); /*===================*/ -#endif /* !UNIV_HOTBACKUP */ /*******************************************************************//** Empties the hash table of stored log records, applying them to appropriate pages. */ @@ -200,14 +120,6 @@ recv_apply_hashed_log_recs( can be generated during the application */ __attribute__((warn_unused_result)); -#ifdef UNIV_HOTBACKUP -/*******************************************************************//** -Applies log records in the hash table to a backup. */ -void -recv_apply_log_recs_for_backup(void); -/*================================*/ -#endif /* UNIV_HOTBACKUP */ - /** Block of log record data */ struct recv_data_t{ recv_data_t* next; /*!< pointer to the next block or NULL */ @@ -262,7 +174,7 @@ struct recv_addr_t{ struct recv_dblwr_t { /** Add a page frame to the doublewrite recovery buffer. */ - void add(const byte* page) { + void add(byte* page) { pages.push_back(page); } @@ -273,25 +185,14 @@ struct recv_dblwr_t { @retval NULL if no page was found */ const byte* find_page(ulint space_id, ulint page_no); - typedef std::list<const byte*, ut_allocator<const byte*> > list; + typedef std::list<byte*, ut_allocator<byte*> > list; /** Recovered doublewrite buffer page frames */ list pages; }; -/* Recovery encryption information */ -typedef struct recv_encryption { - ulint space_id; /*!< the page number */ - byte* key; /*!< encryption key */ - byte* iv; /*!< encryption iv */ -} recv_encryption_t; - -typedef std::vector<recv_encryption_t, ut_allocator<recv_encryption_t> > - encryption_list_t; - /** Recovery system data structure */ struct recv_sys_t{ -#ifndef UNIV_HOTBACKUP ib_mutex_t mutex; /*!< mutex protecting the fields apply_log_recs, n_addrs, and the state field in each recv_addr struct */ @@ -305,7 +206,6 @@ struct recv_sys_t{ buf_flush_t flush_type;/*!< type of the flush request. BUF_FLUSH_LRU: flush end of LRU, keeping free blocks. BUF_FLUSH_LIST: flush all of blocks. */ -#endif /* !UNIV_HOTBACKUP */ ibool apply_log_recs; /*!< this is TRUE when log rec application to pages is allowed; this flag tells the @@ -314,12 +214,6 @@ struct recv_sys_t{ ibool apply_batch_on; /*!< this is TRUE when a log rec application batch is running */ - byte* last_block; - /*!< possible incomplete last recovered log - block */ - byte* last_block_buf_start; - /*!< the nonaligned start address of the - preceding buffer */ byte* buf; /*!< buffer for parsing log records */ ulint len; /*!< amount of data in buf */ lsn_t parse_start_lsn; @@ -357,9 +251,6 @@ struct recv_sys_t{ addresses in the hash table */ recv_dblwr_t dblwr; - - encryption_list_t* /*!< Encryption information list */ - encryption_list; }; /** The recovery system */ @@ -391,15 +282,6 @@ extern bool recv_no_log_write; number (FIL_PAGE_LSN) is in the future. Initially FALSE, and set by recv_recovery_from_checkpoint_start(). */ extern bool recv_lsn_checks_on; -#ifdef UNIV_HOTBACKUP -/** TRUE when the redo log is being backed up */ -extern bool recv_is_making_a_backup; -#endif /* UNIV_HOTBACKUP */ - -#ifndef UNIV_HOTBACKUP -/** Flag indicating if recv_writer thread is active. */ -extern volatile bool recv_writer_thread_active; -#endif /* !UNIV_HOTBACKUP */ /** Size of the parsing buffer; it must accommodate RECV_SCAN_SIZE many times! */ @@ -415,17 +297,4 @@ use these free frames to read in pages when we start applying the log records to the database. */ extern ulint recv_n_pool_free_frames; -/******************************************************//** -Checks the 4-byte checksum to the trailer checksum field of a log -block. */ -bool -log_block_checksum_is_ok( -/*===================================*/ - const byte* block, /*!< in: pointer to a log block */ - bool print_err); /*!< in print error ? */ - -#ifndef UNIV_NONINL -#include "log0recv.ic" -#endif - #endif diff --git a/storage/innobase/include/log0recv.ic b/storage/innobase/include/log0recv.ic deleted file mode 100644 index d197e5e3337..00000000000 --- a/storage/innobase/include/log0recv.ic +++ /dev/null @@ -1,38 +0,0 @@ -/***************************************************************************** - -Copyright (c) 1997, 2013, Oracle and/or its affiliates. All Rights Reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/**************************************************//** -@file include/log0recv.ic -Recovery - -Created 9/20/1997 Heikki Tuuri -*******************************************************/ - -#include "univ.i" - -/*******************************************************************//** -Returns TRUE if recovery is currently running. -@return recv_recovery_on */ -UNIV_INLINE -bool -recv_recovery_is_on() -/*=================*/ -{ - return(recv_recovery_on); -} - diff --git a/storage/innobase/include/mach0data.h b/storage/innobase/include/mach0data.h index 4d32e2e7170..3d6bccec718 100644 --- a/storage/innobase/include/mach0data.h +++ b/storage/innobase/include/mach0data.h @@ -271,7 +271,7 @@ ib_uint64_t mach_u64_parse_compressed( const byte** ptr, const byte* end_ptr); -#ifndef UNIV_HOTBACKUP + /*********************************************************//** Reads a double. It is stored in a little-endian format. @return double read */ @@ -353,17 +353,6 @@ mach_read_int_type( const byte* src, /*!< in: where to read from */ ulint len, /*!< in: length of src */ ibool unsigned_type); /*!< in: signed or unsigned flag */ -/***********************************************************//** -Convert integral type from host byte order to (big-endian) storage -byte order. */ -UNIV_INLINE -void -mach_write_int_type( -/*================*/ - byte* dest, /*!< in: where to write*/ - const byte* src, /*!< in: where to read from */ - ulint len, /*!< in: length of src */ - bool usign); /*!< in: signed or unsigned flag */ /************************************************************* Convert a ulonglong integer from host byte order to (big-endian) @@ -388,7 +377,6 @@ mach_read_ulint( mlog_id_t type) MY_ATTRIBUTE((warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ #endif /* !UNIV_INNOCHECKSUM */ #ifndef UNIV_NONINL diff --git a/storage/innobase/include/mach0data.ic b/storage/innobase/include/mach0data.ic index 20018844e4b..31cb873cdf6 100644 --- a/storage/innobase/include/mach0data.ic +++ b/storage/innobase/include/mach0data.ic @@ -609,7 +609,7 @@ mach_u64_parse_compressed( return(val); } -#ifndef UNIV_HOTBACKUP + /*********************************************************//** Reads a double. It is stored in a little-endian format. @return double read */ @@ -871,31 +871,6 @@ mach_swap_byte_order( } /************************************************************* -Convert integral type from host byte order (big-endian) storage -byte order. */ -UNIV_INLINE -void -mach_write_int_type( -/*================*/ - byte* dest, /*!< in: where to write */ - const byte* src, /*!< in: where to read from */ - ulint len, /*!< in: length of src */ - bool usign) /*!< in: signed or unsigned flag */ -{ - ut_ad(len >= 1 && len <= 8); - -#ifdef WORDS_BIGENDIAN - memcpy(dest, src, len); -#else - mach_swap_byte_order(dest, src, len); -#endif /* WORDS_BIGENDIAN */ - - if (!usign) { - *dest ^= 0x80; - } -} - -/************************************************************* Convert a ulonglong integer from host byte order to (big-endian) storage byte order. */ UNIV_INLINE @@ -947,5 +922,4 @@ mach_read_ulint( return(0); } -#endif /* !UNIV_HOTBACKUP */ #endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/include/mem0mem.h b/storage/innobase/include/mem0mem.h index f8fdb53e132..991d1179098 100644 --- a/storage/innobase/include/mem0mem.h +++ b/storage/innobase/include/mem0mem.h @@ -390,7 +390,7 @@ struct mem_block_info_t { user data in the block */ ulint start; /*!< the value of the struct field 'free' at the creation of the block */ -#ifndef UNIV_HOTBACKUP + void* free_block; /* if the MEM_HEAP_BTR_SEARCH bit is set in type, and this is the heap root, this can contain an @@ -401,7 +401,6 @@ struct mem_block_info_t { /* if this block has been allocated from the buffer pool, this contains the buf_block_t handle; otherwise, this is NULL */ -#endif /* !UNIV_HOTBACKUP */ }; #define MEM_BLOCK_MAGIC_N 764741555 diff --git a/storage/innobase/include/mem0mem.ic b/storage/innobase/include/mem0mem.ic index 3b4109ee52d..82c90584e6f 100644 --- a/storage/innobase/include/mem0mem.ic +++ b/storage/innobase/include/mem0mem.ic @@ -61,14 +61,12 @@ mem_heap_block_free( mem_heap_t* heap, /*!< in: heap */ mem_block_t* block); /*!< in: block to free */ -#ifndef UNIV_HOTBACKUP /******************************************************************//** Frees the free_block field from a memory heap. */ void mem_heap_free_block_free( /*=====================*/ mem_heap_t* heap); /*!< in: heap */ -#endif /* !UNIV_HOTBACKUP */ /***************************************************************//** Adds a new block to a memory heap. @@ -302,11 +300,10 @@ mem_heap_empty( mem_heap_t* heap) { mem_heap_free_heap_top(heap, (byte*) heap + mem_block_get_start(heap)); -#ifndef UNIV_HOTBACKUP + if (heap->free_block) { mem_heap_free_block_free(heap); } -#endif /* !UNIV_HOTBACKUP */ } /** Returns a pointer to the topmost element in a memory heap. @@ -524,11 +521,9 @@ mem_heap_free( block = UT_LIST_GET_LAST(heap->base); -#ifndef UNIV_HOTBACKUP if (heap->free_block) { mem_heap_free_block_free(heap); } -#endif /* !UNIV_HOTBACKUP */ while (block != NULL) { /* Store the contents of info before freeing current block @@ -556,11 +551,9 @@ mem_heap_get_size( size = heap->total_size; -#ifndef UNIV_HOTBACKUP if (heap->free_block) { size += UNIV_PAGE_SIZE; } -#endif /* !UNIV_HOTBACKUP */ return(size); } diff --git a/storage/innobase/include/mtr0log.h b/storage/innobase/include/mtr0log.h index 3819d3f694a..b5c94ff7d46 100644 --- a/storage/innobase/include/mtr0log.h +++ b/storage/innobase/include/mtr0log.h @@ -33,7 +33,6 @@ Created 12/7/1995 Heikki Tuuri // Forward declaration struct dict_index_t; -#ifndef UNIV_HOTBACKUP /********************************************************//** Writes 1, 2 or 4 bytes to a file page. Writes the corresponding log record to the mini-transaction log if mtr is not NULL. */ @@ -178,10 +177,6 @@ mlog_write_initial_log_record_fast( byte* log_ptr,/*!< in: pointer to mtr log which has been opened */ mtr_t* mtr); /*!< in: mtr */ -#else /* !UNIV_HOTBACKUP */ -# define mlog_write_initial_log_record(ptr,type,mtr) ((void) 0) -# define mlog_write_initial_log_record_fast(ptr,type,log_ptr,mtr) ((byte*) 0) -#endif /* !UNIV_HOTBACKUP */ /********************************************************//** Parses an initial log record written by mlog_write_initial_log_record. @return parsed record end, NULL if not a complete record */ @@ -216,7 +211,6 @@ mlog_parse_string( byte* page, /*!< in: page where to apply the log record, or NULL */ void* page_zip);/*!< in/out: compressed page, or NULL */ -#ifndef UNIV_HOTBACKUP /********************************************************//** Opens a buffer for mlog, writes the initial log record and, if needed, the field lengths of an index. Reserves space @@ -233,7 +227,6 @@ mlog_open_and_write_index( ulint size); /*!< in: requested buffer size in bytes (if 0, calls mlog_close() and returns NULL) */ -#endif /* !UNIV_HOTBACKUP */ /********************************************************//** Parses a log record written by mlog_open_and_write_index. @@ -246,11 +239,9 @@ mlog_parse_index( ibool comp, /*!< in: TRUE=compact record format */ dict_index_t** index); /*!< out, own: dummy index */ -#ifndef UNIV_HOTBACKUP /** Insert, update, and maybe other functions may use this value to define an extra mlog buffer size for variable size data */ #define MLOG_BUF_MARGIN 256 -#endif /* !UNIV_HOTBACKUP */ #ifndef UNIV_NONINL #include "mtr0log.ic" diff --git a/storage/innobase/include/mtr0log.ic b/storage/innobase/include/mtr0log.ic index 4015fe36d19..701f1482e64 100644 --- a/storage/innobase/include/mtr0log.ic +++ b/storage/innobase/include/mtr0log.ic @@ -65,7 +65,6 @@ mlog_close( mtr->get_log()->close(ptr); } -#ifndef UNIV_HOTBACKUP /********************************************************//** Catenates 1 - 4 bytes to the mtr log. The value is not compressed. */ UNIV_INLINE @@ -247,4 +246,3 @@ mlog_write_initial_log_record_fast( return(mlog_write_initial_log_record_low(type, space, offset, log_ptr, mtr)); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index 4a1d867015d..e761aaf7ec1 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2016, MariaDB Corporation +Copyright (c) 2013, 2017, MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -84,17 +84,11 @@ savepoint. */ #ifdef UNIV_DEBUG -/** Check if memo contains the given item ignore if table is intrinsic -@return TRUE if contains or table is intrinsic. */ -#define mtr_is_block_fix(m, o, t, table) \ - (mtr_memo_contains(m, o, t) \ - || dict_table_is_intrinsic(table)) +/** Check if memo contains the given item. */ +#define mtr_is_block_fix(m, o, t, table) mtr_memo_contains(m, o, t) -/** Check if memo contains the given page ignore if table is intrinsic -@return TRUE if contains or table is intrinsic. */ -#define mtr_is_page_fix(m, p, t, table) \ - (mtr_memo_contains_page(m, p, t) \ - || dict_table_is_intrinsic(table)) +/** Check if memo contains the given page. */ +#define mtr_is_page_fix(m, p, t, table) mtr_memo_contains_page(m, p, t) /** Check if memo contains the given item. @return TRUE if contains */ @@ -493,9 +487,9 @@ struct mtr_t { @param type type of object @return true if contains */ static bool memo_contains( - mtr_buf_t* memo, - const void* object, - ulint type) + const mtr_buf_t* memo, + const void* object, + ulint type) MY_ATTRIBUTE((warn_unused_result)); /** Check if memo contains the given item. diff --git a/storage/innobase/include/mtr0mtr.ic b/storage/innobase/include/mtr0mtr.ic index b3d9b052d52..f0354756b23 100644 --- a/storage/innobase/include/mtr0mtr.ic +++ b/storage/innobase/include/mtr0mtr.ic @@ -47,13 +47,6 @@ mtr_t::memo_push(void* object, mtr_memo_type_t type) m_impl.m_made_dirty = is_block_dirtied( reinterpret_cast<const buf_block_t*>(object)); - } else if (type == MTR_MEMO_BUF_FIX && !m_impl.m_made_dirty) { - - if (reinterpret_cast<const buf_block_t*>( - object)->made_dirty_with_no_latch) { - - m_impl.m_made_dirty = true; - } } mtr_memo_slot_t* slot; diff --git a/storage/innobase/include/mtr0types.h b/storage/innobase/include/mtr0types.h index 69a68830af1..95879a43872 100644 --- a/storage/innobase/include/mtr0types.h +++ b/storage/innobase/include/mtr0types.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -257,10 +258,10 @@ The record consists of a MLOG_CHECKPOINT byte followed by mach_write_to_8(checkpoint_lsn). */ #define SIZE_OF_MLOG_CHECKPOINT 9 +#ifndef UNIV_INNOCHECKSUM /** Types for the mlock objects to store in the mtr memo; NOTE that the first 3 values must be RW_S_LATCH, RW_X_LATCH, RW_NO_LATCH */ enum mtr_memo_type_t { -#ifndef UNIV_INNOCHECKSUM MTR_MEMO_PAGE_S_FIX = RW_S_LATCH, MTR_MEMO_PAGE_X_FIX = RW_X_LATCH, @@ -268,18 +269,18 @@ enum mtr_memo_type_t { MTR_MEMO_PAGE_SX_FIX = RW_SX_LATCH, MTR_MEMO_BUF_FIX = RW_NO_LATCH, -#endif /* !UNIV_CHECKSUM */ #ifdef UNIV_DEBUG - MTR_MEMO_MODIFY = 32, + MTR_MEMO_MODIFY = 16, #endif /* UNIV_DEBUG */ - MTR_MEMO_S_LOCK = 64, + MTR_MEMO_S_LOCK = RW_S_LATCH << 5, - MTR_MEMO_X_LOCK = 128, + MTR_MEMO_X_LOCK = RW_X_LATCH << 5, - MTR_MEMO_SX_LOCK = 256 + MTR_MEMO_SX_LOCK = RW_SX_LATCH << 5 }; +#endif /* !UNIV_CHECKSUM */ #ifdef UNIV_DEBUG # define MTR_MAGIC_N 54551 diff --git a/storage/innobase/include/os0api.h b/storage/innobase/include/os0api.h new file mode 100644 index 00000000000..63f213b5457 --- /dev/null +++ b/storage/innobase/include/os0api.h @@ -0,0 +1,75 @@ +/*********************************************************************** + +Copyright (c) 2017, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the +Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA + +***********************************************************************/ + +/**************************************************//** +@file os0api.h +The interface to the helper functions. +These functions are used on os0file.h where +including full full header is not feasible and +implemented on buf0buf.cc and fil0fil.cc. +*******************************************************/ + +#ifndef OS_API_H +#define OS_API_H 1 + +/** Page control block */ +class buf_page_t; + +/** File Node */ +struct fil_node_t; + +/** +Should we punch hole to deallocate unused portion of the page. +@param[in] bpage Page control block +@return true if punch hole should be used, false if not */ +bool +buf_page_should_punch_hole( + const buf_page_t* bpage) + MY_ATTRIBUTE((warn_unused_result)); + +/** +Calculate the length of trim (punch_hole) operation. +@param[in] bpage Page control block +@param[in] write_length Write length +@return length of the trim or zero. */ +ulint +buf_page_get_trim_length( + const buf_page_t* bpage, + ulint write_length) + MY_ATTRIBUTE((warn_unused_result)); + +/** +Get should we punch hole to tablespace. +@param[in] space Tablespace +@return true, if punch hole should be tried, false if not. */ +bool +fil_node_should_punch_hole( + const fil_node_t* node) + MY_ATTRIBUTE((warn_unused_result)); + +/** +Set punch hole to tablespace to given value. +@param[in] space Tablespace +@param[in] val value to be set. */ +void +fil_space_set_punch_hole( + fil_node_t* node, + bool val); + +#endif /* OS_API_H */ diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h index 5e36cfc2ac0..ffe7b1a48bf 100644 --- a/storage/innobase/include/os0file.h +++ b/storage/innobase/include/os0file.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -36,7 +36,8 @@ Created 10/21/1995 Heikki Tuuri #ifndef os0file_h #define os0file_h -#include "univ.i" +#include "page0size.h" +#include "os0api.h" #ifndef _WIN32 #include <dirent.h> @@ -46,8 +47,10 @@ Created 10/21/1995 Heikki Tuuri /** File node of a tablespace or the log data space */ struct fil_node_t; +struct fil_space_t; extern bool os_has_said_disk_full; +extern my_bool srv_use_trim; /** Number of pending read operations */ extern ulint os_n_pending_reads; @@ -77,7 +80,7 @@ the OS actually supports it: Win 95 does not, NT does. */ # define UNIV_NON_BUFFERED_IO /** File handle */ -# define os_file_t HANDLE +typedef HANDLE os_file_t; /** Convert a C file descriptor to a native file handle @param fd file descriptor @@ -169,333 +172,6 @@ static const ulint OS_FILE_OPERATION_NOT_SUPPORTED = 125; static const ulint OS_FILE_ERROR_MAX = 200; /* @} */ -/** Compression algorithm. */ -struct Compression { - - /** Algorithm types supported */ - enum Type { - /* Note: During recovery we don't have the compression type - because the .frm file has not been read yet. Therefore - we write the recovered pages out without compression. */ - - /** No compression */ - NONE = 0, - - /** Use ZLib */ - ZLIB = 1, - - /** Use LZ4 faster variant, usually lower compression. */ - LZ4 = 2 - }; - - /** Compressed page meta-data */ - struct meta_t { - - /** Version number */ - uint8_t m_version; - - /** Algorithm type */ - Type m_algorithm; - - /** Original page type */ - uint16_t m_original_type; - - /** Original page size, before compression */ - uint16_t m_original_size; - - /** Size after compression */ - uint16_t m_compressed_size; - }; - - /** Default constructor */ - Compression() : m_type(NONE) { }; - - /** Specific constructor - @param[in] type Algorithm type */ - explicit Compression(Type type) - : - m_type(type) - { -#ifdef UNIV_DEBUG - switch (m_type) { - case NONE: - case ZLIB: - case LZ4: - - default: - ut_error; - } -#endif /* UNIV_DEBUG */ - } - - /** Check the page header type field. - @param[in] page Page contents - @return true if it is a compressed page */ - static bool is_compressed_page(const byte* page) - MY_ATTRIBUTE((warn_unused_result)); - - /** Check wether the compression algorithm is supported. - @param[in] algorithm Compression algorithm to check - @param[out] type The type that algorithm maps to - @return DB_SUCCESS or error code */ - static dberr_t check(const char* algorithm, Compression* type) - MY_ATTRIBUTE((warn_unused_result)); - - /** Validate the algorithm string. - @param[in] algorithm Compression algorithm to check - @return DB_SUCCESS or error code */ - static dberr_t validate(const char* algorithm) - MY_ATTRIBUTE((warn_unused_result)); - - /** Convert to a "string". - @param[in] type The compression type - @return the string representation */ - static const char* to_string(Type type) - MY_ATTRIBUTE((warn_unused_result)); - - /** Convert the meta data to a std::string. - @param[in] meta Page Meta data - @return the string representation */ - static std::string to_string(const meta_t& meta) - MY_ATTRIBUTE((warn_unused_result)); - - /** Deserizlise the page header compression meta-data - @param[in] header Pointer to the page header - @param[out] control Deserialised data */ - static void deserialize_header( - const byte* page, - meta_t* control); - - /** Check if the string is "empty" or "none". - @param[in] algorithm Compression algorithm to check - @return true if no algorithm requested */ - static bool is_none(const char* algorithm) - MY_ATTRIBUTE((warn_unused_result)); - - /** Decompress the page data contents. Page type must be - FIL_PAGE_COMPRESSED, if not then the source contents are - left unchanged and DB_SUCCESS is returned. - @param[in] dblwr_recover true of double write recovery - in progress - @param[in,out] src Data read from disk, decompressed - data will be copied to this page - @param[in,out] dst Scratch area to use for decompression - @param[in] dst_len Size of the scratch area in bytes - @return DB_SUCCESS or error code */ - static dberr_t deserialize( - bool dblwr_recover, - byte* src, - byte* dst, - ulint dst_len) - MY_ATTRIBUTE((warn_unused_result)); - - /** Compression type */ - Type m_type; -}; - -/** Encryption key length */ -static const ulint ENCRYPTION_KEY_LEN = 32; - -/** Encryption magic bytes size */ -static const ulint ENCRYPTION_MAGIC_SIZE = 3; - -/** Encryption magic bytes for 5.7.11, it's for checking the encryption information -version. */ -static const char ENCRYPTION_KEY_MAGIC_V1[] = "lCA"; - -/** Encryption magic bytes for 5.7.12+, it's for checking the encryption information -version. */ -static const char ENCRYPTION_KEY_MAGIC_V2[] = "lCB"; - -/** Encryption master key prifix */ -static const char ENCRYPTION_MASTER_KEY_PRIFIX[] = "INNODBKey"; - -/** Encryption master key prifix size */ -static const ulint ENCRYPTION_MASTER_KEY_PRIFIX_LEN = 9; - -/** Encryption master key prifix size */ -static const ulint ENCRYPTION_MASTER_KEY_NAME_MAX_LEN = 100; - -/** UUID of server instance, it's needed for composing master key name */ -static const ulint ENCRYPTION_SERVER_UUID_LEN = 36; - -/** Encryption information total size for 5.7.11: magic number + master_key_id + -key + iv + checksum */ -static const ulint ENCRYPTION_INFO_SIZE_V1 = (ENCRYPTION_MAGIC_SIZE \ - + (ENCRYPTION_KEY_LEN * 2) \ - + 2 * sizeof(ulint)); - -/** Encryption information total size: magic number + master_key_id + -key + iv + server_uuid + checksum */ -static const ulint ENCRYPTION_INFO_SIZE_V2 = (ENCRYPTION_MAGIC_SIZE \ - + (ENCRYPTION_KEY_LEN * 2) \ - + ENCRYPTION_SERVER_UUID_LEN \ - + 2 * sizeof(ulint)); - -class IORequest; - -/** Encryption algorithm. */ -struct Encryption { - - /** Algorithm types supported */ - enum Type { - - /** No encryption */ - NONE = 0, - - /** Use AES */ - AES = 1, - }; - - /** Encryption information format version */ - enum Version { - - /** Version in 5.7.11 */ - ENCRYPTION_VERSION_1 = 0, - - /** Version in > 5.7.11 */ - ENCRYPTION_VERSION_2 = 1, - }; - - /** Default constructor */ - Encryption() : m_type(NONE) { }; - - /** Specific constructor - @param[in] type Algorithm type */ - explicit Encryption(Type type) - : - m_type(type) - { -#ifdef UNIV_DEBUG - switch (m_type) { - case NONE: - case AES: - - default: - ut_error; - } -#endif /* UNIV_DEBUG */ - } - - /** Copy constructor */ - Encryption(const Encryption& other) - : - m_type(other.m_type), - m_key(other.m_key), - m_klen(other.m_klen), - m_iv(other.m_iv) - { }; - - /** Check if page is encrypted page or not - @param[in] page page which need to check - @return true if it is a encrypted page */ - static bool is_encrypted_page(const byte* page) - MY_ATTRIBUTE((warn_unused_result)); - - /** Check the encryption option and set it - @param[in] option encryption option - @param[in/out] encryption The encryption type - @return DB_SUCCESS or DB_UNSUPPORTED */ - dberr_t set_algorithm(const char* option, Encryption* type) - MY_ATTRIBUTE((warn_unused_result)); - - /** Validate the algorithm string. - @param[in] algorithm Encryption algorithm to check - @return DB_SUCCESS or error code */ - static dberr_t validate(const char* algorithm) - MY_ATTRIBUTE((warn_unused_result)); - - /** Convert to a "string". - @param[in] type The encryption type - @return the string representation */ - static const char* to_string(Type type) - MY_ATTRIBUTE((warn_unused_result)); - - /** Check if the string is "empty" or "none". - @param[in] algorithm Encryption algorithm to check - @return true if no algorithm requested */ - static bool is_none(const char* algorithm) - MY_ATTRIBUTE((warn_unused_result)); - - /** Generate random encryption value for key and iv. - @param[in,out] value Encryption value */ - static void random_value(byte* value); - - /** Create new master key for key rotation. - @param[in,out] master_key master key */ - static void create_master_key(byte** master_key); - - /** Get master key by key id. - @param[in] master_key_id master key id - @param[in] srv_uuid uuid of server instance - @param[in,out] master_key master key */ - static void get_master_key(ulint master_key_id, - char* srv_uuid, - byte** master_key); - - /** Get current master key and key id. - @param[in,out] master_key_id master key id - @param[in,out] master_key master key - @param[in,out] version encryption information version */ - static void get_master_key(ulint* master_key_id, - byte** master_key, - Encryption::Version* version); - - /** Encrypt the page data contents. Page type can't be - FIL_PAGE_ENCRYPTED, FIL_PAGE_COMPRESSED_AND_ENCRYPTED, - FIL_PAGE_ENCRYPTED_RTREE. - @param[in] type IORequest - @param[in,out] src page data which need to encrypt - @param[in] src_len Size of the source in bytes - @param[in,out] dst destination area - @param[in,out] dst_len Size of the destination in bytes - @return buffer data, dst_len will have the length of the data */ - byte* encrypt( - const IORequest& type, - byte* src, - ulint src_len, - byte* dst, - ulint* dst_len) - MY_ATTRIBUTE((warn_unused_result)); - - /** Decrypt the page data contents. Page type must be - FIL_PAGE_ENCRYPTED, FIL_PAGE_COMPRESSED_AND_ENCRYPTED, - FIL_PAGE_ENCRYPTED_RTREE, if not then the source contents are - left unchanged and DB_SUCCESS is returned. - @param[in] type IORequest - @param[in,out] src Data read from disk, decrypt - data will be copied to this page - @param[in] src_len source data length - @param[in,out] dst Scratch area to use for decrypt - @param[in] dst_len Size of the scratch area in bytes - @return DB_SUCCESS or error code */ - dberr_t decrypt( - const IORequest& type, - byte* src, - ulint src_len, - byte* dst, - ulint dst_len) - MY_ATTRIBUTE((warn_unused_result)); - - /** Encrypt type */ - Type m_type; - - /** Encrypt key */ - byte* m_key; - - /** Encrypt key length*/ - ulint m_klen; - - /** Encrypt initial vector */ - byte* m_iv; - - /** Current master key id */ - static ulint master_key_id; - - /** Current uuid of server instance */ - static char uuid[ENCRYPTION_SERVER_UUID_LEN + 1]; -}; - /** Types for AIO operations @{ */ /** No transformations during read/write, write as is. */ @@ -504,6 +180,8 @@ struct Encryption { #define IORequestLogRead IORequest(IORequest::LOG | IORequest::READ) #define IORequestLogWrite IORequest(IORequest::LOG | IORequest::WRITE) + + /** The IO Context that is passed down to the low level IO code */ class IORequest { @@ -539,24 +217,16 @@ public: /** Ignore failed reads of non-existent pages */ IGNORE_MISSING = 128, - /** Use punch hole if available, only makes sense if - compression algorithm != NONE. Ignored if not set */ + /** Use punch hole if available*/ PUNCH_HOLE = 256, - - /** Force raw read, do not try to compress/decompress. - This can be used to force a read and write without any - compression e.g., for redo log, merge sort temporary files - and the truncate redo log. */ - NO_COMPRESSION = 512 - }; /** Default constructor */ IORequest() : - m_block_size(UNIV_SECTOR_SIZE), - m_type(READ), - m_compression() + m_bpage(NULL), + m_fil_node(NULL), + m_type(READ) { /* No op */ } @@ -566,15 +236,30 @@ public: ORed from the above enum */ explicit IORequest(ulint type) : - m_block_size(UNIV_SECTOR_SIZE), - m_type(static_cast<uint16_t>(type)), - m_compression() + m_bpage(NULL), + m_fil_node(NULL), + m_type(static_cast<uint16_t>(type)) + { + if (!is_punch_hole_supported() || !srv_use_trim) { + clear_punch_hole(); + } + } + + /** + @param[in] type Request type, can be a value that is + ORed from the above enum + @param[in] bpage Page to be written */ + IORequest(ulint type, buf_page_t* bpage) + : + m_bpage(bpage), + m_fil_node(NULL), + m_type(static_cast<uint16_t>(type)) { - if (is_log()) { - disable_compression(); + if (bpage && buf_page_should_punch_hole(bpage)) { + set_punch_hole(); } - if (!is_punch_hole_supported()) { + if (!is_punch_hole_supported() || !srv_use_trim) { clear_punch_hole(); } } @@ -617,6 +302,12 @@ public: return((m_type & DO_NOT_WAKE) == 0); } + /** Clear the punch hole flag */ + void clear_punch_hole() + { + m_type &= ~PUNCH_HOLE; + } + /** @return true if partial read warning disabled */ bool is_partial_io_warning_disabled() const MY_ATTRIBUTE((warn_unused_result)) @@ -649,15 +340,13 @@ public: bool validate() const MY_ATTRIBUTE((warn_unused_result)) { - ut_a(is_read() ^ is_write()); - - return(!is_read() || !punch_hole()); + return(is_read() ^ is_write()); } /** Set the punch hole flag */ void set_punch_hole() { - if (is_punch_hole_supported()) { + if (is_punch_hole_supported() && srv_use_trim) { m_type |= PUNCH_HOLE; } } @@ -668,32 +357,16 @@ public: m_type &= ~DO_NOT_WAKE; } - /** Clear the punch hole flag */ - void clear_punch_hole() + /** Set the pointer to file node for IO + @param[in] node File node */ + void set_fil_node(fil_node_t* node) { - m_type &= ~PUNCH_HOLE; - } - - /** @return the block size to use for IO */ - ulint block_size() const - MY_ATTRIBUTE((warn_unused_result)) - { - return(m_block_size); - } - - /** Set the block size for IO - @param[in] block_size Block size to set */ - void block_size(ulint block_size) - { - m_block_size = static_cast<uint32_t>(block_size); - } - - /** Clear all compression related flags */ - void clear_compressed() - { - clear_punch_hole(); + if (!srv_use_trim || + (node && !fil_node_should_punch_hole(node))) { + clear_punch_hole(); + } - m_compression.m_type = Compression::NONE; + m_fil_node = node; } /** Compare two requests @@ -703,95 +376,6 @@ public: return(m_type == rhs.m_type); } - /** Set compression algorithm - @param[in] compression The compression algorithm to use */ - void compression_algorithm(Compression::Type type) - { - if (type == Compression::NONE) { - return; - } - - set_punch_hole(); - - m_compression.m_type = type; - } - - /** Get the compression algorithm. - @return the compression algorithm */ - Compression compression_algorithm() const - MY_ATTRIBUTE((warn_unused_result)) - { - return(m_compression); - } - - /** @return true if the page should be compressed */ - bool is_compressed() const - MY_ATTRIBUTE((warn_unused_result)) - { - return(compression_algorithm().m_type != Compression::NONE); - } - - /** @return true if the page read should not be transformed. */ - bool is_compression_enabled() const - MY_ATTRIBUTE((warn_unused_result)) - { - return((m_type & NO_COMPRESSION) == 0); - } - - /** Disable transformations. */ - void disable_compression() - { - m_type |= NO_COMPRESSION; - } - - /** Set encryption algorithm - @param[in] type The encryption algorithm to use */ - void encryption_algorithm(Encryption::Type type) - { - if (type == Encryption::NONE) { - return; - } - - m_encryption.m_type = type; - } - - /** Set encryption key and iv - @param[in] key The encryption key to use - @param[in] key_len length of the encryption key - @param[in] iv The encryption iv to use */ - void encryption_key(byte* key, - ulint key_len, - byte* iv) - { - m_encryption.m_key = key; - m_encryption.m_klen = key_len; - m_encryption.m_iv = iv; - } - - /** Get the encryption algorithm. - @return the encryption algorithm */ - Encryption encryption_algorithm() const - MY_ATTRIBUTE((warn_unused_result)) - { - return(m_encryption); - } - - /** @return true if the page should be encrypted. */ - bool is_encrypted() const - MY_ATTRIBUTE((warn_unused_result)) - { - return(m_encryption.m_type != Encryption::NONE); - } - - /** Clear all encryption related flags */ - void clear_encrypted() - { - m_encryption.m_key = NULL; - m_encryption.m_klen = 0; - m_encryption.m_iv = NULL; - m_encryption.m_type = Encryption::NONE; - } - /** Note that the IO is for double write recovery. */ void dblwr_recover() { @@ -823,18 +407,44 @@ public: #endif /* HAVE_FALLOC_PUNCH_HOLE_AND_KEEP_SIZE || _WIN32 */ } + ulint get_trim_length(ulint write_length) const + { + return (m_bpage ? + buf_page_get_trim_length(m_bpage, write_length) + : 0); + } + + bool should_punch_hole() const { + return (m_fil_node ? + fil_node_should_punch_hole(m_fil_node) + : false); + } + + void space_no_punch_hole() const { + if (m_fil_node) { + fil_space_set_punch_hole(m_fil_node, false); + } + } + + /** Punch a hole in the file if it was a write + @param[in] fh Open file handle + @param[in] len Compressed buffer length for write + @return DB_SUCCESS or error code */ + + dberr_t punch_hole( + os_file_t fh, + os_offset_t offset, + os_offset_t len); + private: - /* File system best block size */ - uint32_t m_block_size; + /** Page to be written on write operation. */ + buf_page_t* m_bpage; + + /** File node */ + fil_node_t* m_fil_node; /** Request type bit flags */ uint16_t m_type; - - /** Compression algorithm */ - Compression m_compression; - - /** Encryption algorithm */ - Encryption m_encryption; }; /* @} */ @@ -918,7 +528,6 @@ struct os_file_stat_t { if type == OS_FILE_TYPE_FILE */ }; -#ifndef UNIV_HOTBACKUP /** Create a temporary file. This function is like tmpfile(3), but the temporary file is created in the given parameter path. If the path is null then it will create the file in the mysql server configuration @@ -928,7 +537,6 @@ parameter (--tmpdir). FILE* os_file_create_tmpfile( const char* path); -#endif /* !UNIV_HOTBACKUP */ /** The os_file_opendir() function opens a directory stream corresponding to the directory named by the dirname argument. The directory stream is positioned @@ -1199,10 +807,10 @@ The wrapper functions have the prefix of "innodb_". */ # define os_file_close(file) \ pfs_os_file_close_func(file, __FILE__, __LINE__) -# define os_aio(type, mode, name, file, buf, offset, \ - n, read_only, message1, message2, wsize) \ - pfs_os_aio_func(type, mode, name, file, buf, offset, \ - n, read_only, message1, message2, wsize, \ +# define os_aio(type, mode, name, file, buf, offset, \ + n, read_only, message1, message2) \ + pfs_os_aio_func(type, mode, name, file, buf, offset, \ + n, read_only, message1, message2, \ __FILE__, __LINE__) # define os_file_read(type, file, buf, offset, n) \ @@ -1214,7 +822,7 @@ The wrapper functions have the prefix of "innodb_". */ # define os_file_write(type, name, file, buf, offset, n) \ pfs_os_file_write_func(type, name, file, buf, offset, \ - n, __FILE__, __LINE__) + n,__FILE__, __LINE__) # define os_file_flush(file) \ pfs_os_file_flush_func(file, __FILE__, __LINE__) @@ -1419,7 +1027,6 @@ pfs_os_aio_func( bool read_only, fil_node_t* m1, void* m2, - ulint* wsize, const char* src_file, ulint src_line); @@ -1544,9 +1151,9 @@ to original un-instrumented file I/O APIs */ # define os_file_close(file) os_file_close_func(file) # define os_aio(type, mode, name, file, buf, offset, \ - n, read_only, message1, message2, wsize) \ + n, read_only, message1, message2) \ os_aio_func(type, mode, name, file, buf, offset, \ - n, read_only, message1, message2, wsize) + n, read_only, message1, message2) # define os_file_read(type, file, buf, offset, n) \ os_file_read_func(type, file, buf, offset, n) @@ -1554,7 +1161,7 @@ to original un-instrumented file I/O APIs */ # define os_file_read_no_error_handling(type, file, buf, offset, n, o) \ os_file_read_no_error_handling_func(type, file, buf, offset, n, o) -# define os_file_write(type, name, file, buf, offset, n) \ +# define os_file_write(type, name, file, buf, offset, n) \ os_file_write_func(type, name, file, buf, offset, n) # define os_file_flush(file) os_file_flush_func(file) @@ -1569,14 +1176,6 @@ to original un-instrumented file I/O APIs */ #endif /* UNIV_PFS_IO */ -#ifdef UNIV_HOTBACKUP -/** Closes a file handle. -@param[in] file handle to a file -@return true if success */ -bool -os_file_close_no_error_handling(os_file_t file); -#endif /* UNIV_HOTBACKUP */ - /** Gets a file size. @param[in] file handle to a file @return file size if OK, else set m_total_size to ~0 and m_alloc_size @@ -1825,8 +1424,7 @@ os_aio_func( ulint n, bool read_only, fil_node_t* m1, - void* m2, - ulint* wsize); + void* m2); /** Wakes up all async i/o threads so that they know to exit themselves in shutdown. */ @@ -1842,12 +1440,16 @@ os_aio_wait_until_no_pending_writes(); void os_aio_simulated_wake_handler_threads(); +#ifdef _WIN32 /** This function can be called if one wants to post a batch of reads and prefers an i/o-handler thread to handle them all at once later. You must call os_aio_simulated_wake_handler_threads later to ensure the threads are not left sleeping! */ void os_aio_simulated_put_read_threads_to_sleep(); +#else /* _WIN32 */ +# define os_aio_simulated_put_read_threads_to_sleep() +#endif /* _WIN32 */ /** This is the generic AIO handler interface function. Waits for an aio operation to complete. This function is used to wait the @@ -1914,7 +1516,6 @@ os_file_get_status( bool check_rw_perm, bool read_only); -#if !defined(UNIV_HOTBACKUP) /** Creates a temporary file in the location specified by the parameter path. If the path is NULL then it will be created on --tmpdir location. This function is defined in ha_innodb.cc. @@ -1923,40 +1524,12 @@ This function is defined in ha_innodb.cc. int innobase_mysql_tmpfile( const char* path); -#endif /* !UNIV_HOTBACKUP */ - - -/** If it is a compressed page return the compressed page data + footer size -@param[in] buf Buffer to check, must include header + 10 bytes -@return ULINT_UNDEFINED if the page is not a compressed page or length - of the compressed data (including footer) if it is a compressed page */ -ulint -os_file_compressed_page_size(const byte* buf); - -/** If it is a compressed page return the original page data + footer size -@param[in] buf Buffer to check, must include header + 10 bytes -@return ULINT_UNDEFINED if the page is not a compressed page or length - of the original data + footer if it is a compressed page */ -ulint -os_file_original_page_size(const byte* buf); /** Set the file create umask @param[in] umask The umask to use for file creation. */ void os_file_set_umask(ulint umask); -/** Free storage space associated with a section of the file. -@param[in] fh Open file handle -@param[in] off Starting offset (SEEK_SET) -@param[in] len Size of the hole -@return DB_SUCCESS or error code */ -dberr_t -os_file_punch_hole( - os_file_t fh, - os_offset_t off, - os_offset_t len) - MY_ATTRIBUTE((warn_unused_result)); - /** Check if the file system supports sparse files. Warning: On POSIX systems we try and punch a hole from offset 0 to @@ -1974,21 +1547,29 @@ os_is_sparse_file_supported( os_file_t fh) MY_ATTRIBUTE((warn_unused_result)); -/** Decompress the page data contents. Page type must be FIL_PAGE_COMPRESSED, if -not then the source contents are left unchanged and DB_SUCCESS is returned. -@param[in] dblwr_recover true of double write recovery in progress -@param[in,out] src Data read from disk, decompressed data will be - copied to this page -@param[in,out] dst Scratch area to use for decompression -@param[in] dst_len Size of the scratch area in bytes +/** Free storage space associated with a section of the file. +@param[in] fh Open file handle +@param[in] off Starting offset (SEEK_SET) +@param[in] len Size of the hole @return DB_SUCCESS or error code */ +dberr_t +os_file_punch_hole( + IORequest& type, + os_file_t fh, + os_offset_t off, + os_offset_t len) + MY_ATTRIBUTE((warn_unused_result)); +/** Free storage space associated with a section of the file. +@param[in] fh Open file handle +@param[in] off Starting offset (SEEK_SET) +@param[in] len Size of the hole +@return DB_SUCCESS or error code */ dberr_t -os_file_decompress_page( - bool dblwr_recover, - byte* src, - byte* dst, - ulint dst_len) +os_file_punch_hole( + os_file_t fh, + os_offset_t off, + os_offset_t len) MY_ATTRIBUTE((warn_unused_result)); /** Normalizes a directory path for the current OS: @@ -2018,6 +1599,16 @@ is_absolute_path( return(false); } +/***********************************************************************//** +Try to get number of bytes per sector from file system. +@return file block size */ +UNIV_INTERN +ulint +os_file_get_block_size( +/*===================*/ + os_file_t file, /*!< in: handle to a file */ + const char* name); /*!< in: file name */ + #ifndef UNIV_NONINL #include "os0file.ic" #endif /* UNIV_NONINL */ diff --git a/storage/innobase/include/os0file.ic b/storage/innobase/include/os0file.ic index 74d0b2c83a8..5c7c4d45ca6 100644 --- a/storage/innobase/include/os0file.ic +++ b/storage/innobase/include/os0file.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2010, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -219,11 +219,6 @@ an asynchronous i/o operation. @param[in,out] m2 message for the AIO handler (can be used to identify a completed AIO operation); ignored if mode is OS_AIO_SYNC -@param[in,out] write_size Actual write size initialized - after fist successfull trim - operation for this page and if - initialized we do not trim again if - actual page size @param[in] src_file file name where func invoked @param[in] src_line line where the func invoked @return DB_SUCCESS if request was queued successfully, FALSE if fail */ @@ -240,7 +235,6 @@ pfs_os_aio_func( bool read_only, fil_node_t* m1, void* m2, - ulint* write_size, const char* src_file, ulint src_line) { @@ -256,7 +250,7 @@ pfs_os_aio_func( src_file, src_line); dberr_t result = os_aio_func( - type, mode, name, file, buf, offset, n, read_only, m1, m2, write_size); + type, mode, name, file, buf, offset, n, read_only, m1, m2); register_pfs_file_io_end(locker, n); diff --git a/storage/innobase/include/page0cur.h b/storage/innobase/include/page0cur.h index c111717d868..2bb922cc829 100644 --- a/storage/innobase/include/page0cur.h +++ b/storage/innobase/include/page0cur.h @@ -135,7 +135,7 @@ void page_cur_move_to_prev( /*==================*/ page_cur_t* cur); /*!< in/out: cursor; not before first */ -#ifndef UNIV_HOTBACKUP + /***********************************************************//** Inserts a record next to page cursor. Returns pointer to inserted record if succeed, i.e., enough space available, NULL otherwise. The cursor stays at @@ -163,7 +163,6 @@ page_cur_tuple_insert( /*!< in: if true, then use record cache to hold the tuple converted record. */ MY_ATTRIBUTE((nonnull(1,2,3,4,5), warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Inserts a record next to page cursor. Returns pointer to inserted record if succeed, i.e., enough space available, NULL otherwise. The cursor stays at @@ -201,23 +200,6 @@ page_cur_insert_rec_low( mtr_t* mtr) /*!< in: mini-transaction handle, or NULL */ MY_ATTRIBUTE((nonnull(1,2,3,4), warn_unused_result)); -/** Inserts a record next to page cursor on an uncompressed page. -@param[in] current_rec pointer to current record after which - the new record is inserted. -@param[in] index record descriptor -@param[in] tuple pointer to a data tuple -@param[in] n_ext number of externally stored columns -@param[in] mtr mini-transaction handle, or NULL - -@return pointer to record if succeed, NULL otherwise */ -rec_t* -page_cur_direct_insert_rec_low( - rec_t* current_rec, - dict_index_t* index, - const dtuple_t* tuple, - ulint n_ext, - mtr_t* mtr); - /***********************************************************//** Inserts a record next to page cursor on a compressed and uncompressed page. Returns pointer to inserted record if succeed, i.e., @@ -265,7 +247,7 @@ page_cur_delete_rec( const ulint* offsets,/*!< in: rec_get_offsets( cursor->rec, index) */ mtr_t* mtr); /*!< in: mini-transaction handle */ -#ifndef UNIV_HOTBACKUP + /** Search the right position for a page cursor. @param[in] block buffer block @param[in] index index tree @@ -348,7 +330,6 @@ page_cur_open_on_rnd_user_rec( /*==========================*/ buf_block_t* block, /*!< in: page */ page_cur_t* cursor);/*!< out: page cursor */ -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses a log record of a record insert on a page. @return end of log record or NULL */ diff --git a/storage/innobase/include/page0cur.ic b/storage/innobase/include/page0cur.ic index 8f580ef2d43..5eb1bc0cbc5 100644 --- a/storage/innobase/include/page0cur.ic +++ b/storage/innobase/include/page0cur.ic @@ -197,7 +197,6 @@ page_cur_move_to_prev( cur->rec = page_rec_get_prev(cur->rec); } -#ifndef UNIV_HOTBACKUP /** Search the right position for a page cursor. @param[in] block buffer block @param[in] index index tree @@ -295,36 +294,6 @@ page_cur_tuple_insert( return(rec); } -/** Insert a record next to page cursor. Record is directly copied to -the page from tuple without creating intermediate copy of the record. - -@param[in,out] cursor a page cursor -@param[in] tuple pointer to a data tuple -@param[in] index record descriptor -@param[in] n_ext number of externally stored columns -@param[in] mtr mini-transaction handle, or NULL - -@return pointer to record if succeed, NULL otherwise */ -UNIV_INLINE -rec_t* -page_cur_tuple_direct_insert( - page_cur_t* cursor, - const dtuple_t* tuple, - dict_index_t* index, - ulint n_ext, - mtr_t* mtr) -{ - rec_t* rec; - - ut_ad(dict_table_is_intrinsic(index->table)); - - rec = page_cur_direct_insert_rec_low( - cursor->rec, index, tuple, n_ext, mtr); - - return(rec); -} -#endif /* !UNIV_HOTBACKUP */ - /***********************************************************//** Inserts a record next to page cursor. Returns pointer to inserted record if succeed, i.e., enough space available, NULL otherwise. The cursor stays at diff --git a/storage/innobase/include/page0page.h b/storage/innobase/include/page0page.h index 3bb622127fb..e61e7d2ae16 100644 --- a/storage/innobase/include/page0page.h +++ b/storage/innobase/include/page0page.h @@ -1,5 +1,4 @@ /***************************************************************************** - Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2013, 2016, MariaDB Corporation @@ -70,10 +69,12 @@ typedef byte page_header_t; #define PAGE_N_DIRECTION 14 /* number of consecutive inserts to the same direction */ #define PAGE_N_RECS 16 /* number of user records on the page */ -#define PAGE_MAX_TRX_ID 18 /* highest id of a trx which may have modified - a record on the page; trx_id_t; defined only - in secondary indexes and in the insert buffer - tree */ +/** The largest DB_TRX_ID that may have modified a record on the page; +Defined only in secondary index leaf pages and in change buffer leaf pages. +Otherwise written as 0. @see PAGE_ROOT_AUTO_INC */ +#define PAGE_MAX_TRX_ID 18 +/** The AUTO_INCREMENT value (on persistent clustered index root pages). */ +#define PAGE_ROOT_AUTO_INC PAGE_MAX_TRX_ID #define PAGE_HEADER_PRIV_END 26 /* end of private data structure of the page header which are set in a page create */ /*----*/ @@ -213,6 +214,32 @@ page_update_max_trx_id( uncompressed part will be updated, or NULL */ trx_id_t trx_id, /*!< in: transaction id */ mtr_t* mtr); /*!< in/out: mini-transaction */ + +/** Persist the AUTO_INCREMENT value on a clustered index root page. +@param[in,out] block clustered index root page +@param[in] index clustered index +@param[in] autoinc next available AUTO_INCREMENT value +@param[in,out] mtr mini-transaction +@param[in] reset whether to reset the AUTO_INCREMENT + to a possibly smaller value than currently + exists in the page */ +void +page_set_autoinc( + buf_block_t* block, + const dict_index_t* index MY_ATTRIBUTE((unused)), + ib_uint64_t autoinc, + mtr_t* mtr, + bool reset) + MY_ATTRIBUTE((nonnull)); + +/** Read the AUTO_INCREMENT value from a clustered index root page. +@param[in] page clustered index root page +@return the persisted AUTO_INCREMENT value */ +MY_ATTRIBUTE((nonnull, warn_unused_result)) +UNIV_INLINE +ib_uint64_t +page_get_autoinc(const page_t* page); + /*************************************************************//** Returns the RTREE SPLIT SEQUENCE NUMBER (FIL_RTREE_SPLIT_SEQ_NUM). @return SPLIT SEQUENCE NUMBER */ @@ -282,7 +309,7 @@ page_header_set_ptr( uncompressed part will be updated, or NULL */ ulint field, /*!< in/out: PAGE_FREE, ... */ const byte* ptr); /*!< in: pointer or NULL*/ -#ifndef UNIV_HOTBACKUP + /*************************************************************//** Resets the last insert info field in the page header. Writes to mlog about this operation. */ @@ -294,7 +321,6 @@ page_header_reset_last_insert( page_zip_des_t* page_zip,/*!< in/out: compressed page whose uncompressed part will be updated, or NULL */ mtr_t* mtr); /*!< in: mtr */ -#endif /* !UNIV_HOTBACKUP */ /************************************************************//** Gets the offset of the first record on the page. @return offset of the first record in record list, relative from page */ @@ -335,7 +361,7 @@ page_rec_get_nth( page_t* page, /*< in: page */ ulint nth) /*!< in: nth record */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -#ifndef UNIV_HOTBACKUP + /************************************************************//** Returns the middle record of the records on the page. If there is an even number of records in the list, returns the first record of the @@ -347,7 +373,6 @@ page_get_middle_rec( /*================*/ page_t* page) /*!< in: page */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /*************************************************************//** Gets the page number. @return page number */ @@ -729,7 +754,7 @@ rec_t* page_rec_find_owner_rec( /*====================*/ rec_t* rec); /*!< in: the physical record */ -#ifndef UNIV_HOTBACKUP + /***********************************************************************//** Write a 32-bit field in a data dictionary record. */ UNIV_INLINE @@ -741,7 +766,6 @@ page_rec_write_field( ulint val, /*!< in: value to write */ mtr_t* mtr) /*!< in/out: mini-transaction */ MY_ATTRIBUTE((nonnull)); -#endif /* !UNIV_HOTBACKUP */ /************************************************************//** Returns the maximum combined size of records which can be inserted on top of record heap. @@ -1042,7 +1066,7 @@ page_parse_create( buf_block_t* block, ulint comp, bool is_rtree); -#ifndef UNIV_HOTBACKUP + /************************************************************//** Prints record contents including the data relevant only in the index page context. */ @@ -1088,7 +1112,6 @@ page_print( ulint rn); /*!< in: print rn first and last records in directory */ # endif /* UNIV_BTR_PRINT */ -#endif /* !UNIV_HOTBACKUP */ /***************************************************************//** The following is used to validate a record on a page. This function differs from rec_validate as it can also check the n_owned field and diff --git a/storage/innobase/include/page0page.ic b/storage/innobase/include/page0page.ic index 0a0ff41774c..a70ca3da8d8 100644 --- a/storage/innobase/include/page0page.ic +++ b/storage/innobase/include/page0page.ic @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -31,9 +32,7 @@ Created 2/2/1994 Heikki Tuuri #ifdef UNIV_DEBUG # include "log0recv.h" #endif /* !UNIV_DEBUG */ -#ifndef UNIV_HOTBACKUP -# include "rem0cmp.h" -#endif /* !UNIV_HOTBACKUP */ +#include "rem0cmp.h" #include "mtr0log.h" #include "page0zip.h" @@ -108,6 +107,17 @@ page_update_max_trx_id( } } +/** Read the AUTO_INCREMENT value from a clustered index root page. +@param[in] page clustered index root page +@return the persisted AUTO_INCREMENT value */ +UNIV_INLINE +ib_uint64_t +page_get_autoinc(const page_t* page) +{ + ut_ad(page_is_root(page)); + return(mach_read_from_8(PAGE_HEADER + PAGE_ROOT_AUTO_INC + page)); +} + /*************************************************************//** Returns the RTREE SPLIT SEQUENCE NUMBER (FIL_RTREE_SPLIT_SEQ_NUM). @return SPLIT SEQUENCE NUMBER */ @@ -136,20 +146,18 @@ page_set_ssn_id( mtr_t* mtr) /*!< in/out: mini-transaction */ { page_t* page = buf_block_get_frame(block); -#ifndef UNIV_HOTBACKUP - ut_ad(!mtr || mtr_memo_contains(mtr, block, MTR_MEMO_PAGE_SX_FIX) - || mtr_memo_contains(mtr, block, MTR_MEMO_PAGE_X_FIX)); -#endif /* !UNIV_HOTBACKUP */ + + ut_ad(!mtr || mtr_memo_contains_flagged(mtr, block, + MTR_MEMO_PAGE_SX_FIX + | MTR_MEMO_PAGE_X_FIX)); if (page_zip) { mach_write_to_8(page + FIL_RTREE_SPLIT_SEQ_NUM, ssn_id); page_zip_write_header(page_zip, page + FIL_RTREE_SPLIT_SEQ_NUM, 8, mtr); -#ifndef UNIV_HOTBACKUP } else if (mtr) { mlog_write_ull(page + FIL_RTREE_SPLIT_SEQ_NUM, ssn_id, mtr); -#endif /* !UNIV_HOTBACKUP */ } else { mach_write_to_8(page + FIL_RTREE_SPLIT_SEQ_NUM, ssn_id); } @@ -252,7 +260,6 @@ page_header_set_ptr( page_header_set_field(page, page_zip, field, offs); } -#ifndef UNIV_HOTBACKUP /*************************************************************//** Resets the last insert info field in the page header. Writes to mlog about this operation. */ @@ -278,7 +285,6 @@ page_header_reset_last_insert( MLOG_2BYTES, mtr); } } -#endif /* !UNIV_HOTBACKUP */ /************************************************************//** Determine whether the page is in new-style compact format. @@ -606,7 +612,6 @@ page_rec_get_nth( return((rec_t*) page_rec_get_nth_const(page, nth)); } -#ifndef UNIV_HOTBACKUP /************************************************************//** Returns the middle record of the records on the page. If there is an even number of records in the list, returns the first record of the @@ -622,7 +627,6 @@ page_get_middle_rec( return(page_rec_get_nth(page, middle)); } -#endif /* !UNIV_HOTBACKUP */ /*************************************************************//** Gets the page number. @@ -1129,7 +1133,6 @@ page_get_free_space_of_empty( - 2 * PAGE_DIR_SLOT_SIZE)); } -#ifndef UNIV_HOTBACKUP /***********************************************************************//** Write a 32-bit field in a data dictionary record. */ UNIV_INLINE @@ -1150,7 +1153,6 @@ page_rec_write_field( mlog_write_ulint(data, val, MLOG_4BYTES, mtr); } -#endif /* !UNIV_HOTBACKUP */ /************************************************************//** Each user record on a page, and also the deleted user records in the heap diff --git a/storage/innobase/include/page0size.h b/storage/innobase/include/page0size.h index ab917e1ff05..ca173db9b6d 100644 --- a/storage/innobase/include/page0size.h +++ b/storage/innobase/include/page0size.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2013, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -139,9 +140,7 @@ public: @param[in] src page size object whose values to fetch */ inline void copy_from(const page_size_t& src) { - m_physical = src.physical(); - m_logical = src.logical(); - m_is_compressed = src.is_compressed(); + *this = src; } /** Check if a given page_size_t object is equal to the current one. @@ -156,9 +155,6 @@ public: private: - /* Disable implicit copying. */ - void operator=(const page_size_t&); - /* For non compressed tablespaces, physical page size is equal to the logical page size and the data is stored in buf_page_t::frame (and is also always equal to univ_page_size (--innodb-page-size=)). diff --git a/storage/innobase/include/page0zip.h b/storage/innobase/include/page0zip.h index 7b5df3d306b..4a32595af66 100644 --- a/storage/innobase/include/page0zip.h +++ b/storage/innobase/include/page0zip.h @@ -97,7 +97,6 @@ page_zip_set_size( page_zip_des_t* page_zip, /*!< in/out: compressed page */ ulint size); /*!< in: size in bytes */ -#ifndef UNIV_HOTBACKUP /** Determine if a record is so big that it needs to be stored externally. @param[in] rec_size length of the record in bytes @param[in] comp nonzero=compact format @@ -132,7 +131,6 @@ bool page_zip_is_too_big( const dict_index_t* index, const dtuple_t* entry); -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Initialize a compressed page descriptor. */ @@ -459,7 +457,7 @@ page_zip_reorganize( dict_index_t* index, /*!< in: index of the B-tree node */ mtr_t* mtr) /*!< in: mini-transaction */ MY_ATTRIBUTE((nonnull)); -#ifndef UNIV_HOTBACKUP + /**********************************************************************//** Copy the records of a page byte for byte. Do not copy the page header or trailer, except those B-tree header fields that are directly @@ -476,7 +474,6 @@ page_zip_copy_recs( const page_t* src, /*!< in: page */ dict_index_t* index, /*!< in: index of the B-tree */ mtr_t* mtr); /*!< in: mini-transaction */ -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Parses a log record of compressing an index page. diff --git a/storage/innobase/include/page0zip.ic b/storage/innobase/include/page0zip.ic index 9963fe01c82..9cc54dc42fa 100644 --- a/storage/innobase/include/page0zip.ic +++ b/storage/innobase/include/page0zip.ic @@ -148,7 +148,6 @@ page_zip_set_size( ut_ad(page_zip_get_size(page_zip) == size); } -#ifndef UNIV_HOTBACKUP /** Determine if a record is so big that it needs to be stored externally. @param[in] rec_size length of the record in bytes @param[in] comp nonzero=compact format @@ -188,7 +187,6 @@ page_zip_rec_needs_ext( return(rec_size >= page_get_free_space_of_empty(comp) / 2); } -#endif /* !UNIV_HOTBACKUP */ #ifdef UNIV_DEBUG /**********************************************************************//** @@ -364,9 +362,7 @@ page_zip_write_header( /* ut_ad(page_zip_validate(page_zip, str - pos)); */ if (mtr) { -#ifndef UNIV_HOTBACKUP page_zip_write_header_log(str, length, mtr); -#endif /* !UNIV_HOTBACKUP */ } } diff --git a/storage/innobase/include/que0que.h b/storage/innobase/include/que0que.h index 3e90e0b25e3..574b7e6acee 100644 --- a/storage/innobase/include/que0que.h +++ b/storage/innobase/include/que0que.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -315,18 +316,6 @@ que_fork_scheduler_round_robin( que_fork_t* fork, /*!< in: a query fork */ que_thr_t* thr); /*!< in: current pos */ -/*********************************************************************//** -Initialise the query sub-system. */ -void -que_init(void); -/*==========*/ - -/*********************************************************************//** -Close the query sub-system. */ -void -que_close(void); -/*===========*/ - /** Query thread states */ enum que_thr_state_t { QUE_THR_RUNNING, diff --git a/storage/innobase/include/rem0rec.h b/storage/innobase/include/rem0rec.h index 8490e7c9c88..ed700139b53 100644 --- a/storage/innobase/include/rem0rec.h +++ b/storage/innobase/include/rem0rec.h @@ -773,7 +773,6 @@ rec_copy( const rec_t* rec, const ulint* offsets); -#ifndef UNIV_HOTBACKUP /**********************************************************//** Determines the size of a data tuple prefix in a temporary file. @return total size */ @@ -846,7 +845,6 @@ rec_fold( ulint n_bytes, index_id_t tree_id) MY_ATTRIBUTE((warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /*********************************************************//** Builds a physical record out of a data tuple and stores it into the given buffer. @@ -911,7 +909,6 @@ rec_get_converted_size( const dtuple_t* dtuple, /*!< in: data tuple */ ulint n_ext) /*!< in: number of externally stored columns */ MY_ATTRIBUTE((warn_unused_result, nonnull)); -#ifndef UNIV_HOTBACKUP /**************************************************************//** Copies the first n fields of a physical record to a data tuple. The fields are copied to the memory heap. */ @@ -925,7 +922,6 @@ rec_copy_prefix_to_dtuple( to copy */ mem_heap_t* heap) /*!< in: memory heap */ MY_ATTRIBUTE((nonnull)); -#endif /* !UNIV_HOTBACKUP */ /***************************************************************//** Validates the consistency of a physical record. @return TRUE if ok */ @@ -943,7 +939,6 @@ rec_print_old( FILE* file, /*!< in: file where to print */ const rec_t* rec) /*!< in: physical record */ MY_ATTRIBUTE((nonnull)); -#ifndef UNIV_HOTBACKUP /***************************************************************//** Prints a physical record in ROW_FORMAT=COMPACT. Ignores the record header. */ @@ -1104,7 +1099,6 @@ rec_get_trx_id( const dict_index_t* index) /*!< in: clustered index */ MY_ATTRIBUTE((nonnull, warn_unused_result)); # endif /* UNIV_DEBUG */ -#endif /* UNIV_HOTBACKUP */ /* Maximum lengths for the data in a physical record if the offsets are given in one byte (resp. two byte) format. */ diff --git a/storage/innobase/include/rem0rec.ic b/storage/innobase/include/rem0rec.ic index b855a39da9e..e16eab62181 100644 --- a/storage/innobase/include/rem0rec.ic +++ b/storage/innobase/include/rem0rec.ic @@ -1696,7 +1696,6 @@ rec_get_converted_size( return(data_size + extra_size); } -#ifndef UNIV_HOTBACKUP /** Fold a prefix of a physical record. @param[in] rec index record @param[in] offsets return value of rec_get_offsets() @@ -1761,4 +1760,3 @@ rec_fold( return(fold); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/row0merge.h b/storage/innobase/include/row0merge.h index f3b5860910c..9e71ee42bb0 100644 --- a/storage/innobase/include/row0merge.h +++ b/storage/innobase/include/row0merge.h @@ -114,7 +114,6 @@ struct index_field_t { ulint col_no; /*!< column offset */ ulint prefix_len; /*!< column prefix length, or 0 if indexing the whole column */ - const char* col_name; /*!< column name or NULL */ bool is_v_col; /*!< whether this is a virtual column */ }; @@ -432,8 +431,8 @@ row_merge_sort( row_merge_block_t* block, int* tmpfd, const bool update_progress, - const float pct_progress, - const float pct_cost, + const double pct_progress, + const double pct_cost, fil_space_crypt_t* crypt_data, row_merge_block_t* crypt_block, ulint space, diff --git a/storage/innobase/include/row0mysql.h b/storage/innobase/include/row0mysql.h index 2d508c1a7df..607f7c96389 100644 --- a/storage/innobase/include/row0mysql.h +++ b/storage/innobase/include/row0mysql.h @@ -37,7 +37,6 @@ Created 9/17/2000 Heikki Tuuri #include "btr0pcur.h" #include "trx0types.h" #include "fil0crypt.h" -#include "sess0sess.h" // Forward declaration struct SysIndexCallback; @@ -292,14 +291,6 @@ row_update_for_mysql( row_prebuilt_t* prebuilt) MY_ATTRIBUTE((warn_unused_result)); -/** Delete all rows for the given table by freeing/truncating indexes. -@param[in,out] table table handler -@return error code or DB_SUCCESS */ -dberr_t -row_delete_all_rows( - dict_table_t* table) - MY_ATTRIBUTE((warn_unused_result)); - /** This can only be used when srv_locks_unsafe_for_binlog is TRUE or this session is using a READ COMMITTED or READ UNCOMMITTED isolation level. Before calling this function row_search_for_mysql() must have @@ -382,9 +373,6 @@ row_create_table_for_mysql( dict_table_t* table, /*!< in, own: table definition (will be freed, or on DB_SUCCESS added to the data dictionary cache) */ - const char* compression, - /*!< in: compression algorithm to use, - can be NULL */ trx_t* trx, /*!< in/out: transaction */ bool commit, /*!< in: if true, commit the transaction */ fil_encryption_t mode, /*!< in: encryption mode */ @@ -402,13 +390,12 @@ row_create_index_for_mysql( dict_index_t* index, /*!< in, own: index definition (will be freed) */ trx_t* trx, /*!< in: transaction handle */ - const ulint* field_lengths, /*!< in: if not NULL, must contain + const ulint* field_lengths) /*!< in: if not NULL, must contain dict_index_get_n_fields(index) actual field lengths for the index columns, which are then checked for not being too large. */ - dict_table_t* handler) /* ! in/out: table handler. */ MY_ATTRIBUTE((warn_unused_result)); /*********************************************************************//** Scans a table create SQL string and adds to the data dictionary @@ -427,8 +414,6 @@ fields than mentioned in the constraint. database id the database of parameter name @param[in] sql_length length of sql_string @param[in] name table full name in normalized form -@param[in] is_temp_table true if table is temporary -@param[in,out] handler table handler if table is intrinsic @param[in] reject_fks if TRUE, fail with error code DB_CANNOT_ADD_CONSTRAINT if any foreign keys are found. @@ -493,16 +478,9 @@ row_drop_table_for_mysql( ibool create_failed,/*!<in: TRUE=create table failed because e.g. foreign key column type mismatch. */ - bool nonatomic = true, + bool nonatomic = true); /*!< in: whether it is permitted to release and reacquire dict_operation_lock */ - dict_table_t* handler = NULL); - /*!< in/out: table handler. */ -/*********************************************************************//** -Drop all temporary tables during crash recovery. */ -void -row_mysql_drop_temp_tables(void); -/*============================*/ /*********************************************************************//** Discards the tablespace of a table which stored in an .ibd file. Discarding @@ -886,10 +864,6 @@ struct row_prebuilt_t { ulint idx_cond_n_cols;/*!< Number of fields in idx_cond_cols. 0 if and only if idx_cond == NULL. */ /*----------------------*/ - unsigned innodb_api:1; /*!< whether this is a InnoDB API - query */ - const rec_t* innodb_api_rec; /*!< InnoDB API search result */ - /*----------------------*/ /*----------------------*/ rtr_info_t* rtr_info; /*!< R-tree Search Info */ @@ -898,10 +872,6 @@ struct row_prebuilt_t { ulint magic_n2; /*!< this should be the same as magic_n */ - bool ins_sel_stmt; /*!< if true then ins_sel_statement. */ - - innodb_session_t* - session; /*!< InnoDB session handler. */ byte* srch_key_val1; /*!< buffer used in converting search key values from MySQL format to InnoDB format.*/ diff --git a/storage/innobase/include/row0row.h b/storage/innobase/include/row0row.h index 93ff90d020e..9ae1387810a 100644 --- a/storage/innobase/include/row0row.h +++ b/storage/innobase/include/row0row.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -315,6 +316,22 @@ row_get_clust_rec( mtr_t* mtr) /*!< in: mtr */ MY_ATTRIBUTE((nonnull, warn_unused_result)); +/** Parse the integer data from specified data, which could be +DATA_INT, DATA_FLOAT or DATA_DOUBLE. If the value is less than 0 +and the type is not unsigned then we reset the value to 0 +@param[in] data data to read +@param[in] len length of data +@param[in] mtype mtype of data +@param[in] unsigned_type if the data is unsigned +@return the integer value from the data */ +inline +ib_uint64_t +row_parse_int( + const byte* data, + ulint len, + ulint mtype, + bool unsigned_type); + /** Result of row_search_index_entry */ enum row_search_result { ROW_FOUND = 0, /*!< the record was found */ diff --git a/storage/innobase/include/row0row.ic b/storage/innobase/include/row0row.ic index 08c0f18e95b..8a32bb3ffd2 100644 --- a/storage/innobase/include/row0row.ic +++ b/storage/innobase/include/row0row.ic @@ -173,3 +173,52 @@ row_build_row_ref_fast( } } } + +/** Parse the integer data from specified data, which could be +DATA_INT, DATA_FLOAT or DATA_DOUBLE. If the value is less than 0 +and the type is not unsigned then we reset the value to 0 +@param[in] data data to read +@param[in] len length of data +@param[in] mtype mtype of data +@param[in] unsigned_type if the data is unsigned +@return the integer value from the data */ +ib_uint64_t +row_parse_int( + const byte* data, + ulint len, + ulint mtype, + bool unsigned_type) +{ + ib_uint64_t value = 0; + + switch (mtype) { + case DATA_INT: + + ut_a(len <= sizeof value); + value = mach_read_int_type(data, len, unsigned_type); + break; + + case DATA_FLOAT: + + ut_a(len == sizeof(float)); + value = static_cast<ib_uint64_t>(mach_float_read(data)); + break; + + case DATA_DOUBLE: + + ut_a(len == sizeof(double)); + value = static_cast<ib_uint64_t>(mach_double_read(data)); + break; + + default: + ut_error; + + } + + if (!unsigned_type && static_cast<int64_t>(value) < 0) { + value = 0; + } + + return(value); +} + diff --git a/storage/innobase/include/row0sel.h b/storage/innobase/include/row0sel.h index 3e6863208af..3f87d78e25a 100644 --- a/storage/innobase/include/row0sel.h +++ b/storage/innobase/include/row0sel.h @@ -177,34 +177,7 @@ row_search_for_mysql( MY_ATTRIBUTE((warn_unused_result)); /** Searches for rows in the database using cursor. -function is meant for temporary table that are not shared accross connection -and so lot of complexity is reduced especially locking and transaction related. -The cursor is an iterator over the table/index. - -@param[out] buf buffer for the fetched row in MySQL format -@param[in] mode search mode PAGE_CUR_L -@param[in,out] prebuilt prebuilt struct for the table handler; - this contains the info to search_tuple, - index; if search tuple contains 0 field then - we position the cursor at start or the end of - index, depending on 'mode' -@param[in] match_mode 0 or ROW_SEL_EXACT or ROW_SEL_EXACT_PREFIX -@param[in] direction 0 or ROW_SEL_NEXT or ROW_SEL_PREV; - Note: if this is != 0, then prebuilt must has a - pcur with stored position! In opening of a - cursor 'direction' should be 0. -@return DB_SUCCESS or error code */ -dberr_t -row_search_no_mvcc( - byte* buf, - page_cur_mode_t mode, - row_prebuilt_t* prebuilt, - ulint match_mode, - ulint direction) - MY_ATTRIBUTE((warn_unused_result)); - -/** Searches for rows in the database using cursor. -Function is mainly used for tables that are shared accorss connection and +Function is mainly used for tables that are shared across connections and so it employs technique that can help re-construct the rows that transaction is suppose to see. It also has optimization such as pre-caching the rows, using AHI, etc. @@ -221,15 +194,6 @@ It also has optimization such as pre-caching the rows, using AHI, etc. Note: if this is != 0, then prebuilt must has a pcur with stored position! In opening of a cursor 'direction' should be 0. -@param[in] ins_sel_stmt if true, then this statement is - insert .... select statement. For normal table - this can be detected by checking out locked - tables using trx->mysql_n_tables_locked > 0 - condition. For intrinsic table - external_lock is not invoked and so condition - above will not stand valid instead this is - traced using alternative condition - at caller level. @return DB_SUCCESS or error code */ dberr_t row_search_mvcc( @@ -266,15 +230,12 @@ row_search_check_if_query_cache_permitted( trx_t* trx, /*!< in: transaction object */ const char* norm_name); /*!< in: concatenation of database name, '/' char, table name */ -/*******************************************************************//** -Read the max AUTOINC value from an index. -@return DB_SUCCESS if all OK else error code */ -dberr_t -row_search_max_autoinc( -/*===================*/ - dict_index_t* index, /*!< in: index to search */ - const char* col_name, /*!< in: autoinc column name */ - ib_uint64_t* value) /*!< out: AUTOINC value read */ +/** Read the max AUTOINC value from an index. +@param[in] index index starting with an AUTO_INCREMENT column +@return the largest AUTO_INCREMENT value +@retval 0 if no records were found */ +ib_uint64_t +row_search_max_autoinc(dict_index_t* index) MY_ATTRIBUTE((nonnull, warn_unused_result)); /** A structure for caching column values for prefetched rows */ diff --git a/storage/innobase/include/row0sel.ic b/storage/innobase/include/row0sel.ic index a816e4440e2..d14b41e3f5f 100644 --- a/storage/innobase/include/row0sel.ic +++ b/storage/innobase/include/row0sel.ic @@ -134,12 +134,5 @@ row_search_for_mysql( ulint match_mode, ulint direction) { - if (!dict_table_is_intrinsic(prebuilt->table)) { - return(row_search_mvcc( - buf, mode, prebuilt, match_mode, direction)); - } else { - return(row_search_no_mvcc( - buf, mode, prebuilt, match_mode, direction)); - } + return(row_search_mvcc(buf, mode, prebuilt, match_mode, direction)); } - diff --git a/storage/innobase/include/row0upd.h b/storage/innobase/include/row0upd.h index 3c1033fe419..86b282553d8 100644 --- a/storage/innobase/include/row0upd.h +++ b/storage/innobase/include/row0upd.h @@ -33,12 +33,9 @@ Created 12/27/1996 Heikki Tuuri #include "dict0types.h" #include "trx0types.h" #include <stack> - -#ifndef UNIV_HOTBACKUP -# include "btr0pcur.h" -# include "que0types.h" -# include "pars0types.h" -#endif /* !UNIV_HOTBACKUP */ +#include "btr0pcur.h" +#include "que0types.h" +#include "pars0types.h" /** The std::deque to store cascade update nodes, that uses mem_heap_t as allocator. */ @@ -80,7 +77,7 @@ upd_get_nth_field( #else # define upd_get_nth_field(update, n) ((update)->fields + (n)) #endif -#ifndef UNIV_HOTBACKUP + /*********************************************************************//** Sets an index field number to be updated by an update vector field. */ UNIV_INLINE @@ -191,7 +188,6 @@ row_upd_changes_disowned_external( /*==============================*/ const upd_t* update) /*!< in: update vector */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Replaces the new column values stored in the update vector to the record given. No field size changes are allowed. This function is @@ -207,7 +203,7 @@ row_upd_rec_in_place( const upd_t* update, /*!< in: update vector */ page_zip_des_t* page_zip);/*!< in: compressed page with enough space available, or NULL */ -#ifndef UNIV_HOTBACKUP + /***************************************************************//** Builds an update vector from those fields which in a secondary index entry differ from a record that has the equal ordering fields. NOTE: we compare @@ -403,7 +399,6 @@ que_thr_t* row_upd_step( /*=========*/ que_thr_t* thr); /*!< in: query thread */ -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Parses the log data of system field values. @return log data end or NULL */ @@ -449,7 +444,6 @@ struct upd_field_t{ index. If this field is a virtual column, then field_no represents the nth virtual column in the table */ -#ifndef UNIV_HOTBACKUP unsigned orig_len:16; /*!< original length of the locally stored part of an externally stored column, or 0 */ @@ -457,7 +451,6 @@ struct upd_field_t{ value: it refers to column values and constants in the symbol table of the query graph */ -#endif /* !UNIV_HOTBACKUP */ dfield_t new_val; /*!< new value for the column */ dfield_t* old_v_val; /*!< old value for the virtual column */ }; @@ -516,7 +509,6 @@ struct upd_t{ }; -#ifndef UNIV_HOTBACKUP /* Update node structure which also implements the delete operation of a row */ @@ -659,7 +651,6 @@ struct upd_node_t{ #define UPD_NODE_NO_SIZE_CHANGE 2 /* no record field size will be changed in the update */ -#endif /* !UNIV_HOTBACKUP */ #ifndef UNIV_NONINL #include "row0upd.ic" diff --git a/storage/innobase/include/row0upd.ic b/storage/innobase/include/row0upd.ic index ab1dc5c7076..8b794e47a07 100644 --- a/storage/innobase/include/row0upd.ic +++ b/storage/innobase/include/row0upd.ic @@ -24,12 +24,10 @@ Created 12/27/1996 Heikki Tuuri *******************************************************/ #include "mtr0log.h" -#ifndef UNIV_HOTBACKUP -# include "trx0trx.h" -# include "trx0undo.h" -# include "row0row.h" -# include "lock0lock.h" -#endif /* !UNIV_HOTBACKUP */ +#include "trx0trx.h" +#include "trx0undo.h" +#include "row0row.h" +#include "lock0lock.h" #include "page0zip.h" /*********************************************************************//** @@ -87,7 +85,6 @@ upd_get_nth_field( } #endif /* UNIV_DEBUG */ -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Sets an index field number to be updated by an update vector field. */ UNIV_INLINE @@ -220,4 +217,3 @@ row_upd_rec_sys_fields( trx_write_roll_ptr(rec + offset + DATA_TRX_ID_LEN, roll_ptr); } } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/sess0sess.h b/storage/innobase/include/sess0sess.h deleted file mode 100644 index 3778a45d64f..00000000000 --- a/storage/innobase/include/sess0sess.h +++ /dev/null @@ -1,146 +0,0 @@ -/***************************************************************************** - -Copyright (c) 2013, 2014, Oracle and/or its affiliates. All Rights Reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/**************************************************//** -@file include/sess0sess.h -InnoDB session state tracker. -Multi file, shared, system tablespace implementation. - -Created 2014-04-30 by Krunal Bauskar -*******************************************************/ - -#ifndef sess0sess_h -#define sess0sess_h - -#include "univ.i" -#include "dict0mem.h" -#include "ut0new.h" - -#include <map> - -class dict_intrinsic_table_t { - -public: - /** Constructor - @param[in,out] handler table handler. */ - dict_intrinsic_table_t(dict_table_t* handler) - : - m_handler(handler) - { - /* Do nothing. */ - } - - /** Destructor */ - ~dict_intrinsic_table_t() - { - m_handler = NULL; - } - -public: - - /* Table Handler holding other metadata information commonly needed - for any table. */ - dict_table_t* m_handler; -}; - -/** InnoDB private data that is cached in THD */ -typedef std::map< - std::string, - dict_intrinsic_table_t*, - std::less<std::string>, - ut_allocator<std::pair<const std::string, dict_intrinsic_table_t*> > > - table_cache_t; - -class innodb_session_t { -public: - /** Constructor */ - innodb_session_t() - : m_trx(), - m_open_tables() - { - /* Do nothing. */ - } - - /** Destructor */ - ~innodb_session_t() - { - m_trx = NULL; - - for (table_cache_t::iterator it = m_open_tables.begin(); - it != m_open_tables.end(); - ++it) { - delete(it->second); - } - } - - /** Cache table handler. - @param[in] table_name name of the table - @param[in,out] table table handler to register */ - void register_table_handler( - const char* table_name, - dict_table_t* table) - { - ut_ad(lookup_table_handler(table_name) == NULL); - m_open_tables.insert(table_cache_t::value_type( - table_name, new dict_intrinsic_table_t(table))); - } - - /** Lookup for table handler given table_name. - @param[in] table_name name of the table to lookup */ - dict_table_t* lookup_table_handler( - const char* table_name) - { - table_cache_t::iterator it = m_open_tables.find(table_name); - return((it == m_open_tables.end()) - ? NULL : it->second->m_handler); - } - - /** Remove table handler entry. - @param[in] table_name name of the table to remove */ - void unregister_table_handler( - const char* table_name) - { - table_cache_t::iterator it = m_open_tables.find(table_name); - if (it == m_open_tables.end()) { - return; - } - - delete(it->second); - m_open_tables.erase(table_name); - } - - /** Count of register table handler. - @return number of register table handlers */ - uint count_register_table_handler() const - { - return(static_cast<uint>(m_open_tables.size())); - } - -public: - - /** transaction handler. */ - trx_t* m_trx; - - /** Handler of tables that are created or open but not added - to InnoDB dictionary as they are session specific. - Currently, limited to intrinsic temporary tables only. */ - table_cache_t m_open_tables; -}; - - -#endif /* sess0sess_h */ diff --git a/storage/innobase/include/srv0mon.h b/storage/innobase/include/srv0mon.h index 125401373ba..5c19e735806 100644 --- a/storage/innobase/include/srv0mon.h +++ b/storage/innobase/include/srv0mon.h @@ -38,9 +38,6 @@ Created 12/15/2009 Jimmy Yang #include <stdint.h> -#ifndef UNIV_HOTBACKUP - - /** Possible status values for "mon_status" in "struct monitor_value" */ enum monitor_running_status { MONITOR_STARTED = 1, /*!< Monitor has been turned on */ @@ -180,6 +177,7 @@ enum monitor_id_t { MONITOR_OVLD_INDEX_PAGES_WRITTEN, MONITOR_OVLD_NON_INDEX_PAGES_WRITTEN, MONITOR_OVLD_PAGES_READ, + MONITOR_OVLD_PAGES0_READ, MONITOR_OVLD_INDEX_SEC_REC_CLUSTER_READS, MONITOR_OVLD_INDEX_SEC_REC_CLUSTER_READS_AVOIDED, MONITOR_OVLD_BYTE_READ, @@ -345,16 +343,8 @@ enum monitor_id_t { MONITOR_PAD_DECREMENTS, /* New monitor variables for page compression */ MONITOR_OVLD_PAGE_COMPRESS_SAVED, - MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT512, - MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT1024, - MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT2048, - MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT4096, - MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT8192, - MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT16384, - MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT32768, MONITOR_OVLD_PAGES_PAGE_COMPRESSED, MONITOR_OVLD_PAGE_COMPRESSED_TRIM_OP, - MONITOR_OVLD_PAGE_COMPRESSED_TRIM_OP_SAVED, MONITOR_OVLD_PAGES_PAGE_DECOMPRESSED, MONITOR_OVLD_PAGES_PAGE_COMPRESSION_ERROR, @@ -898,9 +888,5 @@ srv_mon_default_on(void); #ifndef UNIV_NONINL #include "srv0mon.ic" #endif -#else /* !UNIV_HOTBACKUP */ -# define MONITOR_INC(x) ((void) 0) -# define MONITOR_DEC(x) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ #endif diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 5f8893b3e28..a8c962d4a45 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -3,7 +3,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, 2009, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2016, MariaDB Corporation +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -48,7 +48,6 @@ Created 10/10/1995 Heikki Tuuri #include "mysql/psi/psi.h" #include "univ.i" -#ifndef UNIV_HOTBACKUP #include "log0log.h" #include "os0event.h" #include "que0types.h" @@ -115,20 +114,6 @@ struct srv_stats_t { /** Number of bytes saved by page compression */ ulint_ctr_64_t page_compression_saved; - /** Number of 512Byte TRIM by page compression */ - ulint_ctr_64_t page_compression_trim_sect512; - /** Number of 1K TRIM by page compression */ - ulint_ctr_64_t page_compression_trim_sect1024; - /** Number of 2K TRIM by page compression */ - ulint_ctr_64_t page_compression_trim_sect2048; - /** Number of 4K TRIM by page compression */ - ulint_ctr_64_t page_compression_trim_sect4096; - /** Number of 8K TRIM by page compression */ - ulint_ctr_64_t page_compression_trim_sect8192; - /** Number of 16K TRIM by page compression */ - ulint_ctr_64_t page_compression_trim_sect16384; - /** Number of 32K TRIM by page compression */ - ulint_ctr_64_t page_compression_trim_sect32768; /* Number of index pages written */ ulint_ctr_64_t index_pages_written; /* Number of non index pages written */ @@ -137,8 +122,6 @@ struct srv_stats_t { ulint_ctr_64_t pages_page_compressed; /* Number of TRIM operations induced by page compression */ ulint_ctr_64_t page_compressed_trim_op; - /* Number of TRIM operations saved by using actual write size knowledge */ - ulint_ctr_64_t page_compressed_trim_op_saved; /* Number of pages decompressed with page compression */ ulint_ctr_64_t pages_page_decompressed; /* Number of page compression errors */ @@ -189,6 +172,15 @@ struct srv_stats_t { /** Number of times prefix optimization avoided triggering cluster lookup */ ulint_ctr_64_t n_sec_rec_cluster_reads_avoided; + + /** Number of times page 0 is read from tablespace */ + ulint_ctr_64_t page0_read; + + /** Number of encryption_get_latest_key_version calls */ + ulint_ctr_64_t n_key_requests; + + /** Number of log scrub operations */ + ulint_ctr_64_t n_log_scrubs; }; extern const char* srv_main_thread_op_info; @@ -196,13 +188,16 @@ extern const char* srv_main_thread_op_info; /** Prefix used by MySQL to indicate pre-5.1 table name encoding */ extern const char srv_mysql50_table_name_prefix[10]; -/* The monitor thread waits on this event. */ +/** Event to signal srv_monitor_thread. Not protected by a mutex. +Set after setting srv_print_innodb_monitor. */ extern os_event_t srv_monitor_event; -/* The error monitor thread waits on this event. */ +/** Event to signal the shutdown of srv_error_monitor_thread. +Not protected by a mutex. */ extern os_event_t srv_error_event; -/** The buffer pool dump/load thread waits on this event. */ +/** Event for waking up buf_dump_thread. Not protected by a mutex. +Set on shutdown or by buf_dump_start() or buf_load_start(). */ extern os_event_t srv_buf_dump_event; /** The buffer pool resize thread waits on this event. */ @@ -287,11 +282,6 @@ extern my_bool srv_numa_interleave; /* Use trim operation */ extern my_bool srv_use_trim; -/* Use posix fallocate */ -#ifdef HAVE_POSIX_FALLOCATE -extern my_bool srv_use_posix_fallocate; -#endif - /* Use atomic writes i.e disable doublewrite buffer */ extern my_bool srv_use_atomic_writes; @@ -308,8 +298,6 @@ extern long srv_mtflush_threads; /* If this flag is TRUE, then we will use multi threaded flush. */ extern my_bool srv_use_mtflush; -#endif /* !UNIV_HOTBACKUP */ - /** Server undo tablespaces directory, can be absolute path. */ extern char* srv_undo_dir; @@ -330,6 +318,9 @@ extern ulong srv_undo_logs; /** Maximum size of undo tablespace. */ extern unsigned long long srv_max_undo_log_size; +extern uint srv_n_fil_crypt_threads; +extern uint srv_n_fil_crypt_threads_started; + /** Rate at which UNDO records should be purged. */ extern ulong srv_purge_rseg_truncate_frequency; @@ -348,7 +339,6 @@ extern const ulint SRV_UNDO_TABLESPACE_SIZE_IN_PAGES; extern char* srv_log_group_home_dir; -#ifndef UNIV_HOTBACKUP /** Maximum number of srv_n_log_files, or innodb_log_files_in_group */ #define SRV_N_LOG_FILES_MAX 100 extern ulong srv_n_log_files; @@ -463,17 +453,19 @@ extern double srv_adaptive_flushing_lwm; extern ulong srv_flushing_avg_loops; extern ulong srv_force_recovery; -#ifndef DBUG_OFF -extern ulong srv_force_recovery_crash; -#endif /* !DBUG_OFF */ -extern ulint srv_fast_shutdown; /*!< If this is 1, do not do a +extern uint srv_fast_shutdown; /*!< If this is 1, do not do a purge and index buffer merge. If this 2, do not even flush the buffer pool to data files at the shutdown: we effectively 'crash' InnoDB (but lose no committed transactions). */ + +/** Signal to shut down InnoDB (NULL if shutdown was signaled, or if +running in innodb_read_only mode, srv_read_only_mode) */ +extern volatile st_my_thread_var *srv_running; + extern ibool srv_innodb_status; extern unsigned long long srv_stats_transient_sample_pages; @@ -501,27 +493,25 @@ extern my_bool srv_print_innodb_monitor; extern my_bool srv_print_innodb_lock_monitor; extern ibool srv_print_verbose_log; -extern ibool srv_monitor_active; -extern ibool srv_error_monitor_active; +extern bool srv_monitor_active; +extern bool srv_error_monitor_active; /* TRUE during the lifetime of the buffer pool dump/load thread */ -extern ibool srv_buf_dump_thread_active; +extern bool srv_buf_dump_thread_active; /* true during the lifetime of the buffer pool resize thread */ extern bool srv_buf_resize_thread_active; /* TRUE during the lifetime of the stats thread */ -extern ibool srv_dict_stats_thread_active; +extern bool srv_dict_stats_thread_active; /* TRUE if enable log scrubbing */ extern my_bool srv_scrub_log; -/* TRUE during the lifetime of the log scrub thread */ -extern ibool srv_log_scrub_thread_active; extern ulong srv_n_spin_wait_rounds; extern ulong srv_n_free_tickets_to_enter; extern ulong srv_thread_sleep_delay; -extern ulong srv_spin_wait_delay; +extern uint srv_spin_wait_delay; extern ibool srv_priority_boost; extern ulint srv_truncated_status_writes; @@ -537,6 +527,7 @@ extern my_bool srv_purge_view_update_only_debug; /** Value of MySQL global used to disable master thread. */ extern my_bool srv_master_thread_disabled_debug; +extern uint srv_sys_space_size_debug; #endif /* UNIV_DEBUG */ #define SRV_SEMAPHORE_WAIT_EXTENSION 7200 @@ -620,6 +611,9 @@ do { \ do { \ PSI_THREAD_CALL(delete_current_thread)(); \ } while (0) +# else +# define pfs_register_thread(key) +# define pfs_delete_thread() # endif /* UNIV_PFS_THREAD */ #ifdef HAVE_PSI_STAGE_INTERFACE @@ -655,8 +649,6 @@ extern PSI_stage_info srv_stage_alter_table_read_pk_internal_sort; extern PSI_stage_info srv_stage_buffer_pool_load; #endif /* HAVE_PSI_STAGE_INTERFACE */ -#endif /* !UNIV_HOTBACKUP */ - #ifndef _WIN32 /** Alternatives for the file flush option in Unix; see the InnoDB manual about what these mean */ @@ -728,12 +720,6 @@ enum srv_stats_method_name_enum { typedef enum srv_stats_method_name_enum srv_stats_method_name_t; -#ifdef UNIV_DEBUG -/** Force all user tables to use page compression. */ -extern ulong srv_debug_compress; -#endif /* UNIV_DEBUG */ - -#ifndef UNIV_HOTBACKUP /** Types of threads existing in the system. */ enum srv_thread_type { SRV_NONE, /*!< None */ @@ -751,22 +737,11 @@ void srv_boot(void); /*==========*/ /*********************************************************************//** -Initializes the server. */ -void -srv_init(void); -/*==========*/ -/*********************************************************************//** Frees the data structures created in srv_init(). */ void srv_free(void); /*==========*/ /*********************************************************************//** -Initializes the synchronization primitives, memory system, and the thread -local storage. */ -void -srv_general_init(void); -/*==================*/ -/*********************************************************************//** Sets the info describing an i/o thread current state. */ void srv_set_io_thread_op_info( @@ -921,34 +896,16 @@ ulint srv_get_task_queue_length(void); /*===========================*/ -/*********************************************************************//** -Releases threads of the type given from suspension in the thread table. -NOTE! The server mutex has to be reserved by the caller! -@return number of threads released: this may be less than n if not -enough threads were suspended at the moment */ -ulint -srv_release_threads( -/*================*/ - enum srv_thread_type type, /*!< in: thread type */ - ulint n); /*!< in: number of threads to release */ - -/**********************************************************************//** -Check whether any background thread are active. If so print which thread -is active. Send the threads wakeup signal. -@return name of thread that is active or NULL */ -const char* -srv_any_background_threads_are_active(void); -/*=======================================*/ - -/**********************************************************************//** -Wakeup the purge threads. */ +/** Ensure that a given number of threads of the type given are running +(or are already terminated). +@param[in] type thread type +@param[in] n number of threads that have to run */ void -srv_purge_wakeup(void); -/*==================*/ +srv_release_threads(enum srv_thread_type type, ulint n); -/** Call exit(3) */ +/** Wakeup the purge threads. */ void -srv_fatal_error(); +srv_purge_wakeup(); /** Check if tablespace is being truncated. (Ignore system-tablespace as we don't re-create the tablespace @@ -1025,7 +982,8 @@ struct export_var_t{ ulint innodb_os_log_pending_fsyncs; /*!< fil_n_pending_log_flushes */ ulint innodb_page_size; /*!< UNIV_PAGE_SIZE */ ulint innodb_pages_created; /*!< buf_pool->stat.n_pages_created */ - ulint innodb_pages_read; /*!< buf_pool->stat.n_pages_read */ + ulint innodb_pages_read; /*!< buf_pool->stat.n_pages_read*/ + ulint innodb_page0_read; /*!< srv_stats.page0_read */ ulint innodb_pages_written; /*!< buf_pool->stat.n_pages_written */ ulint innodb_row_lock_waits; /*!< srv_n_lock_wait_count */ ulint innodb_row_lock_current_waits; /*!< srv_n_lock_wait_current_count */ @@ -1073,20 +1031,6 @@ struct export_var_t{ int64_t innodb_page_compression_saved;/*!< Number of bytes saved by page compression */ - int64_t innodb_page_compression_trim_sect512;/*!< Number of 512b TRIM - by page compression */ - int64_t innodb_page_compression_trim_sect1024;/*!< Number of 1K TRIM - by page compression */ - int64_t innodb_page_compression_trim_sect2048;/*!< Number of 2K TRIM - by page compression */ - int64_t innodb_page_compression_trim_sect4096;/*!< Number of 4K byte TRIM - by page compression */ - int64_t innodb_page_compression_trim_sect8192;/*!< Number of 8K TRIM - by page compression */ - int64_t innodb_page_compression_trim_sect16384;/*!< Number of 16K TRIM - by page compression */ - int64_t innodb_page_compression_trim_sect32768;/*!< Number of 32K TRIM - by page compression */ int64_t innodb_index_pages_written; /*!< Number of index pages written */ int64_t innodb_non_index_pages_written; /*!< Number of non index pages @@ -1095,8 +1039,6 @@ struct export_var_t{ compressed by page compression */ int64_t innodb_page_compressed_trim_op;/*!< Number of TRIM operations induced by page compression */ - int64_t innodb_page_compressed_trim_op_saved;/*!< Number of TRIM operations - saved by page compression */ int64_t innodb_pages_page_decompressed;/*!< Number of pages decompressed by page compression */ @@ -1115,6 +1057,7 @@ struct export_var_t{ ulint innodb_encryption_rotation_pages_modified; ulint innodb_encryption_rotation_pages_flushed; ulint innodb_encryption_rotation_estimated_iops; + int64_t innodb_encryption_key_requests; ulint innodb_scrub_page_reorganizations; ulint innodb_scrub_page_splits; @@ -1122,6 +1065,7 @@ struct export_var_t{ ulint innodb_scrub_page_split_failures_out_of_filespace; ulint innodb_scrub_page_split_failures_missing_index; ulint innodb_scrub_page_split_failures_unknown; + int64_t innodb_scrub_log; }; /** Thread slot in the thread table. */ @@ -1149,20 +1093,6 @@ struct srv_slot_t{ (only used for user threads) */ }; -#else /* !UNIV_HOTBACKUP */ -# define srv_use_adaptive_hash_indexes FALSE -# define srv_use_native_aio FALSE -# define srv_numa_interleave FALSE -# define srv_force_recovery 0UL -# define srv_set_io_thread_op_info(t,info) ((void) 0) -# define srv_reset_io_thread_op_info() ((void) 0) -# define srv_is_being_started 0 -# define srv_win_file_flush_method SRV_WIN_IO_UNBUFFERED -# define srv_unix_file_flush_method SRV_UNIX_O_DSYNC -# define srv_start_raw_disk_in_use 0 -# define srv_file_per_table 1 -#endif /* !UNIV_HOTBACKUP */ - #ifdef WITH_WSREP UNIV_INTERN void diff --git a/storage/innobase/include/srv0start.h b/storage/innobase/include/srv0start.h index 0dd98e5b19b..8de9e3457fc 100644 --- a/storage/innobase/include/srv0start.h +++ b/storage/innobase/include/srv0start.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -33,22 +34,6 @@ Created 10/10/1995 Heikki Tuuri // Forward declaration struct dict_table_t; -#ifdef DBUG_OFF -# define RECOVERY_CRASH(x) do {} while(0) -#else -# define RECOVERY_CRASH(x) do { \ - if (srv_force_recovery_crash == x) { \ - fprintf(stderr, "innodb_force_recovery_crash=%lu\n", \ - srv_force_recovery_crash); \ - fflush(stderr); \ - _exit(3); \ - } \ -} while (0) -#endif /* DBUG_OFF */ - -/** Log 'spaces' have id's >= this */ -#define SRV_LOG_SPACE_FIRST_ID 0xFFFFFFF0UL - /** If buffer pool is less than the size, only one buffer pool instance is used. */ #define BUF_POOL_SIZE_THRESHOLD (1024 * 1024 * 1024) @@ -74,7 +59,7 @@ char* srv_add_path_separator_if_needed( /*=============================*/ char* str); /*!< in: null-terminated character string */ -#ifndef UNIV_HOTBACKUP + /****************************************************************//** Starts Innobase and creates a new database if database files are not found and the user wants. @@ -82,11 +67,14 @@ are not found and the user wants. dberr_t innobase_start_or_create_for_mysql(void); /*====================================*/ +/** Shut down InnoDB. */ +void +innodb_shutdown(); + /****************************************************************//** -Shuts down the Innobase database. -@return DB_SUCCESS or error code */ -dberr_t -innobase_shutdown_for_mysql(void); +Shuts down background threads that can generate undo pages. */ +void +srv_shutdown_bg_undo_sources(void); /******************************************************************** Signal all per-table background threads to shutdown, and wait for them to do @@ -109,11 +97,6 @@ srv_path_copy( const char* table_name) /*!< in: source table name */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -/** -Shutdown all background threads created by InnoDB. */ -void -srv_shutdown_all_bg_threads(); - /** Get the meta-data filename from the table name for a single-table tablespace. @param[in] table table object @@ -154,7 +137,6 @@ extern bool srv_startup_is_before_trx_rollback_phase; /** TRUE if a raw partition is in use */ extern ibool srv_start_raw_disk_in_use; - /** Shutdown state */ enum srv_shutdown_t { SRV_SHUTDOWN_NONE = 0, /*!< Database running normally */ @@ -174,6 +156,4 @@ enum srv_shutdown_t { /** At a shutdown this value climbs from SRV_SHUTDOWN_NONE to SRV_SHUTDOWN_CLEANUP and then to SRV_SHUTDOWN_LAST_PHASE, and so on */ extern enum srv_shutdown_t srv_shutdown_state; -#endif /* !UNIV_HOTBACKUP */ - #endif diff --git a/storage/innobase/include/sync0rw.h b/storage/innobase/include/sync0rw.h index 396156e9518..9a9da2d145a 100644 --- a/storage/innobase/include/sync0rw.h +++ b/storage/innobase/include/sync0rw.h @@ -34,7 +34,6 @@ Created 9/11/1995 Heikki Tuuri #define sync0rw_h #include "univ.i" -#ifndef UNIV_HOTBACKUP #include "ut0counter.h" #include "os0event.h" #include "ut0mutex.h" @@ -42,8 +41,6 @@ Created 9/11/1995 Heikki Tuuri /** Enable semaphore request instrumentation */ extern my_bool srv_instrument_semaphores; -#endif /* !UNIV_HOTBACKUP */ - /** Counters for RW locks. */ struct rw_lock_stats_t { typedef ib_counter_t<int64_t, IB_N_SLOTS> int64_counter_t; @@ -96,7 +93,6 @@ enum rw_lock_type_t { RW_NO_LATCH = 8 }; -#ifndef UNIV_HOTBACKUP /* We decrement lock_word by X_LOCK_DECR for each x_lock. It is also the start value for the lock_word, meaning that it limits the maximum number of concurrent read locks before the rw_lock breaks. */ @@ -897,11 +893,8 @@ pfs_rw_lock_free_func( rw_lock_t* lock); /*!< in: rw-lock */ #endif /* UNIV_PFS_RWLOCK */ - #ifndef UNIV_NONINL #include "sync0rw.ic" #endif /* !UNIV_NONINL */ -#endif /* !UNIV_HOTBACKUP */ - #endif /* sync0rw.h */ diff --git a/storage/innobase/include/sync0sync.h b/storage/innobase/include/sync0sync.h index 7fddada10f8..7157b07e9d0 100644 --- a/storage/innobase/include/sync0sync.h +++ b/storage/innobase/include/sync0sync.h @@ -136,7 +136,6 @@ extern mysql_pfs_key_t index_online_log_key; extern mysql_pfs_key_t dict_table_stats_key; extern mysql_pfs_key_t trx_sys_rw_lock_key; extern mysql_pfs_key_t hash_table_locks_key; -extern mysql_pfs_key_t master_key_id_mutex_key; #endif /* UNIV_PFS_RWLOCK */ /** Prints info of the sync system. diff --git a/storage/innobase/include/sync0types.h b/storage/innobase/include/sync0types.h index 5fdb916c54c..68b6a44dc25 100644 --- a/storage/innobase/include/sync0types.h +++ b/storage/innobase/include/sync0types.h @@ -384,7 +384,6 @@ enum latch_id_t { LATCH_ID_HASH_TABLE_RW_LOCK, LATCH_ID_BUF_CHUNK_MAP_LATCH, LATCH_ID_SYNC_DEBUG_MUTEX, - LATCH_ID_MASTER_KEY_ID_MUTEX, LATCH_ID_SCRUB_STAT_MUTEX, LATCH_ID_DEFRAGMENT_MUTEX, LATCH_ID_BTR_DEFRAGMENT_MUTEX, @@ -1033,10 +1032,10 @@ struct latch_t { return(m_temp_fsp); } - /** Set the temporary tablespace flag. The latch order constraints - are different for intrinsic tables. We don't always acquire the - index->lock. We need to figure out the context and add some special - rules during the checks. */ + /** Set the temporary tablespace flag. (For internal temporary + tables, MySQL 5.7 does not always acquire the index->lock. We + need to figure out the context and add some special rules + during the checks.) */ void set_temp_fsp() UNIV_NOTHROW { @@ -1090,26 +1089,11 @@ struct btrsea_sync_check : public sync_check_functor_t { virtual bool operator()(const latch_level_t level) { /* If calling thread doesn't hold search latch then - check if there are latch level exception provided. - - Note: Optimizer has added InnoDB intrinsic table as an - alternative to MyISAM intrinsic table. With this a new - control flow comes into existence, it is: - - Server -> Plugin -> SE - - Plugin in this case is I_S which is sharing the latch vector - of InnoDB and so there could be lock conflicts. Ideally - the Plugin should use a difference namespace latch vector - as it doesn't have any depedency with SE latching protocol. - - Added check that will allow thread to hold I_S latches */ + check if there are latch level exception provided. */ if (!m_has_search_latch && (level != SYNC_SEARCH_SYS - && level != SYNC_FTS_CACHE - && level != SYNC_TRX_I_S_RWLOCK - && level != SYNC_TRX_I_S_LAST_READ)) { + && level != SYNC_FTS_CACHE)) { m_result = true; diff --git a/storage/innobase/include/trx0purge.h b/storage/innobase/include/trx0purge.h index 8917169dc94..44eaa9d4f1e 100644 --- a/storage/innobase/include/trx0purge.h +++ b/storage/innobase/include/trx0purge.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -399,7 +400,10 @@ struct trx_purge_t{ log operation can prevent this by obtaining an s-latch here. It also protects state and running */ - os_event_t event; /*!< State signal event */ + os_event_t event; /*!< State signal event; + os_event_set() and os_event_reset() + are protected by trx_purge_t::latch + X-lock */ ulint n_stop; /*!< Counter to track number stops */ volatile bool running; /*!< true, if purge is active, we check this without the latch too */ diff --git a/storage/innobase/include/trx0rec.h b/storage/innobase/include/trx0rec.h index b7a2deac63e..90f4604043b 100644 --- a/storage/innobase/include/trx0rec.h +++ b/storage/innobase/include/trx0rec.h @@ -35,9 +35,7 @@ Created 3/26/1996 Heikki Tuuri #include "rem0types.h" #include "page0types.h" #include "row0log.h" - -#ifndef UNIV_HOTBACKUP -# include "que0types.h" +#include "que0types.h" /***********************************************************************//** Copies the undo record to the heap. @@ -276,7 +274,6 @@ trx_undo_prev_version_build( into this function by purge thread or not. And if we read "after image" of undo log */ -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses a redo log record of adding an undo log record. @return end of log record or NULL */ @@ -344,8 +341,6 @@ trx_undo_read_v_idx( bool* is_undo_log, ulint* field_no); -#ifndef UNIV_HOTBACKUP - /* Types of an undo log record: these have to be smaller than 16, as the compilation info multiplied by 16 is ORed to this value in an undo log record */ @@ -373,6 +368,4 @@ record */ #include "trx0rec.ic" #endif -#endif /* !UNIV_HOTBACKUP */ - #endif /* trx0rec_h */ diff --git a/storage/innobase/include/trx0rec.ic b/storage/innobase/include/trx0rec.ic index 111c05c60aa..c2c756484b2 100644 --- a/storage/innobase/include/trx0rec.ic +++ b/storage/innobase/include/trx0rec.ic @@ -23,7 +23,6 @@ Transaction undo log record Created 3/26/1996 Heikki Tuuri *******************************************************/ -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Reads from an undo log record the record type. @return record type */ @@ -98,4 +97,3 @@ trx_undo_rec_copy( ut_ad(len < UNIV_PAGE_SIZE); return((trx_undo_rec_t*) mem_heap_dup(heap, undo_rec, len)); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/trx0sys.h b/storage/innobase/include/trx0sys.h index ddf535158b6..ae46d2abae9 100644 --- a/storage/innobase/include/trx0sys.h +++ b/storage/innobase/include/trx0sys.h @@ -31,7 +31,6 @@ Created 3/26/1996 Heikki Tuuri #include "buf0buf.h" #include "fil0fil.h" #include "trx0types.h" -#ifndef UNIV_HOTBACKUP #include "mem0mem.h" #include "mtr0mtr.h" #include "ut0byte.h" @@ -367,40 +366,6 @@ Check if there are any active (non-prepared) transactions. ulint trx_sys_any_active_transactions(void); /*=================================*/ -#else /* !UNIV_HOTBACKUP */ -/*****************************************************************//** -Prints to stderr the MySQL binlog info in the system header if the -magic number shows it valid. */ -void -trx_sys_print_mysql_binlog_offset_from_page( -/*========================================*/ - const byte* page); /*!< in: buffer containing the trx - system header page, i.e., page number - TRX_SYS_PAGE_NO in the tablespace */ -/*****************************************************************//** -Reads the file format id from the first system table space file. -Even if the call succeeds and returns TRUE, the returned format id -may be ULINT_UNDEFINED signalling that the format id was not present -in the data file. -@return TRUE if call succeeds */ -ibool -trx_sys_read_file_format_id( -/*========================*/ - const char *pathname, /*!< in: pathname of the first system - table space file */ - ulint *format_id); /*!< out: file format of the system table - space */ -/*****************************************************************//** -Reads the file format id from the given per-table data file. -@return TRUE if call succeeds */ -ibool -trx_sys_read_pertable_file_format_id( -/*=================================*/ - const char *pathname, /*!< in: pathname of a per-table - datafile */ - ulint *format_id); /*!< out: file format of the per-table - data file */ -#endif /* !UNIV_HOTBACKUP */ /*****************************************************************//** Get the name representation of the file format from its id. @return pointer to the max format name */ @@ -590,7 +555,6 @@ identifier is added to this 64-bit constant. */ | TRX_SYS_FILE_FORMAT_TAG_MAGIC_N_LOW) /* @} */ -#ifndef UNIV_HOTBACKUP /** The transaction system central memory data structure. */ struct trx_sys_t { @@ -683,7 +647,6 @@ struct trx_sys_t { two) is assigned, the field TRX_SYS_TRX_ID_STORE on the transaction system page is updated */ #define TRX_SYS_TRX_ID_WRITE_MARGIN ((trx_id_t) 256) -#endif /* !UNIV_HOTBACKUP */ /** Test if trx_sys->mutex is owned. */ #define trx_sys_mutex_own() (trx_sys->mutex.is_owned()) diff --git a/storage/innobase/include/trx0sys.ic b/storage/innobase/include/trx0sys.ic index 6158aea0c48..b07f487984c 100644 --- a/storage/innobase/include/trx0sys.ic +++ b/storage/innobase/include/trx0sys.ic @@ -25,9 +25,8 @@ Created 3/26/1996 Heikki Tuuri #include "trx0trx.h" #include "data0type.h" -#ifndef UNIV_HOTBACKUP -# include "srv0srv.h" -# include "mtr0log.h" +#include "srv0srv.h" +#include "mtr0log.h" /* The typedef for rseg slot in the file copy */ typedef byte trx_sysf_rseg_t; @@ -198,7 +197,6 @@ trx_sysf_rseg_set_page_no( page_no, MLOG_4BYTES, mtr); } -#endif /* !UNIV_HOTBACKUP */ /*****************************************************************//** Writes a trx id to an index page. In case that the id size changes in @@ -230,7 +228,6 @@ trx_sys_is_noredo_rseg_slot( return(slot_id > 0 && slot_id < (srv_tmp_undo_logs + 1)); } -#ifndef UNIV_HOTBACKUP /*****************************************************************//** Reads a trx id from an index page. In case that the id size changes in some future version, this function should be used instead of @@ -503,5 +500,3 @@ trx_sys_rw_trx_add(trx_t* trx) trx_sys->rw_trx_set.insert(TrxTrack(trx->id, trx)); ut_d(trx->in_rw_trx_list = true); } - -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index 91d85255fb2..d652c2162dc 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -35,7 +36,6 @@ Created 3/26/1996 Heikki Tuuri #include "trx0types.h" #include "ut0new.h" -#ifndef UNIV_HOTBACKUP #include "lock0types.h" #include "log0log.h" #include "usr0types.h" @@ -420,7 +420,6 @@ trx_set_dict_operation( enum trx_dict_op_t op); /*!< in: operation, not TRX_DICT_OP_NONE */ -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Determines if a transaction is in the given state. The caller must hold trx_sys->mutex, or it must be the thread @@ -432,8 +431,15 @@ bool trx_state_eq( /*=========*/ const trx_t* trx, /*!< in: transaction */ - trx_state_t state) /*!< in: state */ - MY_ATTRIBUTE((warn_unused_result)); + trx_state_t state, /*!< in: state; + if state != TRX_STATE_NOT_STARTED + asserts that + trx->state != TRX_STATE_NOT_STARTED */ + bool relaxed = false) + /*!< in: whether to allow + trx->state == TRX_STATE_NOT_STARTED + after an error has been reported */ + MY_ATTRIBUTE((nonnull, warn_unused_result)); # ifdef UNIV_DEBUG /**********************************************************************//** Asserts that a transaction has been started. @@ -460,9 +466,6 @@ ibool trx_is_strict( /*==========*/ trx_t* trx); /*!< in: transaction */ -#else /* !UNIV_HOTBACKUP */ -#define trx_is_interrupted(trx) FALSE -#endif /* !UNIV_HOTBACKUP */ /*******************************************************************//** Calculates the "weight" of a transaction. The weight of one transaction @@ -1258,11 +1261,6 @@ struct trx_t { transaction branch */ trx_mod_tables_t mod_tables; /*!< List of tables that were modified by this transaction */ - /*------------------------------*/ - bool api_trx; /*!< trx started by InnoDB API */ - bool api_auto_commit;/*!< automatic commit */ - bool read_write; /*!< if read and write operation */ - /*------------------------------*/ char* detailed_error; /*!< detailed error message for last error, or empty. */ @@ -1605,6 +1603,5 @@ private: #ifndef UNIV_NONINL #include "trx0trx.ic" #endif -#endif /* !UNIV_HOTBACKUP */ #endif diff --git a/storage/innobase/include/trx0trx.ic b/storage/innobase/include/trx0trx.ic index 7120a7aaced..f0ed9b61c5a 100644 --- a/storage/innobase/include/trx0trx.ic +++ b/storage/innobase/include/trx0trx.ic @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -36,7 +37,14 @@ bool trx_state_eq( /*=========*/ const trx_t* trx, /*!< in: transaction */ - trx_state_t state) /*!< in: state */ + trx_state_t state, /*!< in: state; + if state != TRX_STATE_NOT_STARTED + asserts that + trx->state != TRX_STATE_NOT_STARTED */ + bool relaxed) + /*!< in: whether to allow + trx->state == TRX_STATE_NOT_STARTED + after an error has been reported */ { #ifdef UNIV_DEBUG switch (trx->state) { @@ -60,7 +68,9 @@ trx_state_eq( /* These states are not allowed for running transactions. */ ut_a(state == TRX_STATE_NOT_STARTED - || state == TRX_STATE_FORCED_ROLLBACK); + || state == TRX_STATE_FORCED_ROLLBACK + || (relaxed + && thd_get_error_number(trx->mysql_thd))); ut_ad(!trx->in_rw_trx_list); diff --git a/storage/innobase/include/trx0undo.h b/storage/innobase/include/trx0undo.h index 60fbb9d2304..1acb5b7c1ad 100644 --- a/storage/innobase/include/trx0undo.h +++ b/storage/innobase/include/trx0undo.h @@ -34,7 +34,6 @@ Created 3/26/1996 Heikki Tuuri #include "page0types.h" #include "trx0xa.h" -#ifndef UNIV_HOTBACKUP /***********************************************************************//** Builds a roll pointer. @return roll pointer */ @@ -75,7 +74,6 @@ trx_undo_trx_id_is_insert( /*======================*/ const byte* trx_id) /*!< in: DB_TRX_ID, followed by DB_ROLL_PTR */ MY_ATTRIBUTE((warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /*****************************************************************//** Writes a roll ptr to an index page. In case that the size changes in some future version, this function should be used instead of @@ -97,7 +95,6 @@ roll_ptr_t trx_read_roll_ptr( /*==============*/ const byte* ptr); /*!< in: pointer to memory from where to read */ -#ifndef UNIV_HOTBACKUP /** Gets an undo log page and x-latches it. @param[in] page_id page id @@ -366,7 +363,6 @@ bool trx_undo_truncate_tablespace( undo::Truncate* undo_trunc); -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses the redo log entry of an undo log page initialization. @return end of log record or NULL */ @@ -427,7 +423,7 @@ trx_undo_mem_free( #define TRX_UNDO_PREPARED 5 /* contains an undo log of an prepared transaction */ -#if !defined UNIV_HOTBACKUP && !defined UNIV_INNOCHECKSUM +#ifndef UNIV_INNOCHECKSUM /** Transaction undo log memory object; this is protected by the undo_mutex in the corresponding transaction object */ @@ -488,7 +484,7 @@ struct trx_undo_t { /*!< undo log objects in the rollback segment are chained into lists */ }; -#endif /* !UNIV_HOTBACKUP && !UNIV_INNOCHECKSUM */ +#endif /* !UNIV_INNOCHECKSUM */ /** The offset of the undo log page header on pages of the undo log */ #define TRX_UNDO_PAGE_HDR FSEG_PAGE_DATA @@ -553,7 +549,6 @@ log segment */ #define TRX_UNDO_SEG_HDR_SIZE (4 + FSEG_HEADER_SIZE + FLST_BASE_NODE_SIZE) /* @} */ - /** The undo log header. There can be several undo log headers on the first page of an update undo log segment. */ /* @{ */ diff --git a/storage/innobase/include/trx0undo.ic b/storage/innobase/include/trx0undo.ic index f8e74d0fb03..62d9e64b13f 100644 --- a/storage/innobase/include/trx0undo.ic +++ b/storage/innobase/include/trx0undo.ic @@ -26,7 +26,6 @@ Created 3/26/1996 Heikki Tuuri #include "data0type.h" #include "page0page.h" -#ifndef UNIV_HOTBACKUP /***********************************************************************//** Builds a roll pointer. @return roll pointer */ @@ -116,7 +115,6 @@ trx_undo_trx_id_is_insert( #endif return(static_cast<bool>(trx_id[DATA_TRX_ID_LEN] >> 7)); } -#endif /* !UNIV_HOTBACKUP */ /*****************************************************************//** Writes a roll ptr to an index page. In case that the size changes in @@ -153,8 +151,6 @@ trx_read_roll_ptr( return(mach_read_from_7(ptr)); } -#ifndef UNIV_HOTBACKUP - /** Gets an undo log page and x-latches it. @param[in] page_id page id @param[in] page_size page size @@ -361,4 +357,3 @@ trx_undo_page_get_first_rec( return(undo_page + start); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index 6907bfec583..908fb60e956 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -1,8 +1,8 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. Copyright (c) 2008, Google Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -34,10 +34,6 @@ Created 1/20/1994 Heikki Tuuri #ifndef univ_i #define univ_i -#ifdef UNIV_HOTBACKUP -#include "hb_univ.i" -#endif /* UNIV_HOTBACKUP */ - /* aux macros to convert M into "123" (string) if M is defined like #define M 123 */ #define _IB_TO_STR(s) #s @@ -79,27 +75,22 @@ the virtual method table (vtable) in GCC 3. */ # include <windows.h> #endif /* _WIN32 */ -/* The defines used with MySQL */ - -#ifndef UNIV_HOTBACKUP - /* Include a minimum number of SQL header files so that few changes made in SQL code cause a complete InnoDB rebuild. These headers are used throughout InnoDB but do not include too much themselves. They support cross-platform development and expose comonly used SQL names. */ -# include <my_global.h> +#include <my_global.h> /* JAN: TODO: missing 5.7 header */ #ifdef HAVE_MY_THREAD_H //# include <my_thread.h> #endif -# ifndef UNIV_INNOCHECKSUM -# include <m_string.h> -# include <mysqld_error.h> -# endif /* !UNIV_INNOCHECKSUM */ -#endif /* !UNIV_HOTBACKUP */ +#ifndef UNIV_INNOCHECKSUM +# include <m_string.h> +# include <mysqld_error.h> +#endif /* !UNIV_INNOCHECKSUM */ /* Include <sys/stat.h> to get S_I... macros defined for os0file.cc */ #include <sys/stat.h> @@ -107,13 +98,7 @@ support cross-platform development and expose comonly used SQL names. */ #ifndef _WIN32 # include <sys/mman.h> /* mmap() for os0proc.cc */ # include <sched.h> -#endif /* !_WIN32 */ - -/* Include the header file generated by CMake */ -#ifndef _WIN32 -# ifndef UNIV_HOTBACKUP -# include "my_config.h" -# endif /* UNIV_HOTBACKUP */ +# include "my_config.h" #endif #include <stdint.h> @@ -122,10 +107,11 @@ support cross-platform development and expose comonly used SQL names. */ #include <unistd.h> #endif +#include "my_pthread.h" /* Following defines are to enable performance schema instrumentation in each of five InnoDB modules if HAVE_PSI_INTERFACE is defined. */ -#if defined(HAVE_PSI_INTERFACE) && !defined(UNIV_HOTBACKUP) +#ifdef HAVE_PSI_INTERFACE # define UNIV_PFS_MUTEX # define UNIV_PFS_RWLOCK /* For I/O instrumentation, performance schema rely @@ -159,7 +145,6 @@ be excluded from instrumentation. */ #include "pfs_thread_provider.h" #endif -#include "my_pthread.h" #include "mysql/psi/mysql_thread.h" /* For PSI_FILE_CALL(). */ /* JAN: TODO: missing 5.7 header */ @@ -199,7 +184,7 @@ command. */ #define UNIV_ENABLE_UNIT_TEST_ROW_RAW_FORMAT_INT */ -#if defined HAVE_VALGRIND +#if defined HAVE_valgrind && defined HAVE_VALGRIND # define UNIV_DEBUG_VALGRIND #endif /* HAVE_VALGRIND */ #if 0 @@ -381,6 +366,12 @@ typedef enum innodb_file_formats_enum innodb_file_formats_t; #define IF_SNAPPY(A,B) B #endif +#if defined (HAVE_FALLOC_PUNCH_HOLE_AND_KEEP_SIZE) || defined(_WIN32) +#define IF_PUNCH_HOLE(A,B) A +#else +#define IF_PUNCH_HOLE(A,B) B +#endif + /** The universal page size of the database */ #define UNIV_PAGE_SIZE ((ulint) srv_page_size) diff --git a/storage/innobase/include/ut0counter.h b/storage/innobase/include/ut0counter.h index 175427df333..70381f26a84 100644 --- a/storage/innobase/include/ut0counter.h +++ b/storage/innobase/include/ut0counter.h @@ -32,19 +32,11 @@ Created 2012/04/12 by Sunny Bains #include "os0thread.h" /** CPU cache line size */ -#ifndef UNIV_HOTBACKUP -# ifdef CPU_LEVEL1_DCACHE_LINESIZE -# define CACHE_LINE_SIZE CPU_LEVEL1_DCACHE_LINESIZE -# else -# error CPU_LEVEL1_DCACHE_LINESIZE is undefined -# endif /* CPU_LEVEL1_DCACHE_LINESIZE */ +#ifdef CPU_LEVEL1_DCACHE_LINESIZE +# define CACHE_LINE_SIZE CPU_LEVEL1_DCACHE_LINESIZE #else -#ifdef powerpc -#define CACHE_LINE_SIZE 128 -#else -# define CACHE_LINE_SIZE 64 -#endif /* __powerpc__ */ -#endif /* UNIV_HOTBACKUP */ +# error CPU_LEVEL1_DCACHE_LINESIZE is undefined +#endif /* CPU_LEVEL1_DCACHE_LINESIZE */ /** Default number of slots to use in ib_counter_t */ #define IB_N_SLOTS 64 diff --git a/storage/innobase/include/ut0crc32.h b/storage/innobase/include/ut0crc32.h index 91af6a910ff..36b389b5bd2 100644 --- a/storage/innobase/include/ut0crc32.h +++ b/storage/innobase/include/ut0crc32.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -54,8 +55,6 @@ extern ut_crc32_func_t ut_crc32_legacy_big_endian; but very slow). */ extern ut_crc32_func_t ut_crc32_byte_by_byte; -/** Flag that tells whether the CPU supports CRC32 or not */ -extern bool ut_crc32_sse2_enabled; -extern bool ut_crc32_power8_enabled; +extern const char* ut_crc32_implementation; #endif /* ut0crc32_h */ diff --git a/storage/innobase/include/ut0mem.h b/storage/innobase/include/ut0mem.h index 6d56be4d820..f20273daca6 100644 --- a/storage/innobase/include/ut0mem.h +++ b/storage/innobase/include/ut0mem.h @@ -27,10 +27,8 @@ Created 5/30/1994 Heikki Tuuri #define ut0mem_h #include "univ.i" -#ifndef UNIV_HOTBACKUP -# include "os0event.h" -# include "ut0mutex.h" -#endif /* !UNIV_HOTBACKUP */ +#include "os0event.h" +#include "ut0mutex.h" /** Wrapper for memcpy(3). Copy memory area when the source and target are not overlapping. diff --git a/storage/innobase/include/ut0mutex.h b/storage/innobase/include/ut0mutex.h index e4ab671eece..354fd9e0a7a 100644 --- a/storage/innobase/include/ut0mutex.h +++ b/storage/innobase/include/ut0mutex.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -28,9 +29,8 @@ Created 2012-03-24 Sunny Bains. #ifndef ut0mutex_h #define ut0mutex_h -extern ulong srv_spin_wait_delay; +extern uint srv_spin_wait_delay; extern ulong srv_n_spin_wait_rounds; -extern ulong srv_force_recovery_crash; #include "sync0policy.h" #include "ib0mutex.h" @@ -75,7 +75,7 @@ typedef BlockSyncArrayMutex ib_bpmutex_t; #error "ib_mutex_t type is unknown" #endif /* MUTEX_FUTEX */ -extern ulong srv_spin_wait_delay; +extern uint srv_spin_wait_delay; extern ulong srv_n_spin_wait_rounds; #define mutex_create(I, M) mutex_init((M), (I), __FILE__, __LINE__) diff --git a/storage/innobase/include/ut0new.h b/storage/innobase/include/ut0new.h index 6f3c06cf978..5a9022e8a77 100644 --- a/storage/innobase/include/ut0new.h +++ b/storage/innobase/include/ut0new.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2014, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -389,25 +390,21 @@ public: } /** Free a memory allocated by allocate() and trace the deallocation. - @param[in,out] ptr pointer to memory to free - @param[in] n_elements number of elements allocated (unused) */ - void - deallocate( - pointer ptr, - size_type n_elements = 0) + @param[in,out] ptr pointer to memory to free */ + void deallocate(pointer ptr, size_type) { +#ifdef UNIV_PFS_MEMORY if (ptr == NULL) { return; } -#ifdef UNIV_PFS_MEMORY ut_new_pfx_t* pfx = reinterpret_cast<ut_new_pfx_t*>(ptr) - 1; deallocate_trace(pfx); free(pfx); #else - // free(ptr); + free(ptr); #endif /* UNIV_PFS_MEMORY */ } diff --git a/storage/innobase/include/ut0timer.ic b/storage/innobase/include/ut0timer.ic index 027e89c6279..46dcd0cb718 100644 --- a/storage/innobase/include/ut0timer.ic +++ b/storage/innobase/include/ut0timer.ic @@ -106,7 +106,7 @@ ut_microseconds_to_timer( /*=====================*/ ulonglong when) /*!< in: time where to calculate */ { - double ret = when; + double ret = (double)when; ret *= (double)(ut_timer.frequency); ret /= 1000000.0; return (ulonglong)ret; diff --git a/storage/innobase/include/ut0ut.h b/storage/innobase/include/ut0ut.h index c5355fd9d5c..fb8b66ddd39 100644 --- a/storage/innobase/include/ut0ut.h +++ b/storage/innobase/include/ut0ut.h @@ -30,6 +30,7 @@ Created 1/20/1994 Heikki Tuuri #include <ostream> #include <sstream> +#include <string.h> #ifndef UNIV_INNOCHECKSUM @@ -51,61 +52,60 @@ Created 1/20/1994 Heikki Tuuri /** Time stamp */ typedef time_t ib_time_t; -#ifndef UNIV_HOTBACKUP -# if defined(HAVE_PAUSE_INSTRUCTION) +#ifdef HAVE_PAUSE_INSTRUCTION /* According to the gcc info page, asm volatile means that the instruction has important side-effects and must not be removed. Also asm volatile may trigger a memory barrier (spilling all registers to memory). */ -# ifdef __SUNPRO_CC -# define UT_RELAX_CPU() asm ("pause" ) -# else -# define UT_RELAX_CPU() __asm__ __volatile__ ("pause") -# endif /* __SUNPRO_CC */ - -# elif defined(HAVE_FAKE_PAUSE_INSTRUCTION) -# define UT_RELAX_CPU() __asm__ __volatile__ ("rep; nop") -# elif defined _WIN32 +# ifdef __SUNPRO_CC +# define UT_RELAX_CPU() asm ("pause" ) +# else +# define UT_RELAX_CPU() __asm__ __volatile__ ("pause") +# endif /* __SUNPRO_CC */ + +#elif defined(HAVE_FAKE_PAUSE_INSTRUCTION) +# define UT_RELAX_CPU() __asm__ __volatile__ ("rep; nop") +#elif defined _WIN32 /* In the Win32 API, the x86 PAUSE instruction is executed by calling the YieldProcessor macro defined in WinNT.h. It is a CPU architecture- independent way by using YieldProcessor. */ -# define UT_RELAX_CPU() YieldProcessor() -# elif defined(__powerpc__) && defined __GLIBC__ -#include <sys/platform/ppc.h> -# define UT_RELAX_CPU() do { \ +# define UT_RELAX_CPU() YieldProcessor() +#elif defined(__powerpc__) && defined __GLIBC__ +# include <sys/platform/ppc.h> +# define UT_RELAX_CPU() do { \ volatile lint volatile_var = __ppc_get_timebase(); \ } while (0) -# else -# define UT_RELAX_CPU() do { \ +#else +# define UT_RELAX_CPU() do { \ volatile int32 volatile_var; \ int32 oldval= 0; \ my_atomic_cas32(&volatile_var, &oldval, 1); \ } while (0) -# endif +#endif #if defined (__GNUC__) -# define UT_COMPILER_BARRIER() __asm__ __volatile__ ("":::"memory") +# define UT_COMPILER_BARRIER() __asm__ __volatile__ ("":::"memory") #elif defined (_MSC_VER) -# define UT_COMPILER_BARRIER() _ReadWriteBarrier() +# define UT_COMPILER_BARRIER() _ReadWriteBarrier() #else -# define UT_COMPILER_BARRIER() +# define UT_COMPILER_BARRIER() #endif -# if defined(HAVE_HMT_PRIORITY_INSTRUCTION) -#include <sys/platform/ppc.h> -# define UT_LOW_PRIORITY_CPU() __ppc_set_ppr_low() -# define UT_RESUME_PRIORITY_CPU() __ppc_set_ppr_med() -# else -# define UT_LOW_PRIORITY_CPU() ((void)0) -# define UT_RESUME_PRIORITY_CPU() ((void)0) -# endif +#if defined(HAVE_HMT_PRIORITY_INSTRUCTION) +# include <sys/platform/ppc.h> +# define UT_LOW_PRIORITY_CPU() __ppc_set_ppr_low() +# define UT_RESUME_PRIORITY_CPU() __ppc_set_ppr_med() +#else +# define UT_LOW_PRIORITY_CPU() ((void)0) +# define UT_RESUME_PRIORITY_CPU() ((void)0) +#endif /*********************************************************************//** Delays execution for at most max_wait_us microseconds or returns earlier if cond becomes true. @param cond in: condition to wait for; evaluated every 2 ms @param max_wait_us in: maximum delay to wait, in microseconds */ -#define UT_WAIT_FOR(cond, max_wait_us) \ +# define UT_WAIT_FOR(cond, max_wait_us) \ do { \ uintmax_t start_us; \ start_us = ut_time_us(NULL); \ @@ -115,7 +115,6 @@ do { \ os_thread_sleep(2000 /* 2 ms */); \ } \ } while (0) -#endif /* !UNIV_HOTBACKUP */ #define ut_max std::max #define ut_min std::min @@ -228,7 +227,7 @@ the only way to manipulate it is to use the function ut_difftime. ib_time_t ut_time(void); /*=========*/ -#ifndef UNIV_HOTBACKUP + /**********************************************************//** Returns system time. Upon successful completion, the value 0 is returned; otherwise the @@ -258,7 +257,6 @@ purposes. ulint ut_time_ms(void); /*============*/ -#endif /* !UNIV_HOTBACKUP */ /**********************************************************//** Returns the number of milliseconds since some epoch. The @@ -313,23 +311,6 @@ void ut_sprintf_timestamp( /*=================*/ char* buf); /*!< in: buffer where to sprintf */ -#ifdef UNIV_HOTBACKUP -/**********************************************************//** -Sprintfs a timestamp to a buffer with no spaces and with ':' characters -replaced by '_'. */ -void -ut_sprintf_timestamp_without_extra_chars( -/*=====================================*/ - char* buf); /*!< in: buffer where to sprintf */ -/**********************************************************//** -Returns current year, month, day. */ -void -ut_get_year_month_day( -/*==================*/ - ulint* year, /*!< out: current year */ - ulint* month, /*!< out: month */ - ulint* day); /*!< out: day */ -#else /* UNIV_HOTBACKUP */ /*************************************************************//** Runs an idle loop on CPU. The argument gives the desired delay in microseconds on 100 MHz Pentium + Visual C++. @@ -338,7 +319,6 @@ ulint ut_delay( /*=====*/ ulint delay); /*!< in: delay in microseconds on 100 MHz Pentium */ -#endif /* UNIV_HOTBACKUP */ /*************************************************************//** Prints the contents of a memory buffer in hex and ascii. */ void @@ -367,7 +347,6 @@ ut_print_buf( ulint len) /*!< in: length of the buffer */ MY_ATTRIBUTE((nonnull)); -#ifndef UNIV_HOTBACKUP /* Forward declaration of transaction handle */ struct trx_t; @@ -417,7 +396,6 @@ ut_copy_file( /*=========*/ FILE* dest, /*!< in: output file */ FILE* src); /*!< in: input file to be appended to output */ -#endif /* !UNIV_HOTBACKUP */ #ifdef _WIN32 /**********************************************************************//** diff --git a/storage/innobase/innodb.cmake b/storage/innobase/innodb.cmake index 15b0f7c9c9a..9cc6e434f38 100644 --- a/storage/innobase/innodb.cmake +++ b/storage/innobase/innodb.cmake @@ -1,4 +1,5 @@ # Copyright (c) 2006, 2016, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2017, MariaDB Corporation. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -135,24 +136,6 @@ IF(HAVE_NANOSLEEP) ADD_DEFINITIONS(-DHAVE_NANOSLEEP=1) ENDIF() -IF(NOT MSVC) - CHECK_C_SOURCE_RUNS( - " - #define _GNU_SOURCE - #include <fcntl.h> - #include <linux/falloc.h> - int main() - { - /* Ignore the return value for now. Check if the flags exist. - The return value is checked at runtime. */ - fallocate(0, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 0, 0); - - return(0); - }" - HAVE_FALLOC_PUNCH_HOLE_AND_KEEP_SIZE - ) -ENDIF() - IF(HAVE_FALLOC_PUNCH_HOLE_AND_KEEP_SIZE) ADD_DEFINITIONS(-DHAVE_FALLOC_PUNCH_HOLE_AND_KEEP_SIZE=1) ENDIF() diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index ce71027be1f..3916bf9961f 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -1754,9 +1754,9 @@ RecLock::lock_alloc( /*********************************************************************//** Check if lock1 has higher priority than lock2. NULL has lowest priority. -If either is a high priority transaction, the lock has higher priority. If neither of them is wait lock, the first one has higher priority. If only one of them is a wait lock, it has lower priority. +If either is a high priority transaction, the lock has higher priority. Otherwise, the one with an older transaction has higher priority. @returns true if lock1 has higher priority, false otherwise. */ bool @@ -1769,27 +1769,15 @@ has_higher_priority( } else if (lock2 == NULL) { return true; } - // Ask the upper server layer if any of the two trx should be prefered. - int preference = thd_deadlock_victim_preference(lock1->trx->mysql_thd, lock2->trx->mysql_thd); - if (preference == -1) { - // lock1 is preferred as a victim, so lock2 has higher priority - return false; - } else if (preference == 1) { - // lock2 is preferred as a victim, so lock1 has higher priority - return true; - } - if (trx_is_high_priority(lock1->trx)) { - return true; - } - if (trx_is_high_priority(lock2->trx)) { - return false; - } - // No preference. Compre them by wait mode and trx age. + // Granted locks has higher priority. if (!lock_get_wait(lock1)) { return true; } else if (!lock_get_wait(lock2)) { return false; } + if (trx_is_high_priority(lock1->trx)) { + return false; + } return lock1->trx->start_time_micro <= lock2->trx->start_time_micro; } @@ -1820,7 +1808,7 @@ lock_rec_insert_by_trx_age( node = (lock_t *) cell->node; // If in_lock is not a wait lock, we insert it to the head of the list. - if (node == NULL || (!lock_get_wait(in_lock) && has_higher_priority(in_lock, node))) { + if (node == NULL || !lock_get_wait(in_lock) || has_higher_priority(in_lock, node)) { cell->node = in_lock; in_lock->hash = node; if (lock_get_wait(in_lock)) { @@ -2792,12 +2780,10 @@ RecLock::jump_queue( ut_ad(conflict_lock->trx->lock.que_state == TRX_QUE_LOCK_WAIT); ut_ad(conflict_lock->trx->lock.wait_lock == conflict_lock); -#ifdef UNIV_DEBUG - ib::info() << "Granting High Priority Transaction (ID): " - << lock->trx->id << " the lock jumping over" - << " waiting Transaction (ID): " - << conflict_lock->trx->id; -#endif /* UNIV_DEBUG */ + DBUG_LOG("trx", + "Granting High Priority Transaction " + << lock->trx->id << " a lock jumping over" + << " waiting Transaction " << conflict_lock->trx->id); lock_reset_lock_and_trx_wait(lock); return(true); @@ -2966,11 +2952,12 @@ RecLock::make_trx_hit_list( /* Assert that it is not waiting for current record. */ ut_ad(trx->lock.wait_lock != next); -#ifdef UNIV_DEBUG - ib::info() << "High Priority Transaction (ID): " - << lock->trx->id << " waking up blocking" - << " transaction (ID): " << trx->id; -#endif /* UNIV_DEBUG */ + + DBUG_LOG("trx", "High Priority Transaction " + << lock->trx->id + << " waking up blocking transaction " + << trx->id); + trx->lock.was_chosen_as_deadlock_victim = true; lock_cancel_waiting_and_release(trx->lock.wait_lock); trx_mutex_exit(trx); @@ -4802,7 +4789,7 @@ lock_table( lock_mutex_enter(); DBUG_EXECUTE_IF("fatal-semaphore-timeout", - { os_thread_sleep(3600000000); }); + { os_thread_sleep(3600000000LL); }); /* We have to check if the new lock is compatible with any locks other transactions have in the table lock queue. */ @@ -7783,6 +7770,9 @@ lock_trx_handle_wait( trx_mutex_exit(trx); } + ut_ad(err == DB_SUCCESS || err == DB_LOCK_WAIT + || err == DB_DEADLOCK); + return(err); } diff --git a/storage/innobase/lock/lock0wait.cc b/storage/innobase/lock/lock0wait.cc index 1330658626e..867e596bca2 100644 --- a/storage/innobase/lock/lock0wait.cc +++ b/storage/innobase/lock/lock0wait.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2016, MariaDB Corporation. +Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -509,11 +509,7 @@ A thread which wakes up threads whose lock wait may have lasted too long. @return a dummy parameter */ extern "C" os_thread_ret_t -DECLARE_THREAD(lock_wait_timeout_thread)( -/*=====================================*/ - void* arg MY_ATTRIBUTE((unused))) - /* in: a dummy parameter required by - os_thread_create */ +DECLARE_THREAD(lock_wait_timeout_thread)(void*) { int64_t sig_count = 0; os_event_t event = lock_sys->timeout_event; @@ -524,8 +520,6 @@ DECLARE_THREAD(lock_wait_timeout_thread)( pfs_register_thread(srv_lock_timeout_thread_key); #endif /* UNIV_PFS_THREAD */ - lock_sys->timeout_thread_active = true; - do { srv_slot_t* slot; diff --git a/storage/innobase/log/log0crypt.cc b/storage/innobase/log/log0crypt.cc index 3a4447d2b85..dbfda8ab7c4 100644 --- a/storage/innobase/log/log0crypt.cc +++ b/storage/innobase/log/log0crypt.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -22,55 +22,46 @@ Innodb log encrypt/decrypt Created 11/25/2013 Minli Zhu Google Modified Jan Lindström jan.lindstrom@mariadb.com +MDEV-11782: Rewritten for MariaDB 10.2 by Marko Mäkelä, MariaDB Corporation. *******************************************************/ #include "m_string.h" #include "log0crypt.h" -#include <my_crypt.h> -#include <my_crypt.h> +#include "my_crypt.h" -#include "log0log.h" +#include "log0crypt.h" #include "srv0start.h" // for srv_start_lsn #include "log0recv.h" // for recv_sys -#include "ha_prototypes.h" // IB_LOG_ - -#include "my_crypt.h" - -/* Used for debugging */ -// #define DEBUG_CRYPT 1 -#define UNENCRYPTED_KEY_VER 0 - -/* If true, enable redo log encryption. */ -extern my_bool srv_encrypt_log; - - -#include <algorithm> // std::sort -#include <deque> - -/* If true, enable redo log encryption. */ -UNIV_INTERN my_bool srv_encrypt_log = FALSE; -/* - Sub system type for InnoDB redo log crypto. - Set and used to validate crypto msg. -*/ -static const byte redo_log_purpose_byte = 0x02; +/** innodb_encrypt_log: whether to encrypt the redo log */ +my_bool srv_encrypt_log; +/** Redo log encryption key ID */ #define LOG_DEFAULT_ENCRYPTION_KEY 1 -/* - Store this many keys into each checkpoint info -*/ -static const size_t kMaxSavedKeys = LOG_CRYPT_MAX_ENTRIES; +typedef union { + uint32_t words[MY_AES_BLOCK_SIZE / sizeof(uint32_t)]; + byte bytes[MY_AES_BLOCK_SIZE]; +} aes_block_t; struct crypt_info_t { - ib_uint64_t checkpoint_no; /*!< checkpoint no */ + ulint checkpoint_no; /*!< checkpoint no; 32 bits */ uint key_version; /*!< mysqld key version */ - byte crypt_msg[MY_AES_BLOCK_SIZE]; - byte crypt_key[MY_AES_BLOCK_SIZE]; - byte crypt_nonce[MY_AES_BLOCK_SIZE]; + /** random string for encrypting the key */ + aes_block_t crypt_msg; + /** the secret key */ + aes_block_t crypt_key; + /** a random string for the per-block initialization vector */ + union { + uint32_t word; + byte bytes[4]; + } crypt_nonce; }; -static std::deque<crypt_info_t> crypt_info; +/** The crypt info */ +static crypt_info_t info; + +/** Crypt info when upgrading from 10.1 */ +static crypt_info_t infos[5]; /*********************************************************************//** Get a log block's start lsn. @@ -93,547 +84,292 @@ Get crypt info from checkpoint. @return a crypt info or NULL if not present. */ static const crypt_info_t* -get_crypt_info( -/*===========*/ - ib_uint64_t checkpoint_no) +get_crypt_info(ulint checkpoint_no) { - size_t items = crypt_info.size(); - /* a log block only stores 4-bytes of checkpoint no */ checkpoint_no &= 0xFFFFFFFF; - for (size_t i = 0; i < items; i++) { - struct crypt_info_t* it = &crypt_info[i]; + for (unsigned i = 0; i < 5; i++) { + const crypt_info_t* it = &infos[i]; - if (it->checkpoint_no == checkpoint_no) { + if (it->key_version && it->checkpoint_no == checkpoint_no) { return it; } } /* If checkpoint contains more than one key and we did not find the correct one use the first one. */ - if (items) { - return (&crypt_info[0]); - } - - return NULL; -} - -/*********************************************************************//** -Get crypt info from log block -@return a crypt info or NULL if not present. */ -static -const crypt_info_t* -get_crypt_info( -/*===========*/ - const byte* log_block) -{ - ib_uint64_t checkpoint_no = log_block_get_checkpoint_no(log_block); - return get_crypt_info(checkpoint_no); + return infos; } -/*********************************************************************//** -Print checkpoint no from log block and all encryption keys from -checkpoints if they are present. Used for problem analysis. */ +/** Encrypt or decrypt log blocks. +@param[in,out] buf log blocks to encrypt or decrypt +@param[in] size size of the buffer, in bytes +@param[in] decrypt whether to decrypt instead of encrypting */ +UNIV_INTERN void -log_crypt_print_checkpoint_keys( -/*============================*/ - const byte* log_block) +log_crypt(byte* buf, ulint size, bool decrypt) { - ib_uint64_t checkpoint_no = log_block_get_checkpoint_no(log_block); - - if (crypt_info.size()) { - fprintf(stderr, "InnoDB: redo log checkpoint: %lu [ chk key ]: ", (ulong) checkpoint_no); - for (size_t i = 0; i < crypt_info.size(); i++) { - struct crypt_info_t* it = &crypt_info[i]; - fprintf(stderr, "[ %lu %u ] ", - (ulong) it->checkpoint_no, - it->key_version); - } - fprintf(stderr, "\n"); - } -} + ut_ad(size % OS_FILE_LOG_BLOCK_SIZE == 0); + ut_a(info.key_version); -/*********************************************************************//** -Call AES CTR to encrypt/decrypt log blocks. */ -static -Crypt_result -log_blocks_crypt( -/*=============*/ - const byte* block, /*!< in: blocks before encrypt/decrypt*/ - ulint size, /*!< in: size of block */ - byte* dst_block, /*!< out: blocks after encrypt/decrypt */ - int what, /*!< in: encrypt or decrypt*/ - const crypt_info_t* crypt_info) /*!< in: crypt info or NULL */ -{ - byte *log_block = (byte*)block; - Crypt_result rc = MY_AES_OK; uint dst_len; - byte aes_ctr_counter[MY_AES_BLOCK_SIZE]; - byte is_encrypt= what == ENCRYPTION_FLAG_ENCRYPT; - lsn_t lsn = is_encrypt ? log_sys->lsn : srv_start_lsn; + uint32_t aes_ctr_iv[MY_AES_BLOCK_SIZE / sizeof(uint32_t)]; + compile_time_assert(sizeof(uint32_t) == 4); - const uint src_len = OS_FILE_LOG_BLOCK_SIZE - LOG_BLOCK_HDR_SIZE; - for (ulint i = 0; i < size ; i += OS_FILE_LOG_BLOCK_SIZE) { - ulint log_block_no = log_block_get_hdr_no(log_block); - lsn_t log_block_start_lsn = log_block_get_start_lsn( - lsn, log_block_no); - - const crypt_info_t* info = crypt_info == NULL ? get_crypt_info(log_block) : - crypt_info; -#ifdef DEBUG_CRYPT - fprintf(stderr, - "%s %lu chkpt: %lu key: %u lsn: %lu\n", - is_encrypt ? "crypt" : "decrypt", - log_block_no, - log_block_get_checkpoint_no(log_block), - info ? info->key_version : 0, - log_block_start_lsn); -#endif - /* If no key is found from checkpoint assume the log_block - to be unencrypted. If checkpoint contains the encryption key - compare log_block current checksum, if checksum matches, - block can't be encrypted. */ - if (info == NULL || - info->key_version == UNENCRYPTED_KEY_VER || - (log_block_checksum_is_ok(log_block, false) && - what == ENCRYPTION_FLAG_DECRYPT)) { - memcpy(dst_block, log_block, OS_FILE_LOG_BLOCK_SIZE); - goto next; - } +#define LOG_CRYPT_HDR_SIZE 4 - ut_ad(what == ENCRYPTION_FLAG_DECRYPT ? !log_block_checksum_is_ok(log_block, false) : - log_block_checksum_is_ok(log_block, false)); - - // Assume log block header is not encrypted - memcpy(dst_block, log_block, LOG_BLOCK_HDR_SIZE); - - // aes_ctr_counter = nonce(3-byte) + start lsn to a log block - // (8-byte) + lbn (4-byte) + abn - // (1-byte, only 5 bits are used). "+" means concatenate. - bzero(aes_ctr_counter, MY_AES_BLOCK_SIZE); - memcpy(aes_ctr_counter, info->crypt_nonce, 3); - mach_write_to_8(aes_ctr_counter + 3, log_block_start_lsn); - mach_write_to_4(aes_ctr_counter + 11, log_block_no); - bzero(aes_ctr_counter + 15, 1); - - int rc; - rc = encryption_crypt(log_block + LOG_BLOCK_HDR_SIZE, src_len, - dst_block + LOG_BLOCK_HDR_SIZE, &dst_len, - (unsigned char*)(info->crypt_key), 16, - aes_ctr_counter, MY_AES_BLOCK_SIZE, - what | ENCRYPTION_FLAG_NOPAD, - LOG_DEFAULT_ENCRYPTION_KEY, - info->key_version); + for (const byte* const end = buf + size; buf != end; + buf += OS_FILE_LOG_BLOCK_SIZE) { + byte dst[OS_FILE_LOG_BLOCK_SIZE - LOG_CRYPT_HDR_SIZE]; + const ulint log_block_no = log_block_get_hdr_no(buf); + + /* The log block number is not encrypted. */ + *aes_ctr_iv = +#ifdef WORDS_BIGENDIAN + ~LOG_BLOCK_FLUSH_BIT_MASK +#else + ~(LOG_BLOCK_FLUSH_BIT_MASK >> 24) +#endif + & (*reinterpret_cast<uint32_t*>(dst) + = *reinterpret_cast<const uint32_t*>( + buf + LOG_BLOCK_HDR_NO)); +#if LOG_BLOCK_HDR_NO + 4 != LOG_CRYPT_HDR_SIZE +# error "LOG_BLOCK_HDR_NO has been moved; redo log format affected!" +#endif + aes_ctr_iv[1] = info.crypt_nonce.word; + mach_write_to_8(reinterpret_cast<byte*>(aes_ctr_iv + 2), + log_block_get_start_lsn( + decrypt ? srv_start_lsn : log_sys->lsn, + log_block_no)); + + int rc = encryption_crypt( + buf + LOG_CRYPT_HDR_SIZE, sizeof dst, dst, &dst_len, + const_cast<byte*>(info.crypt_key.bytes), + sizeof info.crypt_key, + reinterpret_cast<byte*>(aes_ctr_iv), sizeof aes_ctr_iv, + decrypt + ? ENCRYPTION_FLAG_DECRYPT | ENCRYPTION_FLAG_NOPAD + : ENCRYPTION_FLAG_ENCRYPT | ENCRYPTION_FLAG_NOPAD, + LOG_DEFAULT_ENCRYPTION_KEY, + info.key_version); ut_a(rc == MY_AES_OK); - ut_a(dst_len == src_len); -next: - log_block += OS_FILE_LOG_BLOCK_SIZE; - dst_block += OS_FILE_LOG_BLOCK_SIZE; - } + ut_a(dst_len == sizeof dst); + if (decrypt) { + std::ostringstream s; + ut_print_buf_hex(s, buf + LOG_CRYPT_HDR_SIZE, + OS_FILE_LOG_BLOCK_SIZE + - LOG_CRYPT_HDR_SIZE); + ib::info() << "S: " << s.str(); + std::ostringstream d; + ut_print_buf_hex(d, dst, + OS_FILE_LOG_BLOCK_SIZE + - LOG_CRYPT_HDR_SIZE); + ib::info() << "c: " << d.str(); + } - return rc; + memcpy(buf + LOG_CRYPT_HDR_SIZE, dst, sizeof dst); + } } -/*********************************************************************//** -Generate crypt key from crypt msg. -@return true if successfull, false if not. */ +/** Generate crypt key from crypt msg. +@param[in,out] info encryption key +@param[in] upgrade whether to use the key in MariaDB 10.1 format +@return whether the operation was successful */ static bool -init_crypt_key( -/*===========*/ - crypt_info_t* info) /*< in/out: crypt info */ +init_crypt_key(crypt_info_t* info, bool upgrade = false) { - if (info->key_version == UNENCRYPTED_KEY_VER) { - memset(info->crypt_key, 0, sizeof(info->crypt_key)); - memset(info->crypt_msg, 0, sizeof(info->crypt_msg)); - memset(info->crypt_nonce, 0, sizeof(info->crypt_nonce)); - return true; - } + byte mysqld_key[MY_AES_MAX_KEY_LENGTH]; + uint keylen = sizeof mysqld_key; - byte mysqld_key[MY_AES_MAX_KEY_LENGTH] = {0}; - uint keylen= sizeof(mysqld_key); - uint rc; + compile_time_assert(16 == sizeof info->crypt_key); - rc = encryption_key_get(LOG_DEFAULT_ENCRYPTION_KEY, info->key_version, mysqld_key, &keylen); - - if (rc) { + if (uint rc = encryption_key_get(LOG_DEFAULT_ENCRYPTION_KEY, + info->key_version, mysqld_key, + &keylen)) { ib::error() - << "Redo log crypto: getting mysqld crypto key " - << "from key version failed err = " << rc - << " Reason could be that requested key_version " - << info->key_version - << "is not found or required encryption " - << " key management is not found."; + << "Obtaining redo log encryption key version " + << info->key_version << " failed (" << rc + << "). Maybe the key or the required encryption " + << " key management plugin was not found."; return false; } + if (upgrade) { + while (keylen < sizeof mysqld_key) { + mysqld_key[keylen++] = 0; + } + } + uint dst_len; - int err= my_aes_crypt(MY_AES_ECB, ENCRYPTION_FLAG_NOPAD|ENCRYPTION_FLAG_ENCRYPT, - info->crypt_msg, sizeof(info->crypt_msg), //src, srclen - info->crypt_key, &dst_len, //dst, &dstlen - (unsigned char*)&mysqld_key, sizeof(mysqld_key), - NULL, 0); + int err= my_aes_crypt(MY_AES_ECB, + ENCRYPTION_FLAG_NOPAD | ENCRYPTION_FLAG_ENCRYPT, + info->crypt_msg.bytes, sizeof info->crypt_msg, + info->crypt_key.bytes, &dst_len, + mysqld_key, keylen, NULL, 0); if (err != MY_AES_OK || dst_len != MY_AES_BLOCK_SIZE) { - fprintf(stderr, - "\nInnodb redo log crypto: getting redo log crypto key " - "failed err = %d len = %u.\n", err, dst_len); + ib::error() << "Getting redo log crypto key failed: err = " + << err << ", len = " << dst_len; return false; } return true; } -/*********************************************************************//** -Compare function for checkpoint numbers -@return true if first checkpoint is larger than second one */ -static +/** Initialize the redo log encryption key. +@return whether the operation succeeded */ +UNIV_INTERN bool -mysort(const crypt_info_t& i, - const crypt_info_t& j) +log_crypt_init() { - return i.checkpoint_no > j.checkpoint_no; -} + ut_ad(log_mutex_own()); + ut_ad(log_sys->is_encrypted()); -/*********************************************************************//** -Add crypt info to set if it is not already present -@return true if successfull, false if not- */ -static -bool -add_crypt_info( -/*===========*/ - crypt_info_t* info, /*!< in: crypt info */ - bool checkpoint_read)/*!< in: do we read checkpoint */ -{ - const crypt_info_t* found=NULL; - /* so that no one is searching array while we modify it */ - ut_ad(mutex_own(&(log_sys->mutex))); - - found = get_crypt_info(info->checkpoint_no); - - /* If one crypt info is found then we add a new one only if we - are reading checkpoint from the log. New checkpoints will always - use the first created crypt info. */ - if (found != NULL && - ( found->checkpoint_no == info->checkpoint_no || !checkpoint_read)) { - // already present... - return true; - } + info.key_version = encryption_key_get_latest_version( + LOG_DEFAULT_ENCRYPTION_KEY); - if (!init_crypt_key(info)) { + if (info.key_version == ENCRYPTION_KEY_VERSION_INVALID) { + ib::error() << "innodb_encrypt_log: cannot get key version"; + info.key_version = 0; return false; } - crypt_info.push_back(*info); - - /* a log block only stores 4-bytes of checkpoint no */ - crypt_info.back().checkpoint_no &= 0xFFFFFFFF; - - // keep keys sorted, assuming that last added key will be used most - std::sort(crypt_info.begin(), crypt_info.end(), mysort); + if (my_random_bytes(info.crypt_msg.bytes, sizeof info.crypt_msg) + != MY_AES_OK + || my_random_bytes(info.crypt_nonce.bytes, sizeof info.crypt_nonce) + != MY_AES_OK) { + ib::error() << "innodb_encrypt_log: my_random_bytes() failed"; + return false; + } - return true; + return init_crypt_key(&info); } -/*********************************************************************//** -Encrypt log blocks. */ +/** Read the MariaDB 10.1 checkpoint crypto (version, msg and iv) info. +@param[in] buf checkpoint buffer +@return whether the operation was successful */ UNIV_INTERN -Crypt_result -log_blocks_encrypt( -/*===============*/ - const byte* block, /*!< in: blocks before encryption */ - const ulint size, /*!< in: size of blocks, must be multiple of a log block */ - byte* dst_block) /*!< out: blocks after encryption */ -{ - return log_blocks_crypt(block, size, dst_block, ENCRYPTION_FLAG_ENCRYPT, NULL); -} - -/*********************************************************************//** -Set next checkpoint's key version to latest one, and generate current -key. Key version 0 means no encryption. */ -UNIV_INTERN -void -log_crypt_set_ver_and_key( -/*======================*/ - ib_uint64_t next_checkpoint_no) +bool +log_crypt_101_read_checkpoint(const byte* buf) { - crypt_info_t info; - info.checkpoint_no = next_checkpoint_no; + buf += 20 + 32 * 9; - if (!srv_encrypt_log) { - info.key_version = UNENCRYPTED_KEY_VER; - } else { - info.key_version = encryption_key_get_latest_version(LOG_DEFAULT_ENCRYPTION_KEY); - } + const size_t n = *buf++ == 2 ? std::min(unsigned(*buf++), 5U) : 0; - if (info.key_version == UNENCRYPTED_KEY_VER) { - memset(info.crypt_msg, 0, sizeof(info.crypt_msg)); - memset(info.crypt_nonce, 0, sizeof(info.crypt_nonce)); - } else { - if (my_random_bytes(info.crypt_msg, MY_AES_BLOCK_SIZE) != MY_AES_OK) { - ib::error() - << "Redo log crypto: generate " - << MY_AES_BLOCK_SIZE - << "-byte random number as crypto msg failed."; - ut_error; - } + for (size_t i = 0; i < n; i++) { + struct crypt_info_t& info = infos[i]; + info.checkpoint_no = mach_read_from_4(buf); + info.key_version = mach_read_from_4(buf + 4); + memcpy(info.crypt_msg.bytes, buf + 8, sizeof info.crypt_msg); + memcpy(info.crypt_nonce.bytes, buf + 24, + sizeof info.crypt_nonce); - if (my_random_bytes(info.crypt_nonce, MY_AES_BLOCK_SIZE) != MY_AES_OK) { - ib::error() - << "Redo log crypto: generate " - << MY_AES_BLOCK_SIZE - << "-byte random number as AES_CTR nonce failed."; - ut_error; + if (!init_crypt_key(&info, true)) { + return false; } - + buf += 4 + 4 + 2 * MY_AES_BLOCK_SIZE; } - add_crypt_info(&info, false); + return true; } -/******************************************************** -Encrypt one or more log block before it is flushed to disk */ +/** Decrypt a MariaDB 10.1 redo log block. +@param[in,out] buf log block +@return whether the decryption was successful */ UNIV_INTERN -void -log_encrypt_before_write( -/*=====================*/ - ib_uint64_t next_checkpoint_no, /*!< in: log group to be flushed */ - byte* block, /*!< in/out: pointer to a log block */ - const ulint size) /*!< in: size of log blocks */ +bool +log_crypt_101_read_block(byte* buf) { - ut_ad(size % OS_FILE_LOG_BLOCK_SIZE == 0); - - const crypt_info_t* info = get_crypt_info(next_checkpoint_no); - if (info == NULL) { - return; - } + ut_ad(log_block_calc_checksum_format_0(buf) + != log_block_get_checksum(buf)); + const crypt_info_t* info = get_crypt_info( + log_block_get_checkpoint_no(buf)); - /* If the key is not encrypted or user has requested not to - encrypt, do not change log block. */ - if (info->key_version == UNENCRYPTED_KEY_VER || !srv_encrypt_log) { - return; - } - - byte* dst_frame = (byte*)malloc(size); - - //encrypt log blocks content - Crypt_result result = log_blocks_crypt(block, size, dst_frame, ENCRYPTION_FLAG_ENCRYPT, NULL); - - if (result == MY_AES_OK) { - ut_ad(block[0] == dst_frame[0]); - memcpy(block, dst_frame, size); - } - free(dst_frame); - - if (unlikely(result != MY_AES_OK)) { - ut_error; + if (!info || info->key_version == 0) { + return false; } -} -/******************************************************** -Decrypt a specified log segment after they are read from a log file to a buffer. -*/ -void -log_decrypt_after_read( -/*===================*/ - byte* frame, /*!< in/out: log segment */ - const ulint size) /*!< in: log segment size */ -{ - ut_ad(size % OS_FILE_LOG_BLOCK_SIZE == 0); - byte* dst_frame = (byte*)malloc(size); + byte dst[OS_FILE_LOG_BLOCK_SIZE]; + uint dst_len; + byte aes_ctr_iv[MY_AES_BLOCK_SIZE]; - // decrypt log blocks content - Crypt_result result = log_blocks_crypt(frame, size, dst_frame, ENCRYPTION_FLAG_DECRYPT, NULL); + const uint src_len = OS_FILE_LOG_BLOCK_SIZE - LOG_BLOCK_HDR_SIZE; - if (result == MY_AES_OK) { - memcpy(frame, dst_frame, size); + ulint log_block_no = log_block_get_hdr_no(buf); + + /* The log block header is not encrypted. */ + memcpy(dst, buf, LOG_BLOCK_HDR_SIZE); + + memcpy(aes_ctr_iv, info->crypt_nonce.bytes, 3); + mach_write_to_8(aes_ctr_iv + 3, + log_block_get_start_lsn(srv_start_lsn, log_block_no)); + memcpy(aes_ctr_iv + 11, buf, 4); + aes_ctr_iv[11] &= ~(LOG_BLOCK_FLUSH_BIT_MASK >> 24); + aes_ctr_iv[15] = 0; + + int rc = encryption_crypt(buf + LOG_BLOCK_HDR_SIZE, src_len, + dst + LOG_BLOCK_HDR_SIZE, &dst_len, + const_cast<byte*>(info->crypt_key.bytes), + MY_AES_BLOCK_SIZE, + aes_ctr_iv, MY_AES_BLOCK_SIZE, + ENCRYPTION_FLAG_DECRYPT + | ENCRYPTION_FLAG_NOPAD, + LOG_DEFAULT_ENCRYPTION_KEY, + info->key_version); + + if (rc != MY_AES_OK || dst_len != src_len + || log_block_calc_checksum_format_0(dst) + != log_block_get_checksum(dst)) { + return false; } - free(dst_frame); - if (unlikely(result != MY_AES_OK)) { - ut_error; - } + memcpy(buf, dst, sizeof dst); + return true; } -/*********************************************************************//** -Writes the crypto (version, msg and iv) info, which has been used for -log blocks with lsn <= this checkpoint's lsn, to a log header's -checkpoint buf. */ +/** Add the encryption information to a redo log checkpoint buffer. +@param[in,out] buf checkpoint buffer */ UNIV_INTERN void -log_crypt_write_checkpoint_buf( -/*===========================*/ - byte* buf) /*!< in/out: checkpoint buffer */ +log_crypt_write_checkpoint_buf(byte* buf) { - byte *save = buf; - - // Only write kMaxSavedKeys (sort keys to remove oldest) - std::sort(crypt_info.begin(), crypt_info.end(), mysort); - while (crypt_info.size() > kMaxSavedKeys) { - crypt_info.pop_back(); - } - - bool encrypted = false; - for (size_t i = 0; i < crypt_info.size(); i++) { - const crypt_info_t & it = crypt_info[i]; - if (it.key_version != UNENCRYPTED_KEY_VER) { - encrypted = true; - break; - } - } - - if (encrypted == false) { - // if no encryption is inuse then zero out - // crypt data for upward/downward compability - memset(buf + LOG_CRYPT_VER, 0, LOG_CRYPT_SIZE); - return; - } - - ib_uint64_t checkpoint_no = mach_read_from_8(buf + LOG_CHECKPOINT_NO); - buf += LOG_CRYPT_VER; - - mach_write_to_1(buf + 0, redo_log_purpose_byte); - mach_write_to_1(buf + 1, crypt_info.size()); - buf += 2; - for (size_t i = 0; i < crypt_info.size(); i++) { - struct crypt_info_t* it = &crypt_info[i]; - mach_write_to_4(buf + 0, it->checkpoint_no); - mach_write_to_4(buf + 4, it->key_version); - memcpy(buf + 8, it->crypt_msg, MY_AES_BLOCK_SIZE); - memcpy(buf + 24, it->crypt_nonce, MY_AES_BLOCK_SIZE); - buf += LOG_CRYPT_ENTRY_SIZE; - } - -#ifdef DEBUG_CRYPT - fprintf(stderr, "write chk: %lu [ chk key ]: ", checkpoint_no); - for (size_t i = 0; i < crypt_info.size(); i++) { - struct crypt_info_t* it = &crypt_info[i]; - fprintf(stderr, "[ %lu %u ] ", - it->checkpoint_no, - it->key_version); - } - fprintf(stderr, "\n"); -#else - (void)checkpoint_no; // unused variable -#endif - ut_a((ulint)(buf - save) <= OS_FILE_LOG_BLOCK_SIZE); + ut_ad(info.key_version); + compile_time_assert(16 == sizeof info.crypt_msg); + compile_time_assert(LOG_CHECKPOINT_CRYPT_MESSAGE + - LOG_CHECKPOINT_CRYPT_NONCE + == sizeof info.crypt_nonce); + + memcpy(buf + LOG_CHECKPOINT_CRYPT_MESSAGE, info.crypt_msg.bytes, + sizeof info.crypt_msg); + memcpy(buf + LOG_CHECKPOINT_CRYPT_NONCE, info.crypt_nonce.bytes, + sizeof info.crypt_nonce); + mach_write_to_4(buf + LOG_CHECKPOINT_CRYPT_KEY, info.key_version); } -/*********************************************************************//** -Read the crypto (version, msg and iv) info, which has been used for -log blocks with lsn <= this checkpoint's lsn, from a log header's -checkpoint buf. */ +/** Read the checkpoint crypto (version, msg and iv) info. +@param[in] buf checkpoint buffer +@return whether the operation was successful */ UNIV_INTERN bool -log_crypt_read_checkpoint_buf( -/*===========================*/ - const byte* buf) { /*!< in: checkpoint buffer */ - - buf += LOG_CRYPT_VER; - - byte scheme = buf[0]; - if (scheme != redo_log_purpose_byte) { - return true; - } - buf++; - size_t n = buf[0]; - buf++; - - for (size_t i = 0; i < n; i++) { - struct crypt_info_t info; - info.checkpoint_no = mach_read_from_4(buf + 0); - info.key_version = mach_read_from_4(buf + 4); - memcpy(info.crypt_msg, buf + 8, MY_AES_BLOCK_SIZE); - memcpy(info.crypt_nonce, buf + 24, MY_AES_BLOCK_SIZE); - - if (!add_crypt_info(&info, true)) { - return false; - } - buf += LOG_CRYPT_ENTRY_SIZE; - } - -#ifdef DEBUG_CRYPT - fprintf(stderr, "read [ chk key ]: "); - for (size_t i = 0; i < crypt_info.size(); i++) { - struct crypt_info_t* it = &crypt_info[i]; - fprintf(stderr, "[ %lu %u ] ", - it->checkpoint_no, - it->key_version); - } - fprintf(stderr, "\n"); -#endif - return true; -} - -/******************************************************** -Check is the checkpoint information encrypted. This check -is based on fact has log group crypt info and based -on this crypt info was the key version different from -unencrypted key version. There is no realible way to -distinguish encrypted log block from corrupted log block, -but if log block corruption is found this function is -used to find out if log block is maybe encrypted but -encryption key, key management plugin or encryption -algorithm does not match. -@return TRUE, if log block may be encrypted */ -UNIV_INTERN -ibool -log_crypt_block_maybe_encrypted( -/*============================*/ - const byte* log_block, /*!< in: log block */ - log_crypt_err_t* err_info) /*!< out: error info */ +log_crypt_read_checkpoint_buf(const byte* buf) { - ibool maybe_encrypted = FALSE; - const crypt_info_t* crypt_info; - - *err_info = LOG_UNENCRYPTED; - crypt_info = get_crypt_info(log_block); + info.checkpoint_no = mach_read_from_4(buf + (LOG_CHECKPOINT_NO + 4)); + info.key_version = mach_read_from_4(buf + LOG_CHECKPOINT_CRYPT_KEY); - if (crypt_info && - crypt_info->key_version != UNENCRYPTED_KEY_VER) { - byte mysqld_key[MY_AES_BLOCK_SIZE] = {0}; - uint keylen= sizeof(mysqld_key); - - /* Log block contains crypt info and based on key - version block could be encrypted. */ - *err_info = LOG_DECRYPT_MAYBE_FAILED; - maybe_encrypted = TRUE; - - if (encryption_key_get(LOG_DEFAULT_ENCRYPTION_KEY, - crypt_info->key_version, mysqld_key, &keylen)) { - *err_info = LOG_CRYPT_KEY_NOT_FOUND; - } - } +#if MY_AES_BLOCK_SIZE != 16 +# error "MY_AES_BLOCK_SIZE != 16; redo log checkpoint format affected" +#endif + compile_time_assert(16 == sizeof info.crypt_msg); + compile_time_assert(LOG_CHECKPOINT_CRYPT_MESSAGE + - LOG_CHECKPOINT_CRYPT_NONCE + == sizeof info.crypt_nonce); - return (maybe_encrypted); -} + memcpy(info.crypt_msg.bytes, buf + LOG_CHECKPOINT_CRYPT_MESSAGE, + sizeof info.crypt_msg); + memcpy(info.crypt_nonce.bytes, buf + LOG_CHECKPOINT_CRYPT_NONCE, + sizeof info.crypt_nonce); -/******************************************************** -Print crypt error message to error log */ -UNIV_INTERN -void -log_crypt_print_error( -/*==================*/ - log_crypt_err_t err_info) /*!< out: error info */ -{ - switch(err_info) { - case LOG_CRYPT_KEY_NOT_FOUND: - ib::error() - << "Redo log crypto: getting mysqld crypto key " - << "from key version failed. Reason could be that " - << "requested key version is not found or required " - << "encryption key management plugin is not found."; - break; - case LOG_DECRYPT_MAYBE_FAILED: - ib::error() - << "Redo log crypto: failed to decrypt log block. " - << "Reason could be that requested key version is " - << "not found, required encryption key management " - << "plugin is not found or configured encryption " - << "algorithm and/or method does not match."; - break; - default: - ut_error; /* Real bug */ - } + return init_crypt_key(&info); } diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index bc421774320..99ae5799bb1 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Google Inc. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -40,11 +40,11 @@ Created 12/9/1995 Heikki Tuuri #include "log0log.ic" #endif +#include "log0crypt.h" #include "mem0mem.h" #include "buf0buf.h" -#ifndef UNIV_HOTBACKUP #include "buf0flu.h" -#include "srv0srv.h" +#include "lock0lock.h" #include "log0recv.h" #include "fil0fil.h" #include "dict0boot.h" @@ -56,10 +56,6 @@ Created 12/9/1995 Heikki Tuuri #include "trx0roll.h" #include "srv0mon.h" #include "sync0sync.h" -#endif /* !UNIV_HOTBACKUP */ - -/* Used for debugging */ -// #define DEBUG_CRYPT 1 /* General philosophy of InnoDB redo-logs: @@ -137,6 +133,15 @@ the previous */ #define LOG_UNLOCK_NONE_FLUSHED_LOCK 1 #define LOG_UNLOCK_FLUSH_LOCK 2 +/** Event to wake up log_scrub_thread */ +os_event_t log_scrub_event; +/** Whether log_scrub_thread is active */ +bool log_scrub_thread_active; + +extern "C" UNIV_INTERN +os_thread_ret_t +DECLARE_THREAD(log_scrub_thread)(void*); + /******************************************************//** Completes a checkpoint write i/o to a log file. */ static @@ -144,7 +149,6 @@ void log_io_complete_checkpoint(void); /*============================*/ -#ifndef UNIV_HOTBACKUP /****************************************************************//** Returns the oldest modified block lsn in the pool, or log_sys->lsn if none exists. @@ -167,7 +171,6 @@ log_buf_pool_get_oldest_modification(void) return(lsn); } -#endif /* !UNIV_HOTBACKUP */ /** Extends the log buffer. @param[in] len requested minimum size in bytes */ @@ -261,7 +264,6 @@ log_buffer_extend( << LOG_BUFFER_SIZE << "."; } -#ifndef UNIV_HOTBACKUP /** Calculate actual length in redo buffer and file including block header and trailer. @param[in] len length to write @@ -352,7 +354,7 @@ log_margin_checkpoint_age( return; } -#endif /* !UNIV_HOTBACKUP */ + /** Open the log for log_write_low. The log must be closed with log_close. @param[in] len length of the data to be written @return start lsn of the log record */ @@ -712,7 +714,7 @@ log_group_set_fields( group->lsn_offset = log_group_calc_lsn_offset(lsn, group); group->lsn = lsn; } -#ifndef UNIV_HOTBACKUP + /*****************************************************************//** Calculates the recommended highest values for lsn - last_checkpoint_lsn and lsn - buf_get_oldest_modification(). @@ -865,6 +867,12 @@ log_init(void) MONITOR_SET(MONITOR_LSN_CHECKPOINT_AGE, log_sys->lsn - log_sys->last_checkpoint_lsn); + + log_scrub_thread_active = !srv_read_only_mode && srv_scrub_log; + if (log_scrub_thread_active) { + log_scrub_event = os_event_create("log_scrub_event"); + os_thread_create(log_scrub_thread, NULL, NULL); + } } /******************************************************************//** @@ -888,7 +896,9 @@ log_group_init( group->id = id; group->n_files = n_files; - group->format = LOG_HEADER_FORMAT_CURRENT; + group->format = srv_encrypt_log + ? LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED + : LOG_HEADER_FORMAT_CURRENT; group->file_size = file_size; group->space_id = space_id; group->state = LOG_GROUP_OK; @@ -920,7 +930,7 @@ log_group_init( UT_LIST_ADD_LAST(log_sys->log_groups, group); return(log_calc_max_ages()); } -#endif /* !UNIV_HOTBACKUP */ + /******************************************************//** Completes an i/o to a log file. */ void @@ -977,11 +987,13 @@ log_group_file_header_flush( ut_ad(!recv_no_log_write); ut_ad(group->id == 0); ut_a(nth_file < group->n_files); + ut_ad((group->format & ~LOG_HEADER_FORMAT_ENCRYPTED) + == LOG_HEADER_FORMAT_CURRENT); buf = *(group->file_header_bufs + nth_file); memset(buf, 0, OS_FILE_LOG_BLOCK_SIZE); - mach_write_to_4(buf + LOG_HEADER_FORMAT, LOG_HEADER_FORMAT_CURRENT); + mach_write_to_4(buf + LOG_HEADER_FORMAT, group->format); mach_write_to_8(buf + LOG_HEADER_START_LSN, start_lsn); strcpy(reinterpret_cast<char*>(buf) + LOG_HEADER_CREATOR, LOG_HEADER_CREATOR_CURRENT); @@ -1009,7 +1021,7 @@ log_group_file_header_flush( page_id_t(group->space_id, page_no), univ_page_size, (ulint) (dest_offset % univ_page_size.physical()), - OS_FILE_LOG_BLOCK_SIZE, buf, group, NULL); + OS_FILE_LOG_BLOCK_SIZE, buf, group); srv_stats.os_log_pending_writes.dec(); } @@ -1105,6 +1117,10 @@ loop: || log_block_get_hdr_no(buf) == log_block_convert_lsn_to_no(start_lsn)); + if (log_sys->is_encrypted()) { + log_crypt(buf, write_len); + } + /* Calculate the checksums for each log block and write them to the trailer fields of the log blocks */ @@ -1125,8 +1141,6 @@ loop: ut_a(next_offset / UNIV_PAGE_SIZE <= ULINT_MAX); - log_encrypt_before_write(log_sys->next_checkpoint_no, - buf, write_len); const ulint page_no = (ulint) (next_offset / univ_page_size.physical()); @@ -1134,7 +1148,7 @@ loop: page_id_t(group->space_id, page_no), univ_page_size, (ulint) (next_offset % UNIV_PAGE_SIZE), write_len, buf, - group, NULL); + group); srv_stats.os_log_pending_writes.dec(); @@ -1152,11 +1166,14 @@ loop: } } -/** Flush the log has been written to the log file. */ +/** Flush the recently written changes to the log file. +and invoke log_mutex_enter(). */ static void log_write_flush_to_disk_low() { + /* FIXME: This is not holding log_sys->mutex while + calling os_event_set()! */ ut_a(log_sys->n_pending_flushes == 1); /* No other threads here */ #ifndef _WIN32 @@ -1165,13 +1182,17 @@ log_write_flush_to_disk_low() bool do_flush = true; #endif if (do_flush) { - log_group_t* group = UT_LIST_GET_FIRST(log_sys->log_groups); - fil_flush(group->space_id); + fil_flush(SRV_LOG_SPACE_FIRST_ID); + } + + MONITOR_DEC(MONITOR_PENDING_LOG_FLUSH); + + log_mutex_enter(); + if (do_flush) { log_sys->flushed_to_disk_lsn = log_sys->current_flush_lsn; } log_sys->n_pending_flushes--; - MONITOR_DEC(MONITOR_PENDING_LOG_FLUSH); os_event_set(log_sys->flush_event); } @@ -1265,7 +1286,6 @@ loop: } #ifdef _WIN32 -# ifndef UNIV_HOTBACKUP /* write requests during fil_flush() might not be good for Windows */ if (log_sys->n_pending_flushes > 0 || !os_event_is_set(log_sys->flush_event)) { @@ -1273,11 +1293,6 @@ loop: os_event_wait(log_sys->flush_event); goto loop; } -# else - if (log_sys->n_pending_flushes > 0) { - goto loop; - } -# endif /* !UNIV_HOTBACKUP */ #endif /* _WIN32 */ /* If it is a write call we should just go ahead and do it @@ -1332,6 +1347,7 @@ loop: /* Nothing to write, flush only */ log_mutex_exit_all(); log_write_flush_to_disk_low(); + log_mutex_exit(); return; } } @@ -1411,6 +1427,7 @@ loop: log_write_flush_to_disk_low(); ib_uint64_t write_lsn = log_sys->write_lsn; ib_uint64_t flush_lsn = log_sys->flushed_to_disk_lsn; + log_mutex_exit(); innobase_mysql_log_notify(write_lsn, flush_lsn); } @@ -1481,7 +1498,7 @@ log_flush_margin(void) log_write_up_to(lsn, false); } } -#ifndef UNIV_HOTBACKUP + /** Advances the smallest lsn for which there are unflushed dirty blocks in the buffer pool. NOTE: this function may only be called if the calling thread owns no @@ -1550,7 +1567,7 @@ log_preflush_pool_modified_pages( return(success); } -#endif /* !UNIV_HOTBACKUP */ + /******************************************************//** Completes a checkpoint. */ static @@ -1608,9 +1625,6 @@ log_group_checkpoint( ut_ad(!srv_read_only_mode); ut_ad(log_mutex_own()); -#if LOG_CHECKPOINT_SIZE > OS_FILE_LOG_BLOCK_SIZE -# error "LOG_CHECKPOINT_SIZE > OS_FILE_LOG_BLOCK_SIZE" -#endif DBUG_PRINT("ib_log", ("checkpoint " UINT64PF " at " LSN_PF " written to group " ULINTPF, @@ -1624,7 +1638,9 @@ log_group_checkpoint( mach_write_to_8(buf + LOG_CHECKPOINT_NO, log_sys->next_checkpoint_no); mach_write_to_8(buf + LOG_CHECKPOINT_LSN, log_sys->next_checkpoint_lsn); - log_crypt_write_checkpoint_buf(buf); + if (log_sys->is_encrypted()) { + log_crypt_write_checkpoint_buf(buf); + } lsn_offset = log_group_calc_lsn_offset(log_sys->next_checkpoint_lsn, group); @@ -1660,55 +1676,11 @@ log_group_checkpoint( (log_sys->next_checkpoint_no & 1) ? LOG_CHECKPOINT_2 : LOG_CHECKPOINT_1, OS_FILE_LOG_BLOCK_SIZE, - buf, (byte*) group + 1, NULL); + buf, (byte*) group + 1); ut_ad(((ulint) group & 0x1UL) == 0); } -#ifdef UNIV_HOTBACKUP -/******************************************************//** -Writes info to a buffer of a log group when log files are created in -backup restoration. */ -void -log_reset_first_header_and_checkpoint( -/*==================================*/ - byte* hdr_buf,/*!< in: buffer which will be written to the - start of the first log file */ - ib_uint64_t start) /*!< in: lsn of the start of the first log file; - we pretend that there is a checkpoint at - start + LOG_BLOCK_HDR_SIZE */ -{ - byte* buf; - ib_uint64_t lsn; - - mach_write_to_4(hdr_buf + LOG_HEADER_FORMAT, - LOG_HEADER_FORMAT_CURRENT); - mach_write_to_8(hdr_buf + LOG_HEADER_START_LSN, start); - - lsn = start + LOG_BLOCK_HDR_SIZE; - - /* Write the label of mysqlbackup --restore */ - strcpy((char*)hdr_buf + LOG_HEADER_CREATOR, LOG_HEADER_CREATOR_CURRENT); - ut_sprintf_timestamp((char*) hdr_buf - + (LOG_HEADER_CREATOR - + (sizeof LOG_HEADER_CREATOR_CURRENT) - 1)); - buf = hdr_buf + LOG_CHECKPOINT_1; - memset(buf, 0, OS_FILE_LOG_BLOCK_SIZE); - - /*mach_write_to_8(buf + LOG_CHECKPOINT_NO, 0);*/ - mach_write_to_8(buf + LOG_CHECKPOINT_LSN, lsn); - - log_crypt_write_checkpoint_buf(buf); - - mach_write_to_8(buf + LOG_CHECKPOINT_OFFSET, - LOG_FILE_HDR_SIZE + LOG_BLOCK_HDR_SIZE); - mach_write_to_8(buf + LOG_CHECKPOINT_LOG_BUF_SIZE, 2 * 1024 * 1024); - - log_block_set_checksum(buf, log_block_calc_checksum_crc32(buf)); -} -#endif /* UNIV_HOTBACKUP */ - -#ifndef UNIV_HOTBACKUP /** Read a log group header page to log_sys->checkpoint_buf. @param[in] group log group @param[in] header 0 or LOG_CHEKCPOINT_1 or LOG_CHECKPOINT2 */ @@ -1726,36 +1698,22 @@ log_group_header_read( fil_io(IORequestLogRead, true, page_id_t(group->space_id, header / univ_page_size.physical()), univ_page_size, header % univ_page_size.physical(), - OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL, NULL); + OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL); } /** Write checkpoint info to the log header and invoke log_mutex_exit(). @param[in] sync whether to wait for the write to complete */ void -log_write_checkpoint_info( - bool sync) +log_write_checkpoint_info(bool sync) { - log_group_t* group; - ut_ad(log_mutex_own()); + ut_ad(!srv_read_only_mode); - if (!srv_read_only_mode) { - for (group = UT_LIST_GET_FIRST(log_sys->log_groups); - group; - group = UT_LIST_GET_NEXT(log_groups, group)) { - - log_group_checkpoint(group); - } - } + for (log_group_t* group = UT_LIST_GET_FIRST(log_sys->log_groups); + group; + group = UT_LIST_GET_NEXT(log_groups, group)) { - /* generate key version and key used to encrypt future blocks, - * - * NOTE: the +1 is as the next_checkpoint_no will be updated once - * the checkpoint info has been written and THEN blocks will be encrypted - * with new key - */ - if (srv_encrypt_log) { - log_crypt_set_ver_and_key(log_sys->next_checkpoint_no + 1); + log_group_checkpoint(group); } log_mutex_exit(); @@ -2034,78 +1992,6 @@ loop: } } -/******************************************************//** -Reads a specified log segment to a buffer. */ -void -log_group_read_log_seg( -/*===================*/ - byte* buf, /*!< in: buffer where to read */ - log_group_t* group, /*!< in: log group */ - lsn_t start_lsn, /*!< in: read area start */ - lsn_t end_lsn) /*!< in: read area end */ -{ - ulint len; - lsn_t source_offset; - - ut_ad(log_mutex_own()); - -loop: - source_offset = log_group_calc_lsn_offset(start_lsn, group); - - ut_a(end_lsn - start_lsn <= ULINT_MAX); - len = (ulint) (end_lsn - start_lsn); - - ut_ad(len != 0); - - if ((source_offset % group->file_size) + len > group->file_size) { - - /* If the above condition is true then len (which is ulint) - is > the expression below, so the typecast is ok */ - len = (ulint) (group->file_size - - (source_offset % group->file_size)); - } - - log_sys->n_log_ios++; - - MONITOR_INC(MONITOR_LOG_IO); - - ut_a(source_offset / UNIV_PAGE_SIZE <= ULINT_MAX); - - const ulint page_no - = (ulint) (source_offset / univ_page_size.physical()); - - fil_io(IORequestLogRead, true, - page_id_t(group->space_id, page_no), - univ_page_size, - (ulint) (source_offset % univ_page_size.physical()), - len, buf, NULL, NULL); - -#ifdef DEBUG_CRYPT - fprintf(stderr, "BEFORE DECRYPT: block: %lu checkpoint: %lu %.8lx %.8lx offset %lu\n", - log_block_get_hdr_no(buf), - log_block_get_checkpoint_no(buf), - log_block_calc_checksum(buf), - log_block_get_checksum(buf), source_offset); -#endif - - log_decrypt_after_read(buf, len); - -#ifdef DEBUG_CRYPT - fprintf(stderr, "AFTER DECRYPT: block: %lu checkpoint: %lu %.8lx %.8lx\n", - log_block_get_hdr_no(buf), - log_block_get_checkpoint_no(buf), - log_block_calc_checksum(buf), - log_block_get_checksum(buf)); -#endif - start_lsn += len; - buf += len; - - if (start_lsn != end_lsn) { - - goto loop; - } -} - /** Checks that there is enough free space in the log to start a new query step. Flushes the log buffer or makes a new checkpoint if necessary. NOTE: this @@ -2137,11 +2023,7 @@ logs_empty_and_mark_files_at_shutdown(void) { lsn_t lsn; ulint count = 0; - ulint total_trx; ulint pending_io; - enum srv_thread_type active_thd; - const char* thread_name; - dberr_t err = DB_SUCCESS; ib::info() << "Starting shutdown..."; @@ -2156,37 +2038,31 @@ logs_empty_and_mark_files_at_shutdown(void) srv_shutdown_state = SRV_SHUTDOWN_CLEANUP; loop: - os_thread_sleep(100000); + os_event_set(srv_buf_resize_event); - count++; - - /* We need the monitor threads to stop before we proceed with - a shutdown. */ - - thread_name = srv_any_background_threads_are_active(); - - if (thread_name != NULL) { - /* Print a message every 60 seconds if we are waiting - for the monitor thread to exit. Master and worker - threads check will be done later. */ - - if (srv_print_verbose_log && count > 600) { - ib::info() << "Waiting for " << thread_name - << " to exit"; - count = 0; + if (!srv_read_only_mode) { + os_event_set(srv_error_event); + os_event_set(srv_monitor_event); + os_event_set(srv_buf_dump_event); + os_event_set(lock_sys->timeout_event); + if (dict_stats_event) { + os_event_set(dict_stats_event); + } else { + ut_ad(!srv_dict_stats_thread_active); } - - goto loop; } + os_thread_sleep(100000); + + count++; /* Check that there are no longer transactions, except for PREPARED ones. We need this wait even for the 'very fast' shutdown, because the InnoDB layer may have committed or prepared transactions and we don't want to lose them. */ - total_trx = trx_sys_any_active_transactions(); - - if (total_trx > 0) { + if (ulint total_trx = srv_was_started && !srv_read_only_mode + && srv_force_recovery < SRV_FORCE_NO_TRX_UNDO + ? trx_sys_any_active_transactions() : 0) { if (srv_print_verbose_log && count > 600) { ib::info() << "Waiting for " << total_trx << " active" @@ -2198,55 +2074,61 @@ loop: goto loop; } - /* Check that the background threads are suspended */ - - active_thd = srv_get_active_thread_type(); - - if (active_thd != SRV_NONE) { - - if (active_thd == SRV_PURGE) { - srv_purge_wakeup(); - } + /* We need these threads to stop early in shutdown. */ + const char* thread_name; + + if (srv_error_monitor_active) { + thread_name = "srv_error_monitor_thread"; + } else if (srv_monitor_active) { + thread_name = "srv_monitor_thread"; + } else if (srv_buf_resize_thread_active) { + thread_name = "buf_resize_thread"; + } else if (srv_dict_stats_thread_active) { + thread_name = "dict_stats_thread"; + } else if (lock_sys->timeout_thread_active) { + thread_name = "lock_wait_timeout_thread"; + } else if (srv_buf_dump_thread_active) { + thread_name = "buf_dump_thread"; + } else { + thread_name = NULL; + } - /* The srv_lock_timeout_thread, srv_error_monitor_thread - and srv_monitor_thread should already exit by now. The - only threads to be suspended are the master threads - and worker threads (purge threads). Print the thread - type if any of such threads not in suspended mode */ + if (thread_name) { + ut_ad(!srv_read_only_mode); +wait_suspend_loop: if (srv_print_verbose_log && count > 600) { - const char* thread_type = "<null>"; - - switch (active_thd) { - case SRV_NONE: - /* This shouldn't happen because we've - already checked for this case before - entering the if(). We handle it here - to avoid a compiler warning. */ - ut_error; - case SRV_WORKER: - thread_type = "worker threads"; - break; - case SRV_MASTER: - thread_type = "master thread"; - break; - case SRV_PURGE: - thread_type = "purge thread"; - break; - } - - ib::info() << "Waiting for " << thread_type - << " to be suspended"; - + ib::info() << "Waiting for " << thread_name + << "to exit"; count = 0; } - goto loop; } + /* Check that the background threads are suspended */ + + switch (srv_get_active_thread_type()) { + case SRV_NONE: + srv_shutdown_state = SRV_SHUTDOWN_FLUSH_PHASE; + if (!srv_n_fil_crypt_threads_started) { + break; + } + os_event_set(fil_crypt_threads_event); + thread_name = "fil_crypt_thread"; + goto wait_suspend_loop; + case SRV_PURGE: + case SRV_WORKER: + srv_purge_wakeup(); + thread_name = "purge thread"; + goto wait_suspend_loop; + case SRV_MASTER: + thread_name = "master thread"; + goto wait_suspend_loop; + } + /* At this point only page_cleaner should be active. We wait here to let it complete the flushing of the buffer pools before proceeding further. */ - srv_shutdown_state = SRV_SHUTDOWN_FLUSH_PHASE; + count = 0; while (buf_page_cleaner_is_active) { ++count; @@ -2258,12 +2140,17 @@ loop: } } + if (log_scrub_thread_active) { + ut_ad(!srv_read_only_mode); + os_event_set(log_scrub_event); + } + log_mutex_enter(); const ulint n_write = log_sys->n_pending_checkpoint_writes; const ulint n_flush = log_sys->n_pending_flushes; log_mutex_exit(); - if (n_write != 0 || n_flush != 0) { + if (log_scrub_thread_active || n_write || n_flush) { if (srv_print_verbose_log && count > 600) { ib::info() << "Pending checkpoint_writes: " << n_write << ". Pending log flush writes: " << n_flush; @@ -2272,6 +2159,8 @@ loop: goto loop; } + ut_ad(!log_scrub_thread_active); + pending_io = buf_pool_check_no_pending_io(); if (pending_io) { @@ -2284,8 +2173,8 @@ loop: goto loop; } - if (srv_fast_shutdown == 2) { - if (!srv_read_only_mode) { + if (srv_fast_shutdown == 2 || !srv_was_started) { + if (!srv_read_only_mode && srv_was_started) { ib::info() << "MySQL has requested a very fast" " shutdown without flushing the InnoDB buffer" " pool to data files. At the next mysqld" @@ -2302,80 +2191,54 @@ loop: from the stamps if the previous shutdown was clean. */ log_buffer_flush_to_disk(); - - /* Check that the background threads stay suspended */ - thread_name = srv_any_background_threads_are_active(); - - if (thread_name != NULL) { - ib::warn() << "Background thread " - << thread_name << " woke up during" - " shutdown"; - goto loop; - } } srv_shutdown_state = SRV_SHUTDOWN_LAST_PHASE; fil_close_all_files(); - - thread_name = srv_any_background_threads_are_active(); - - ut_a(!thread_name); - return; } if (!srv_read_only_mode) { log_make_checkpoint_at(LSN_MAX, TRUE); - } - log_mutex_enter(); - - lsn = log_sys->lsn; - - ut_ad(lsn >= log_sys->last_checkpoint_lsn); + log_mutex_enter(); - log_mutex_exit(); + lsn = log_sys->lsn; - if (lsn != log_sys->last_checkpoint_lsn) { - goto loop; - } + const bool lsn_changed = lsn != log_sys->last_checkpoint_lsn; + ut_ad(lsn >= log_sys->last_checkpoint_lsn); - /* Check that the background threads stay suspended */ - thread_name = srv_any_background_threads_are_active(); - if (thread_name != NULL) { - ib::warn() << "Background thread " << thread_name << " woke up" - " during shutdown"; + log_mutex_exit(); - goto loop; - } + if (lsn_changed) { + goto loop; + } - if (!srv_read_only_mode) { fil_flush_file_spaces(FIL_TYPE_TABLESPACE); fil_flush_file_spaces(FIL_TYPE_LOG); - } - /* The call fil_write_flushed_lsn() will bypass the buffer - pool: therefore it is essential that the buffer pool has been - completely flushed to disk! (We do not call fil_write... if the - 'very fast' shutdown is enabled.) */ + /* The call fil_write_flushed_lsn_to_data_files() will + bypass the buffer pool: therefore it is essential that + the buffer pool has been completely flushed to disk! */ - if (!buf_all_freed()) { + if (!buf_all_freed()) { + if (srv_print_verbose_log && count > 600) { + ib::info() << "Waiting for dirty buffer pages" + " to be flushed"; + count = 0; + } - if (srv_print_verbose_log && count > 600) { - ib::info() << "Waiting for dirty buffer pages to be" - " flushed"; - count = 0; + goto loop; } - - goto loop; + } else { + lsn = srv_start_lsn; } srv_shutdown_state = SRV_SHUTDOWN_LAST_PHASE; /* Make some checks that the server really is quiet */ - srv_thread_type type = srv_get_active_thread_type(); - ut_a(type == SRV_NONE); + ut_a(srv_get_active_thread_type() == SRV_NONE); bool freed = buf_all_freed(); ut_a(freed); @@ -2383,27 +2246,25 @@ loop: ut_a(lsn == log_sys->lsn); if (lsn < srv_start_lsn) { - ib::error() << "Log sequence number at shutdown " << lsn - << " is lower than at startup " << srv_start_lsn - << "!"; + ib::error() << "Shutdown LSN=" << lsn + << " is less than start LSN=" << srv_start_lsn; } srv_shutdown_lsn = lsn; if (!srv_read_only_mode) { - err = fil_write_flushed_lsn(lsn); + dberr_t err = fil_write_flushed_lsn(lsn); if (err != DB_SUCCESS) { ib::error() << "Writing flushed lsn " << lsn - << " failed at shutdown error " << err; + << " failed; error=" << err; } } fil_close_all_files(); /* Make some checks that the server really is quiet */ - type = srv_get_active_thread_type(); - ut_a(type == SRV_NONE); + ut_a(srv_get_active_thread_type() == SRV_NONE); freed = buf_all_freed(); ut_a(freed); @@ -2552,6 +2413,10 @@ log_shutdown(void) mutex_free(&log_sys->write_mutex); mutex_free(&log_sys->log_flush_order_mutex); + if (!srv_read_only_mode && srv_scrub_log) { + os_event_destroy(log_scrub_event); + } + recv_sys_close(); } @@ -2569,11 +2434,6 @@ log_mem_free(void) } } -/** Event to wake up the log scrub thread */ -UNIV_INTERN os_event_t log_scrub_event = NULL; - -UNIV_INTERN ibool srv_log_scrub_thread_active = FALSE; - /******************************************************//** Pads the current log block full with dummy log records. Used in producing consistent archived log files and scrubbing redo log. */ @@ -2600,6 +2460,10 @@ log_pad_current_log_block(void) pad_length = 0; } + if (pad_length) { + srv_stats.n_log_scrubs.inc(); + } + for (i = 0; i < pad_length; i++) { log_write_low(&b, 1); } @@ -2619,6 +2483,7 @@ void log_scrub() /*=========*/ { + log_mutex_enter(); ulint cur_lbn = log_block_convert_lsn_to_no(log_sys->lsn); if (next_lbn_to_pad == cur_lbn) @@ -2627,6 +2492,7 @@ log_scrub() } next_lbn_to_pad = log_block_convert_lsn_to_no(log_sys->lsn); + log_mutex_exit(); } /* log scrubbing speed, in bytes/sec */ @@ -2639,28 +2505,22 @@ sleeps again. @return this function does not return, it calls os_thread_exit() */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(log_scrub_thread)( -/*===============================*/ - void* arg __attribute__((unused))) /*!< in: a dummy parameter - required by os_thread_create */ +DECLARE_THREAD(log_scrub_thread)(void*) { ut_ad(!srv_read_only_mode); - srv_log_scrub_thread_active = TRUE; - - while(srv_shutdown_state == SRV_SHUTDOWN_NONE) - { + while (srv_shutdown_state < SRV_SHUTDOWN_FLUSH_PHASE) { /* log scrubbing interval in µs. */ ulonglong interval = 1000*1000*512/innodb_scrub_log_speed; - os_event_wait_time(log_scrub_event, interval); + os_event_wait_time(log_scrub_event, static_cast<ulint>(interval)); log_scrub(); os_event_reset(log_scrub_event); } - srv_log_scrub_thread_active = FALSE; + log_scrub_thread_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ @@ -2668,4 +2528,3 @@ DECLARE_THREAD(log_scrub_thread)( OS_THREAD_DUMMY_RETURN; } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index a7c3f337287..f7c7c45240b 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2,7 +2,7 @@ Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -34,16 +34,11 @@ Created 9/20/1997 Heikki Tuuri #include "log0recv.h" -#ifdef UNIV_NONINL -#include "log0recv.ic" -#endif - #ifdef HAVE_MY_AES_H #include <my_aes.h> #endif #include "log0crypt.h" - #include "mem0mem.h" #include "buf0buf.h" #include "buf0flu.h" @@ -60,19 +55,11 @@ Created 9/20/1997 Heikki Tuuri #include "fsp0sysspace.h" #include "ut0new.h" #include "row0trunc.h" -#ifndef UNIV_HOTBACKUP -# include "buf0rea.h" -# include "srv0srv.h" -# include "srv0start.h" -# include "trx0roll.h" -# include "row0merge.h" -#else /* !UNIV_HOTBACKUP */ -/** This is set to false if the backup was originally taken with the -mysqlbackup --include regexp option: then we do not want to create tables in -directories which were not included */ -bool recv_replay_file_ops = true; -#include "fut0lst.h" -#endif /* !UNIV_HOTBACKUP */ +#include "buf0rea.h" +#include "srv0srv.h" +#include "srv0start.h" +#include "trx0roll.h" +#include "row0merge.h" /** Log records are stored in the hash table in chunks at most of this size; this must be less than UNIV_PAGE_SIZE as it is stored in the buffer pool */ @@ -88,18 +75,13 @@ otherwise. Note that this is FALSE while a background thread is rolling back incomplete transactions. */ volatile bool recv_recovery_on; -#ifndef UNIV_HOTBACKUP /** TRUE when recv_init_crash_recovery() has been called. */ bool recv_needed_recovery; -#else -# define recv_needed_recovery false -# define buf_pool_get_curr_size() (5 * 1024 * 1024) -#endif /* !UNIV_HOTBACKUP */ -# ifdef UNIV_DEBUG +#ifdef UNIV_DEBUG /** TRUE if writing to the redo log (mtr_commit) is forbidden. Protected by log_sys->mutex. */ bool recv_no_log_write = false; -# endif /* UNIV_DEBUG */ +#endif /* UNIV_DEBUG */ /** TRUE if buf_page_is_corrupted() should check if the log sequence number (FIL_PAGE_LSN) is in the future. Initially FALSE, and set by @@ -115,24 +97,7 @@ buffer pool before the pages have been recovered to the up-to-date state. TRUE means that recovery is running and no operations on the log files are allowed yet: the variable name is misleading. */ -#ifndef UNIV_HOTBACKUP bool recv_no_ibuf_operations; -/** TRUE when the redo log is being backed up */ -# define recv_is_making_a_backup false -/** TRUE when recovering from a backed up redo log file */ -# define recv_is_from_backup false -#else /* !UNIV_HOTBACKUP */ -/** true if the backup is an offline backup */ -volatile bool is_online_redo_copy = true; -/**true if the last flushed lsn read at the start of backup */ -volatile lsn_t backup_redo_log_flushed_lsn; - -/** TRUE when the redo log is being backed up */ -bool recv_is_making_a_backup = false; -/** TRUE when recovering from a backed up redo log file */ -bool recv_is_from_backup = false; -# define buf_pool_get_curr_size() (5 * 1024 * 1024) -#endif /* !UNIV_HOTBACKUP */ /** The following counter is used to decide when to print info on log scan */ static ulint recv_scan_print_counter; @@ -159,16 +124,11 @@ lsn_t recv_max_page_lsn; #ifdef UNIV_PFS_THREAD mysql_pfs_key_t trx_rollback_clean_thread_key; -#endif /* UNIV_PFS_THREAD */ - -#ifndef UNIV_HOTBACKUP -# ifdef UNIV_PFS_THREAD mysql_pfs_key_t recv_writer_thread_key; -# endif /* UNIV_PFS_THREAD */ +#endif /* UNIV_PFS_THREAD */ /** Flag indicating if recv_writer thread is active. */ -volatile bool recv_writer_thread_active = false; -#endif /* !UNIV_HOTBACKUP */ +static volatile bool recv_writer_thread_active; #ifndef DBUG_OFF /** Return string name of the redo log record type. @@ -180,7 +140,6 @@ get_mlog_string(mlog_id_t type); /* prototypes */ -#ifndef UNIV_HOTBACKUP /*******************************************************//** Initialize crash recovery environment. Can be called iff recv_needed_recovery == false. */ @@ -188,7 +147,6 @@ static void recv_init_crash_recovery(void); /*===========================*/ -#endif /* !UNIV_HOTBACKUP */ /** Tablespace item during recovery */ struct file_name_t { @@ -266,42 +224,6 @@ fil_name_process( case FIL_LOAD_OK: ut_ad(space != NULL); -#ifdef MYSQL_ENCRYPTION - /* For encrypted tablespace, set key and iv. */ - if (FSP_FLAGS_GET_ENCRYPTION(space->flags) - && recv_sys->encryption_list != NULL) { - dberr_t err; - encryption_list_t::iterator it; - - for (it = recv_sys->encryption_list->begin(); - it != recv_sys->encryption_list->end(); - it++) { - if (it->space_id == space->id) { - err = fil_set_encryption( - space->id, - Encryption::AES, - it->key, - it->iv); - if (err != DB_SUCCESS) { - ib::error() - << "Can't set" - " encryption" - " information" - " for" - " tablespace" - << space->name - << "!"; - } - ut_free(it->key); - ut_free(it->iv); - it->key = NULL; - it->iv = NULL; - it->space_id = 0; - } - } - } -#endif /* MYSQL_ENCRYPTION */ - if (f.space == NULL || f.space == space) { f.name = fname.name; f.space = space; @@ -344,7 +266,6 @@ fil_name_process( case FIL_LOAD_INVALID: ut_ad(space == NULL); if (srv_force_recovery == 0) { -#ifndef UNIV_HOTBACKUP ib::warn() << "We do not continue the crash" " recovery, because the table may" " become corrupt if we cannot apply" @@ -366,13 +287,6 @@ fil_name_process( " remove the .ibd file, you can set" " --innodb_force_recovery."; recv_sys->found_corrupt_fs = true; -#else - ib::warn() << "We do not continue the apply-log" - " operation because the tablespace may" - " become corrupt if we cannot apply" - " the log records in the redo log" - " records to it."; -#endif /* !UNIV_BACKUP */ processed = false; break; } @@ -387,7 +301,6 @@ fil_name_process( return(processed); } -#ifndef UNIV_HOTBACKUP /** Parse or process a MLOG_FILE_* record. @param[in] ptr redo log record @param[in] end end of the redo log buffer @@ -514,286 +427,6 @@ fil_name_parse( return(end_ptr); } -#else /* !UNIV_HOTBACKUP */ -/** Parse a file name retrieved from a MLOG_FILE_* record, -and return the absolute file path corresponds to backup dir -as well as in the form of database/tablespace -@param[in] file_name path emitted by the redo log -@param[out] absolute_path absolute path of tablespace -corresponds to backup dir -@param[out] tablespace_name name in the form of database/table */ -static -void -make_abs_file_path( - const std::string& name, - std::string& absolute_path, - std::string& tablespace_name) -{ - std::string file_name = name; - std::string path = fil_path_to_mysql_datadir; - size_t pos = std::string::npos; - - if (is_absolute_path(file_name.c_str())) { - - pos = file_name.rfind(OS_PATH_SEPARATOR); - std::string temp_name = file_name.substr(0, pos); - pos = temp_name.rfind(OS_PATH_SEPARATOR); - ++pos; - file_name = file_name.substr(pos, file_name.length()); - path += OS_PATH_SEPARATOR + file_name; - } else { - pos = file_name.find(OS_PATH_SEPARATOR); - ++pos; - file_name = file_name.substr(pos, file_name.length()); - path += OS_PATH_SEPARATOR + file_name; - } - - absolute_path = path; - - /* remove the .ibd extension */ - pos = file_name.rfind(".ibd"); - if (pos != std::string::npos) - tablespace_name = file_name.substr(0, pos); - - /* space->name uses '/', not OS_PATH_SEPARATOR, - update the seperator */ - if (OS_PATH_SEPARATOR != '/') { - pos = tablespace_name.find(OS_PATH_SEPARATOR); - while (pos != std::string::npos) { - tablespace_name[pos] = '/'; - pos = tablespace_name.find(OS_PATH_SEPARATOR); - } - } - -} - -/** Wrapper around fil_name_process() -@param[in] name absolute path of tablespace file -@param[in] space_id the tablespace ID -@retval true if able to process file successfully. -@retval false if unable to process the file */ -bool -fil_name_process( - const char* name, - ulint space_id) -{ - size_t length = strlen(name); - ++length; - - char* file_name = static_cast<char*>(ut_malloc_nokey(length)); - strncpy(file_name, name,length); - - bool processed = fil_name_process(file_name, length, space_id, false); - - ut_free(file_name); - return(processed); -} - -/** Parse or process a MLOG_FILE_* record. -@param[in] ptr redo log record -@param[in] end end of the redo log buffer -@param[in] space_id the tablespace ID -@param[in] first_page_no first page number in the file -@param[in] type MLOG_FILE_NAME or MLOG_FILE_DELETE -or MLOG_FILE_CREATE2 or MLOG_FILE_RENAME2 -@param[in] apply whether to apply the record -@retval pointer to next redo log record -@retval NULL if this log record was truncated */ -static -byte* -fil_name_parse( - byte* ptr, - const byte* end, - ulint space_id, - ulint first_page_no, - mlog_id_t type, - bool apply) -{ - - ulint flags = mach_read_from_4(ptr); - - if (type == MLOG_FILE_CREATE2) { - if (end < ptr + 4) { - return(NULL); - } - ptr += 4; - } - - if (end < ptr + 2) { - return(NULL); - } - - ulint len = mach_read_from_2(ptr); - ptr += 2; - if (end < ptr + len) { - return(NULL); - } - - os_normalize_path(reinterpret_cast<char*>(ptr)); - - /* MLOG_FILE_* records should only be written for - user-created tablespaces. The name must be long enough - and end in .ibd. */ - bool corrupt = is_predefined_tablespace(space_id) - || first_page_no != 0 // TODO: multi-file user tablespaces - || len < sizeof "/a.ibd\0" - || memcmp(ptr + len - 5, DOT_IBD, 5) != 0 - || memchr(ptr, OS_PATH_SEPARATOR, len) == NULL; - - byte* end_ptr = ptr + len; - - if (corrupt) { - recv_sys->found_corrupt_log = true; - return(end_ptr); - } - - std::string abs_file_path, tablespace_name; - char* name = reinterpret_cast<char*>(ptr); - char* new_name = NULL; - recv_spaces_t::iterator itr; - - make_abs_file_path(name, abs_file_path, tablespace_name); - - if (!recv_is_making_a_backup) { - - name = static_cast<char*>(ut_malloc_nokey( - (abs_file_path.length() + 1))); - strcpy(name, abs_file_path.c_str()); - len = strlen(name) + 1; - } - switch (type) { - default: - ut_ad(0); // the caller checked this - case MLOG_FILE_NAME: - /* Don't validate tablespaces while copying redo logs - because backup process might keep some tablespace handles - open in server datadir. - Maintain "map of dirty tablespaces" so that assumptions - for other redo log records are not broken even for dirty - tablespaces during apply log */ - if (!recv_is_making_a_backup) { - recv_spaces.insert(std::make_pair(space_id, - file_name_t(abs_file_path, - false))); - } - break; - case MLOG_FILE_DELETE: - /* Don't validate tablespaces while copying redo logs - because backup process might keep some tablespace handles - open in server datadir. */ - if (recv_is_making_a_backup) - break; - - fil_name_process( - name, len, space_id, true); - - if (apply && recv_replay_file_ops - && fil_space_get(space_id)) { - dberr_t err = fil_delete_tablespace( - space_id, BUF_REMOVE_FLUSH_NO_WRITE); - ut_a(err == DB_SUCCESS); - } - - break; - case MLOG_FILE_CREATE2: - if (recv_is_making_a_backup - || (!recv_replay_file_ops) - || (is_intermediate_file(abs_file_path.c_str())) - || (fil_space_get(space_id)) - || (fil_space_get_id_by_name( - tablespace_name.c_str()) != ULINT_UNDEFINED)) { - /* Don't create table while :- - 1. scanning the redo logs during backup - 2. apply-log on a partial backup - 3. if it is intermediate file - 4. tablespace is already loaded in memory */ - } else { - itr = recv_spaces.find(space_id); - if (itr == recv_spaces.end() - || (itr->second.name != abs_file_path)) { - - dberr_t ret = fil_ibd_create( - space_id, tablespace_name.c_str(), - abs_file_path.c_str(), - flags, FIL_IBD_FILE_INITIAL_SIZE); - - if (ret != DB_SUCCESS) { - ib::fatal() << "Could not create the" - << " tablespace : " - << abs_file_path - << " with space Id : " - << space_id; - } - } - } - break; - case MLOG_FILE_RENAME2: - /* The new name follows the old name. */ - byte* new_table_name = end_ptr + 2; - if (end < new_table_name) { - return(NULL); - } - - ulint new_len = mach_read_from_2(end_ptr); - - if (end < end_ptr + 2 + new_len) { - return(NULL); - } - - end_ptr += 2 + new_len; - - char* new_table = reinterpret_cast<char*>(new_table_name); - os_normalize_path(new_table); - - corrupt = corrupt - || new_len < sizeof "/a.ibd\0" - || memcmp(new_table_name + new_len - 5, DOT_IBD, 5) != 0 - || !memchr(new_table_name, OS_PATH_SEPARATOR, new_len); - - if (corrupt) { - recv_sys->found_corrupt_log = true; - break; - } - - if (recv_is_making_a_backup - || (!recv_replay_file_ops) - || (is_intermediate_file(name)) - || (is_intermediate_file(new_table))) { - /* Don't rename table while :- - 1. scanning the redo logs during backup - 2. apply-log on a partial backup - 3. The new name is already used. - 4. A tablespace is not open in memory with the old name. - This will prevent unintended renames during recovery. */ - break; - } else { - make_abs_file_path(new_table, abs_file_path, - tablespace_name); - - new_name = static_cast<char*>(ut_malloc_nokey( - (abs_file_path.length() + 1))); - strcpy(new_name, abs_file_path.c_str()); - new_len = strlen(new_name) + 1; - } - - fil_name_process(name, len, space_id, false); - fil_name_process( new_name, new_len, space_id, false); - - if (!fil_op_replay_rename( - space_id, first_page_no, - name, - new_name)) { - recv_sys->found_corrupt_fs = true; - } - } - - if (!recv_is_making_a_backup) { - ut_free(name); - ut_free(new_name); - } - return(end_ptr); -} -#endif /* UNIV_HOTBACKUP */ /********************************************************//** Creates the recovery system. */ @@ -822,6 +455,8 @@ recv_sys_close(void) /*================*/ { if (recv_sys != NULL) { + recv_sys->dblwr.pages.clear(); + if (recv_sys->addr_hash != NULL) { hash_table_free(recv_sys->addr_hash); } @@ -829,7 +464,7 @@ recv_sys_close(void) if (recv_sys->heap != NULL) { mem_heap_free(recv_sys->heap); } -#ifndef UNIV_HOTBACKUP + if (recv_sys->flush_start != NULL) { os_event_destroy(recv_sys->flush_start); } @@ -837,14 +472,11 @@ recv_sys_close(void) if (recv_sys->flush_end != NULL) { os_event_destroy(recv_sys->flush_end); } -#endif /* !UNIV_HOTBACKUP */ + ut_free(recv_sys->buf); - ut_free(recv_sys->last_block_buf_start); -#ifndef UNIV_HOTBACKUP ut_ad(!recv_writer_thread_active); mutex_free(&recv_sys->writer_mutex); -#endif /* !UNIV_HOTBACKUP */ mutex_free(&recv_sys->mutex); @@ -869,7 +501,7 @@ recv_sys_mem_free(void) if (recv_sys->heap != NULL) { mem_heap_free(recv_sys->heap); } -#ifndef UNIV_HOTBACKUP + if (recv_sys->flush_start != NULL) { os_event_destroy(recv_sys->flush_start); } @@ -877,15 +509,13 @@ recv_sys_mem_free(void) if (recv_sys->flush_end != NULL) { os_event_destroy(recv_sys->flush_end); } -#endif /* !UNIV_HOTBACKUP */ + ut_free(recv_sys->buf); - ut_free(recv_sys->last_block_buf_start); ut_free(recv_sys); recv_sys = NULL; } } -#ifndef UNIV_HOTBACKUP /************************************************************ Reset the state of the recovery system variables. */ void @@ -927,8 +557,6 @@ DECLARE_THREAD(recv_writer_thread)( << os_thread_pf(os_thread_get_curr_id()); #endif /* UNIV_DEBUG_THREAD_CREATION */ - recv_writer_thread_active = true; - while (srv_shutdown_state == SRV_SHUTDOWN_NONE) { /* Wait till we get a signal to clean the LRU list. @@ -961,7 +589,6 @@ DECLARE_THREAD(recv_writer_thread)( OS_THREAD_DUMMY_RETURN; } -#endif /* !UNIV_HOTBACKUP */ /************************************************************ Inits the recovery system for a recovery operation. */ @@ -975,7 +602,6 @@ recv_sys_init( return; } -#ifndef UNIV_HOTBACKUP mutex_enter(&(recv_sys->mutex)); recv_sys->heap = mem_heap_create_typed(256, @@ -985,10 +611,6 @@ recv_sys_init( recv_sys->flush_start = os_event_create(0); recv_sys->flush_end = os_event_create(0); } -#else /* !UNIV_HOTBACKUP */ - recv_sys->heap = mem_heap_create(256); - recv_is_from_backup = true; -#endif /* !UNIV_HOTBACKUP */ /* Set appropriate value of recv_n_pool_free_frames. */ if (buf_pool_get_curr_size() >= (10 * 1024 * 1024)) { @@ -1007,12 +629,6 @@ recv_sys_init( recv_sys->apply_log_recs = FALSE; recv_sys->apply_batch_on = FALSE; - recv_sys->last_block_buf_start = static_cast<byte*>( - ut_malloc_nokey(2 * OS_FILE_LOG_BLOCK_SIZE)); - - recv_sys->last_block = static_cast<byte*>(ut_align( - recv_sys->last_block_buf_start, OS_FILE_LOG_BLOCK_SIZE)); - recv_sys->found_corrupt_log = false; recv_sys->found_corrupt_fs = false; recv_sys->mlog_checkpoint_lsn = 0; @@ -1022,7 +638,6 @@ recv_sys_init( /* Call the constructor for recv_sys_t::dblwr member */ new (&recv_sys->dblwr) recv_dblwr_t(); - recv_sys->encryption_list = NULL; mutex_exit(&(recv_sys->mutex)); } @@ -1046,8 +661,6 @@ recv_sys_empty_hash(void) recv_sys->addr_hash = hash_create(buf_pool_get_curr_size() / 512); } -#ifndef UNIV_HOTBACKUP - /********************************************************//** Frees the recovery system. */ void @@ -1059,12 +672,10 @@ recv_sys_debug_free(void) hash_table_free(recv_sys->addr_hash); mem_heap_free(recv_sys->heap); ut_free(recv_sys->buf); - ut_free(recv_sys->last_block_buf_start); recv_sys->buf = NULL; recv_sys->heap = NULL; recv_sys->addr_hash = NULL; - recv_sys->last_block_buf_start = NULL; /* wake page cleaner up to progress */ if (!srv_read_only_mode) { @@ -1074,29 +685,100 @@ recv_sys_debug_free(void) os_event_set(recv_sys->flush_start); } - if (recv_sys->encryption_list != NULL) { - encryption_list_t::iterator it; + mutex_exit(&(recv_sys->mutex)); +} + +/** Read a log segment to a buffer. +@param[out] buf buffer +@param[in] group redo log files +@param[in] start_lsn read area start +@param[in] end_lsn read area end +@return valid end_lsn */ +static +lsn_t +log_group_read_log_seg( + byte* buf, + const log_group_t* group, + lsn_t start_lsn, + lsn_t end_lsn) +{ + ulint len; + lsn_t source_offset; + + ut_ad(log_mutex_own()); + +loop: + source_offset = log_group_calc_lsn_offset(start_lsn, group); + + ut_a(end_lsn - start_lsn <= ULINT_MAX); + len = (ulint) (end_lsn - start_lsn); + + ut_ad(len != 0); + + const bool at_eof = (source_offset % group->file_size) + len + > group->file_size; + if (at_eof) { + /* If the above condition is true then len (which is ulint) + is > the expression below, so the typecast is ok */ + len = (ulint) (group->file_size - + (source_offset % group->file_size)); + } - for (it = recv_sys->encryption_list->begin(); - it != recv_sys->encryption_list->end(); - it++) { - if (it->key != NULL) { - ut_free(it->key); - it->key = NULL; + log_sys->n_log_ios++; + + MONITOR_INC(MONITOR_LOG_IO); + + ut_a(source_offset / UNIV_PAGE_SIZE <= ULINT_MAX); + + const ulint page_no + = (ulint) (source_offset / univ_page_size.physical()); + + fil_io(IORequestLogRead, true, + page_id_t(group->space_id, page_no), + univ_page_size, + (ulint) (source_offset % univ_page_size.physical()), + len, buf, NULL); + + for (ulint l = 0; l < len; l += OS_FILE_LOG_BLOCK_SIZE, + buf += OS_FILE_LOG_BLOCK_SIZE, + start_lsn += OS_FILE_LOG_BLOCK_SIZE) { + const ulint block_number = log_block_get_hdr_no(buf); + + if (block_number != log_block_convert_lsn_to_no(start_lsn)) { + /* Garbage or an incompletely written log block. + We will not report any error, because this can + happen when InnoDB was killed while it was + writing redo log. We simply treat this as an + abrupt end of the redo log. */ + return(start_lsn); + } + + if (innodb_log_checksums || group->is_encrypted()) { + ulint crc = log_block_calc_checksum_crc32(buf); + ulint cksum = log_block_get_checksum(buf); + + if (crc != cksum) { + ib::error() << "Invalid log block checksum." + << " block: " << block_number + << " checkpoint no: " + << log_block_get_checkpoint_no(buf) + << " expected: " << crc + << " found: " << cksum; + return(start_lsn); } - if (it->iv != NULL) { - ut_free(it->iv); - it->iv = NULL; + + if (group->is_encrypted()) { + log_crypt(buf, OS_FILE_LOG_BLOCK_SIZE, true); } } + } - recv_sys->encryption_list->swap(*recv_sys->encryption_list); + if (start_lsn != end_lsn) { - UT_DELETE(recv_sys->encryption_list); - recv_sys->encryption_list = NULL; + goto loop; } - mutex_exit(&(recv_sys->mutex)); + return(start_lsn); } /********************************************************//** @@ -1106,27 +788,20 @@ about the latest checkpoint to the groups, and inits the fields in the group memory structs to up-to-date values. */ static void -recv_synchronize_groups(void) -/*=========================*/ +recv_synchronize_groups() { - lsn_t start_lsn; - lsn_t end_lsn; - lsn_t recovered_lsn; - - recovered_lsn = recv_sys->recovered_lsn; + const lsn_t recovered_lsn = recv_sys->recovered_lsn; /* Read the last recovered log block to the recovery system buffer: the block is always incomplete */ - start_lsn = ut_uint64_align_down(recovered_lsn, - OS_FILE_LOG_BLOCK_SIZE); - end_lsn = ut_uint64_align_up(recovered_lsn, OS_FILE_LOG_BLOCK_SIZE); - - ut_a(start_lsn != end_lsn); - - log_group_read_log_seg(recv_sys->last_block, + const lsn_t start_lsn = ut_uint64_align_down(recovered_lsn, + OS_FILE_LOG_BLOCK_SIZE); + log_group_read_log_seg(log_sys->buf, UT_LIST_GET_FIRST(log_sys->log_groups), - start_lsn, end_lsn); + start_lsn, start_lsn + OS_FILE_LOG_BLOCK_SIZE); + + ut_ad(UT_LIST_GET_LEN(log_sys->log_groups) == 1); for (log_group_t* group = UT_LIST_GET_FIRST(log_sys->log_groups); group; @@ -1142,10 +817,11 @@ recv_synchronize_groups(void) over the max checkpoint info, thus making the preservation of max checkpoint info on disk certain */ - log_write_checkpoint_info(true); - log_mutex_enter(); + if (!srv_read_only_mode) { + log_write_checkpoint_info(true); + log_mutex_enter(); + } } -#endif /* !UNIV_HOTBACKUP */ /** Check the consistency of a log header block. @param[in] log header block @@ -1159,7 +835,6 @@ recv_check_log_header_checksum( == log_block_calc_checksum_crc32(buf)); } -#ifndef UNIV_HOTBACKUP /** Find the latest checkpoint in the format-0 log header. @param[out] max_group log group, or NULL @param[out] max_field LOG_CHECKPOINT_1 or LOG_CHECKPOINT_2 @@ -1197,9 +872,8 @@ recv_find_max_checkpoint_0( ut_fold_binary(buf + LOG_CHECKPOINT_LSN, CHECKSUM_2 - LOG_CHECKPOINT_LSN)) != mach_read_from_4(buf + CHECKSUM_2)) { - DBUG_PRINT("ib_log", - ("invalid pre-5.7.9 checkpoint " ULINTPF, - field)); + DBUG_LOG("ib_log", + "invalid pre-10.2.2 checkpoint " << field); continue; } @@ -1213,9 +887,9 @@ recv_find_max_checkpoint_0( checkpoint_no = mach_read_from_8( buf + LOG_CHECKPOINT_NO); - if (!log_crypt_read_checkpoint_buf(buf)) { - ib::error() << "Reading checkpoint encryption info failed."; - return DB_ERROR; + if (!log_crypt_101_read_checkpoint(buf)) { + ib::error() << "Decrypting checkpoint failed"; + continue; } DBUG_PRINT("ib_log", @@ -1235,14 +909,14 @@ recv_find_max_checkpoint_0( } ib::error() << "Upgrade after a crash is not supported." - " This redo log was created before MySQL 5.7.9," + " This redo log was created before MariaDB 10.2.2," " and we did not find a valid checkpoint." " Please follow the instructions at" " " REFMAN "upgrading.html"; return(DB_ERROR); } -/** Determine if a pre-5.7.9 redo log is clean. +/** Determine if a pre-MySQL 5.7.9/MariaDB 10.2.2 redo log is clean. @param[in] lsn checkpoint LSN @return error code @retval DB_SUCCESS if the redo log is clean @@ -1262,7 +936,7 @@ recv_log_format_0_recover(lsn_t lsn) static const char* NO_UPGRADE_RECOVERY_MSG = "Upgrade after a crash is not supported." - " This redo log was created before MySQL 5.7.9"; + " This redo log was created before MariaDB 10.2.2"; static const char* NO_UPGRADE_RTFM_MSG = ". Please follow the instructions at " REFMAN "upgrading.html"; @@ -1272,10 +946,11 @@ recv_log_format_0_recover(lsn_t lsn) univ_page_size, (ulint) ((source_offset & ~(OS_FILE_LOG_BLOCK_SIZE - 1)) % univ_page_size.physical()), - OS_FILE_LOG_BLOCK_SIZE, buf, NULL, NULL); + OS_FILE_LOG_BLOCK_SIZE, buf, NULL); if (log_block_calc_checksum_format_0(buf) - != log_block_get_checksum(buf)) { + != log_block_get_checksum(buf) + && !log_crypt_101_read_block(buf)) { ib::error() << NO_UPGRADE_RECOVERY_MSG << ", and it appears corrupted" << NO_UPGRADE_RTFM_MSG; @@ -1344,6 +1019,7 @@ recv_find_max_checkpoint( return(recv_find_max_checkpoint_0( max_group, max_field)); case LOG_HEADER_FORMAT_CURRENT: + case LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED: break; default: /* Ensure that the string is NUL-terminated. */ @@ -1377,6 +1053,13 @@ recv_find_max_checkpoint( continue; } + if (group->is_encrypted() + && !log_crypt_read_checkpoint_buf(buf)) { + ib::error() << "Reading checkpoint" + " encryption info failed."; + continue; + } + group->state = LOG_GROUP_OK; group->lsn = mach_read_from_8( @@ -1386,11 +1069,6 @@ recv_find_max_checkpoint( checkpoint_no = mach_read_from_8( buf + LOG_CHECKPOINT_NO); - if (!log_crypt_read_checkpoint_buf(buf)) { - ib::error() << "Reading checkpoint encryption info failed."; - return DB_ERROR; - } - DBUG_PRINT("ib_log", ("checkpoint " UINT64PF " at " LSN_PF " found in group " ULINTPF, @@ -1421,282 +1099,6 @@ recv_find_max_checkpoint( return(DB_SUCCESS); } -#else /* !UNIV_HOTBACKUP */ -/*******************************************************************//** -Reads the checkpoint info needed in hot backup. -@return TRUE if success */ -ibool -recv_read_checkpoint_info_for_backup( -/*=================================*/ - const byte* hdr, /*!< in: buffer containing the log group - header */ - lsn_t* lsn, /*!< out: checkpoint lsn */ - lsn_t* offset, /*!< out: checkpoint offset in the log group */ - lsn_t* cp_no, /*!< out: checkpoint number */ - lsn_t* first_header_lsn) - /*!< out: lsn of of the start of the - first log file */ -{ - ulint max_cp = 0; - ib_uint64_t max_cp_no = 0; - const byte* cp_buf; - - cp_buf = hdr + LOG_CHECKPOINT_1; - - if (recv_check_log_header_checksum(cp_buf)) { - max_cp_no = mach_read_from_8(cp_buf + LOG_CHECKPOINT_NO); - max_cp = LOG_CHECKPOINT_1; - } - - cp_buf = hdr + LOG_CHECKPOINT_2; - - if (recv_check_log_header_checksum(cp_buf)) { - if (mach_read_from_8(cp_buf + LOG_CHECKPOINT_NO) > max_cp_no) { - max_cp = LOG_CHECKPOINT_2; - } - } - - if (max_cp == 0) { - return(FALSE); - } - - cp_buf = hdr + max_cp; - - *lsn = mach_read_from_8(cp_buf + LOG_CHECKPOINT_LSN); - *offset = mach_read_from_8( - cp_buf + LOG_CHECKPOINT_OFFSET); - - *cp_no = mach_read_from_8(cp_buf + LOG_CHECKPOINT_NO); - - *first_header_lsn = mach_read_from_8(hdr + LOG_HEADER_START_LSN); - - return(TRUE); -} -#endif /* !UNIV_HOTBACKUP */ - -/** Check the 4-byte checksum to the trailer checksum field of a log -block. -@param[in] log block -@return whether the checksum matches */ -bool -log_block_checksum_is_ok( - const byte* block, /*!< in: pointer to a log block */ - bool print_err) /*!< in print error ? */ -{ - if (log_block_get_checksum(block) != log_block_calc_checksum(block) && - print_err) { - ib::error() << " Log block checkpoint not correct." - << " block: " << log_block_get_hdr_no(block) - << " checkpoint no: " << log_block_get_checkpoint_no(block) - << " calc checkpoint: " << log_block_calc_checksum(block) - << " stored checkpoint: " << log_block_get_checksum(block); - } - - return(!innodb_log_checksums - || log_block_get_checksum(block) - == log_block_calc_checksum(block)); -} - -#ifdef UNIV_HOTBACKUP -/*******************************************************************//** -Scans the log segment and n_bytes_scanned is set to the length of valid -log scanned. */ -void -recv_scan_log_seg_for_backup( -/*=========================*/ - byte* buf, /*!< in: buffer containing log data */ - ulint buf_len, /*!< in: data length in that buffer */ - lsn_t* scanned_lsn, /*!< in/out: lsn of buffer start, - we return scanned lsn */ - ulint* scanned_checkpoint_no, - /*!< in/out: 4 lowest bytes of the - highest scanned checkpoint number so - far */ - ulint* n_bytes_scanned)/*!< out: how much we were able to - scan, smaller than buf_len if log - data ended here */ -{ - ulint data_len; - byte* log_block; - ulint no; - - *n_bytes_scanned = 0; - - for (log_block = buf; log_block < buf + buf_len; - log_block += OS_FILE_LOG_BLOCK_SIZE) { - - no = log_block_get_hdr_no(log_block); - -#if 0 - fprintf(stderr, "Log block header no %lu\n", no); -#endif - - if (no != log_block_convert_lsn_to_no(*scanned_lsn) - || !log_block_checksum_is_ok(log_block)) { -#if 0 - fprintf(stderr, - "Log block n:o %lu, scanned lsn n:o %lu\n", - no, log_block_convert_lsn_to_no(*scanned_lsn)); -#endif - /* Garbage or an incompletely written log block */ - - log_block += OS_FILE_LOG_BLOCK_SIZE; -#if 0 - fprintf(stderr, - "Next log block n:o %lu\n", - log_block_get_hdr_no(log_block)); -#endif - break; - } - - if (*scanned_checkpoint_no > 0 - && log_block_get_checkpoint_no(log_block) - < *scanned_checkpoint_no - && *scanned_checkpoint_no - - log_block_get_checkpoint_no(log_block) - > 0x80000000UL) { - - /* Garbage from a log buffer flush which was made - before the most recent database recovery */ -#if 0 - fprintf(stderr, - "Scanned cp n:o %lu, block cp n:o %lu\n", - *scanned_checkpoint_no, - log_block_get_checkpoint_no(log_block)); -#endif - break; - } - - data_len = log_block_get_data_len(log_block); - - *scanned_checkpoint_no - = log_block_get_checkpoint_no(log_block); - *scanned_lsn += data_len; - - *n_bytes_scanned += data_len; - - if (data_len < OS_FILE_LOG_BLOCK_SIZE) { - /* Log data ends here */ - -#if 0 - fprintf(stderr, "Log block data len %lu\n", - data_len); -#endif - break; - } - } -} -#endif /* UNIV_HOTBACKUP */ - -#ifdef MYSQL_ENCRYPTION - -/** Parse or process a write encryption info record. -@param[in] ptr redo log record -@param[in] end end of the redo log buffer -@param[in] space_id the tablespace ID -@return log record end, NULL if not a complete record */ -static -byte* -fil_write_encryption_parse( - byte* ptr, - const byte* end, - ulint space_id) -{ - fil_space_t* space; - ulint offset; - ulint len; - byte* key = NULL; - byte* iv = NULL; - bool is_new = false; - - space = fil_space_get(space_id); - if (space == NULL) { - encryption_list_t::iterator it; - - if (recv_sys->encryption_list == NULL) { - recv_sys->encryption_list = - UT_NEW_NOKEY(encryption_list_t()); - } - - for (it = recv_sys->encryption_list->begin(); - it != recv_sys->encryption_list->end(); - it++) { - if (it->space_id == space_id) { - key = it->key; - iv = it->iv; - } - } - - if (key == NULL) { - key = static_cast<byte*>(ut_malloc_nokey( - ENCRYPTION_KEY_LEN)); - iv = static_cast<byte*>(ut_malloc_nokey( - ENCRYPTION_KEY_LEN)); - is_new = true; - } - } else { - key = space->encryption_key; - iv = space->encryption_iv; - } - - offset = mach_read_from_2(ptr); - ptr += 2; - len = mach_read_from_2(ptr); - - ptr += 2; - if (end < ptr + len) { - return(NULL); - } - - if (offset >= UNIV_PAGE_SIZE - || len + offset > UNIV_PAGE_SIZE - || (len != ENCRYPTION_INFO_SIZE_V1 - && len != ENCRYPTION_INFO_SIZE_V2)) { - recv_sys->found_corrupt_log = TRUE; - return(NULL); - } - -#ifdef UNIV_ENCRYPT_DEBUG - if (space) { - fprintf(stderr, "Got %lu from redo log:", space->id); - } -#endif - - if (!fsp_header_decode_encryption_info(key, - iv, - ptr)) { - recv_sys->found_corrupt_log = TRUE; - ib::warn() << "Encryption information" - << " in the redo log of space " - << space_id << " is invalid"; - } - - ut_ad(len == ENCRYPTION_INFO_SIZE_V1 - || len == ENCRYPTION_INFO_SIZE_V2); - - ptr += len; - - if (space == NULL) { - if (is_new) { - recv_encryption_t info; - - /* Add key and iv to list */ - info.space_id = space_id; - info.key = key; - info.iv = iv; - - recv_sys->encryption_list->push_back(info); - } - } else { - ut_ad(FSP_FLAGS_GET_ENCRYPTION(space->flags)); - - space->encryption_type = Encryption::AES; - space->encryption_klen = ENCRYPTION_KEY_LEN; - } - - return(ptr); -} -#endif /* MYSQL_ENCRYPTION */ /** Try to parse a single log record body and also applies it if specified. @@ -1725,9 +1127,7 @@ recv_parse_or_apply_log_rec_body( mtr_t* mtr) { ut_ad(!block == !mtr); -#ifndef UNIV_HOTBACKUP ut_ad(!apply || recv_sys->mlog_checkpoint_lsn != 0); -#endif /* !UNIV_HOTBACKUP */ switch (type) { case MLOG_FILE_NAME: @@ -1740,86 +1140,12 @@ recv_parse_or_apply_log_rec_body( return(fil_name_parse(ptr, end_ptr, space_id, page_no, type, apply)); case MLOG_INDEX_LOAD: -#ifdef UNIV_HOTBACKUP - /* While scaning redo logs during backup phase a - MLOG_INDEX_LOAD type redo log record indicates a DDL - (create index, alter table...)is performed with - 'algorithm=inplace'. This redo log indicates that - - 1. The DDL was started after MEB started backing up, in which - case MEB will not be able to take a consistent backup and should - fail. or - 2. There is a possibility of this record existing in the REDO - even after the completion of the index create operation. This is - because of InnoDB does not checkpointing after the flushing the - index pages. - - If MEB gets the last_redo_flush_lsn and that is less than the - lsn of the current record MEB fails the backup process. - Error out in case of online backup and emit a warning in case - of offline backup and continue. - */ - if (!recv_recovery_on) { - if (is_online_redo_copy) { - if (backup_redo_log_flushed_lsn - < recv_sys->recovered_lsn) { - ib::trace() << "Last flushed lsn: " - << backup_redo_log_flushed_lsn - << " load_index lsn " - << recv_sys->recovered_lsn; - - if (backup_redo_log_flushed_lsn == 0) - ib::error() << "MEB was not " - "able to determine the" - "InnoDB Engine Status"; - - ib::fatal() << "An optimized(without" - " redo logging) DDLoperation" - " has been performed. All" - " modified pages may not have" - " been flushed to the disk yet." - " \n MEB will not be able" - " take a consistent backup." - " Retry the backup operation"; - } - /** else the index is flushed to disk before - backup started hence no error */ - } else { - /* offline backup */ - ib::trace() << "Last flushed lsn: " - << backup_redo_log_flushed_lsn - << " load_index lsn " - << recv_sys->recovered_lsn; - - ib::warn() << "An optimized(without redo" - " logging) DDL operation has been" - " performed. All modified pages may not" - " have been flushed to the disk yet." - " \n This offline backup may not" - " be consistent"; - } - } -#endif /* UNIV_HOTBACKUP */ if (end_ptr < ptr + 8) { return(NULL); } return(ptr + 8); case MLOG_TRUNCATE: return(truncate_t::parse_redo_entry(ptr, end_ptr, space_id)); - case MLOG_WRITE_STRING: - /* For encrypted tablespace, we need to get the - encryption key information before the page 0 is recovered. - Otherwise, redo will not find the key to decrypt - the data pages. */ -#ifdef MYSQL_ENCRYPTION - if (page_no == 0 && !is_system_tablespace(space_id) - && !apply) { - return(fil_write_encryption_parse(ptr, - end_ptr, - space_id)); - } -#endif - break; default: break; @@ -2377,19 +1703,12 @@ recv_data_copy_to_buf( } } -/************************************************************************//** -Applies the hashed log records to the page, if the page lsn is less than the -lsn of a log record. This can be called when a buffer page has just been -read in, or also for a page already in the buffer pool. */ +/** Apply the hashed log records to the page, if the page lsn is less than the +lsn of a log record. +@param just_read_in whether the page recently arrived to the I/O handler +@param block the page in the buffer pool */ void -recv_recover_page_func( -/*===================*/ -#ifndef UNIV_HOTBACKUP - ibool just_read_in, - /*!< in: TRUE if the i/o handler calls - this for a freshly read page */ -#endif /* !UNIV_HOTBACKUP */ - buf_block_t* block) /*!< in/out: buffer block */ +recv_recover_page(bool just_read_in, buf_block_t* block) { page_t* page; page_zip_des_t* page_zip; @@ -2427,13 +1746,11 @@ recv_recover_page_func( return; } -#ifndef UNIV_HOTBACKUP ut_ad(recv_needed_recovery); DBUG_PRINT("ib_log", ("Applying log to page %u:%u", recv_addr->space, recv_addr->page_no)); -#endif /* !UNIV_HOTBACKUP */ recv_addr->state = RECV_BEING_PROCESSED; @@ -2445,7 +1762,6 @@ recv_recover_page_func( page = block->frame; page_zip = buf_block_get_page_zip(block); -#ifndef UNIV_HOTBACKUP if (just_read_in) { /* Move the ownership of the x-latch on the page to this OS thread, so that we can acquire a second @@ -2461,12 +1777,10 @@ recv_recover_page_func( ut_a(success); buf_block_dbg_add_level(block, SYNC_NO_ORDER_CHECK); -#endif /* !UNIV_HOTBACKUP */ /* Read the newest modification lsn from the page */ page_lsn = mach_read_from_8(page + FIL_PAGE_LSN); -#ifndef UNIV_HOTBACKUP /* It may be that the page has been modified in the buffer pool: read the newest modification lsn there */ @@ -2476,10 +1790,6 @@ recv_recover_page_func( page_lsn = page_newest_lsn; } -#else /* !UNIV_HOTBACKUP */ - /* In recovery from a backup we do not really use the buffer pool */ - page_newest_lsn = 0; -#endif /* !UNIV_HOTBACKUP */ modification_to_page = FALSE; start_lsn = end_lsn = 0; @@ -2592,7 +1902,6 @@ recv_recover_page_func( } #endif /* UNIV_ZIP_DEBUG */ -#ifndef UNIV_HOTBACKUP if (modification_to_page) { ut_a(block); @@ -2600,9 +1909,6 @@ recv_recover_page_func( buf_flush_recv_note_modification(block, start_lsn, end_lsn); log_flush_order_mutex_exit(); } -#else /* !UNIV_HOTBACKUP */ - start_lsn = start_lsn; /* Silence compiler */ -#endif /* !UNIV_HOTBACKUP */ /* Make sure that committing mtr does not change the modification lsn values of page */ @@ -2626,7 +1932,6 @@ recv_recover_page_func( } -#ifndef UNIV_HOTBACKUP /** Reads in pages which have hashed log records, from an area around a given page number. @param[in] page_id page id @@ -2864,174 +2169,6 @@ loop: return err; } -#else /* !UNIV_HOTBACKUP */ -/*******************************************************************//** -Applies log records in the hash table to a backup. */ -void -recv_apply_log_recs_for_backup(void) -/*================================*/ -{ - recv_addr_t* recv_addr; - ulint n_hash_cells; - buf_block_t* block; - bool success; - ulint error; - ulint i; - fil_space_t* space = NULL; - page_id_t page_id; - recv_sys->apply_log_recs = TRUE; - recv_sys->apply_batch_on = TRUE; - - block = back_block1; - - ib::info() << "Starting an apply batch of log records to the" - " database...\n"; - - fputs("InnoDB: Progress in percent: ", stderr); - - n_hash_cells = hash_get_n_cells(recv_sys->addr_hash); - - for (i = 0; i < n_hash_cells; i++) { - /* The address hash table is externally chained */ - recv_addr = static_cast<recv_addr_t*>(hash_get_nth_cell( - recv_sys->addr_hash, i)->node); - - while (recv_addr != NULL) { - - ib::trace() << "recv_addr {State: " << recv_addr->state - << ", Space id: " << recv_addr->space - << "Page no: " << recv_addr->page_no - << ". index i: " << i << "\n"; - - bool found; - const page_size_t& page_size - = fil_space_get_page_size(recv_addr->space, - &found); - - if (!found) { -#if 0 - fprintf(stderr, - "InnoDB: Warning: cannot apply" - " log record to" - " tablespace %lu page %lu,\n" - "InnoDB: because tablespace with" - " that id does not exist.\n", - recv_addr->space, recv_addr->page_no); -#endif - recv_addr->state = RECV_DISCARDED; - - ut_a(recv_sys->n_addrs); - recv_sys->n_addrs--; - - goto skip_this_recv_addr; - } - - /* We simulate a page read made by the buffer pool, to - make sure the recovery apparatus works ok. We must init - the block. */ - - buf_page_init_for_backup_restore( - page_id_t(recv_addr->space, recv_addr->page_no), - page_size, block); - - /* Extend the tablespace's last file if the page_no - does not fall inside its bounds; we assume the last - file is auto-extending, and mysqlbackup copied the file - when it still was smaller */ - fil_space_t* space - = fil_space_get(recv_addr->space); - - success = fil_space_extend( - space, recv_addr->page_no + 1); - if (!success) { - ib::fatal() << "Cannot extend tablespace " - << recv_addr->space << " to hold " - << recv_addr->page_no << " pages"; - } - - /* Read the page from the tablespace file using the - fil0fil.cc routines */ - - const page_id_t page_id(recv_addr->space, - recv_addr->page_no); - - if (page_size.is_compressed()) { - - error = fil_io( - IORequestRead, true, - page_id, - page_size, 0, page_size.physical(), - block->page.zip.data, NULL); - - if (error == DB_SUCCESS - && !buf_zip_decompress(block, TRUE)) { - ut_error; - } - } else { - - error = fil_io( - IORequestRead, true, - page_id, page_size, 0, - page_size.logical(), - block->frame, NULL); - } - - if (error != DB_SUCCESS) { - ib::fatal() << "Cannot read from tablespace " - << recv_addr->space << " page number " - << recv_addr->page_no; - } - - /* Apply the log records to this page */ - recv_recover_page(FALSE, block); - - /* Write the page back to the tablespace file using the - fil0fil.cc routines */ - - buf_flush_init_for_writing( - block, block->frame, - buf_block_get_page_zip(block), - mach_read_from_8(block->frame + FIL_PAGE_LSN), - fsp_is_checksum_disabled( - block->page.id.space())); - - if (page_size.is_compressed()) { - - error = fil_io( - IORequestWrite, true, page_id, - page_size, 0, page_size.physical(), - block->page.zip.data, NULL); - } else { - error = fil_io( - IORequestWrite, true, page_id, - page_size, 0, page_size.logical(), - block->frame, NULL); - } -skip_this_recv_addr: - recv_addr = static_cast<recv_addr_t*>(HASH_GET_NEXT( - addr_hash, recv_addr)); - } - - if ((100 * i) / n_hash_cells - != (100 * (i + 1)) / n_hash_cells) { - fprintf(stderr, "%lu ", - (ulint) ((100 * i) / n_hash_cells)); - fflush(stderr); - sd_notifyf(0, "STATUS=Applying batch of log records for" - " backup InnoDB: Progress %lu", - (ulint) (100 * i) / n_hash_cells); - } - } - - sd_notify(0, "STATUS=InnoDB: Apply batch for backup completed"); - - /* write logs in next line */ - fprintf(stderr, "\n"); - recv_sys->apply_log_recs = FALSE; - recv_sys->apply_batch_on = FALSE; - recv_sys_empty_hash(); -} -#endif /* !UNIV_HOTBACKUP */ /** Tries to parse a single log record. @param[out] type log record type @@ -3109,6 +2246,7 @@ recv_parse_log_rec( return(0); } + const byte* old_ptr = new_ptr; new_ptr = recv_parse_or_apply_log_rec_body( *type, new_ptr, end_ptr, *space, *page_no, apply, NULL, NULL); @@ -3117,6 +2255,14 @@ recv_parse_log_rec( return(0); } + if (*page_no == 0 && *type == MLOG_4BYTES + && mach_read_from_2(old_ptr) == FSP_HEADER_OFFSET + FSP_SIZE) { + old_ptr += 2; + fil_space_set_recv_size(*space, + mach_parse_compressed(&old_ptr, + end_ptr)); + } + return(new_ptr - ptr); } @@ -3188,12 +2334,10 @@ recv_report_corrupt_log( - recv_previous_parsed_rec_offset); putc('\n', stderr); -#ifndef UNIV_HOTBACKUP if (!srv_force_recovery) { ib::info() << "Set innodb_force_recovery to ignore this error."; return(false); } -#endif /* !UNIV_HOTBACKUP */ ib::warn() << "The log file may have been corrupt and it is possible" " that the log scan did not proceed far enough in recovery!" @@ -3226,8 +2370,7 @@ bool recv_parse_log_recs( lsn_t checkpoint_lsn, store_t store, - bool apply, - dberr_t* err) + bool apply) { byte* ptr; byte* end_ptr; @@ -3339,9 +2482,7 @@ loop: } recv_sys->mlog_checkpoint_lsn = recv_sys->recovered_lsn; -#ifndef UNIV_HOTBACKUP return(true); -#endif /* !UNIV_HOTBACKUP */ } break; case MLOG_FILE_NAME: @@ -3622,11 +2763,9 @@ recv_sys_justify_left_parsing_buf(void) recv_sys->recovered_offset = 0; } -/*******************************************************//** -Scans log from a buffer and stores new log data to the parsing buffer. -Parses and hashes the log records if new data found. Unless -UNIV_HOTBACKUP is defined, this function will apply log records -automatically when the hash table becomes full. +/** Scan redo log from a buffer and stores new log data to the parsing buffer. +Parse and hash the log records if new data found. +Apply log records automatically when the hash table becomes full. @return true if not able to scan any more in this log group */ static bool @@ -3646,68 +2785,22 @@ recv_scan_log_recs( lsn_t* contiguous_lsn, /*!< in/out: it is known that all log groups contain contiguous log data up to this lsn */ - lsn_t* group_scanned_lsn,/*!< out: scanning succeeded up to + lsn_t* group_scanned_lsn)/*!< out: scanning succeeded up to this lsn */ - dberr_t* err) /*!< out: error code or DB_SUCCESS */ { const byte* log_block = buf; - ulint no; lsn_t scanned_lsn = start_lsn; bool finished = false; ulint data_len; bool more_data = false; bool apply = recv_sys->mlog_checkpoint_lsn != 0; - bool maybe_encrypted = false; ut_ad(start_lsn % OS_FILE_LOG_BLOCK_SIZE == 0); ut_ad(len % OS_FILE_LOG_BLOCK_SIZE == 0); ut_ad(len >= OS_FILE_LOG_BLOCK_SIZE); - *err = DB_SUCCESS; do { ut_ad(!finished); - no = log_block_get_hdr_no(log_block); - ulint expected_no = log_block_convert_lsn_to_no(scanned_lsn); - if (no != expected_no) { - /* Garbage or an incompletely written log block. - - We will not report any error, because this can - happen when InnoDB was killed while it was - writing redo log. We simply treat this as an - abrupt end of the redo log. */ - finished = true; - break; - } - - if (!log_block_checksum_is_ok(log_block, true)) { - log_crypt_err_t log_crypt_err; - - maybe_encrypted = log_crypt_block_maybe_encrypted(log_block, - &log_crypt_err); - - /* Print checkpoint encryption keys if present */ - log_crypt_print_checkpoint_keys(log_block); - if (maybe_encrypted) { - /* Log block maybe encrypted finish processing*/ - log_crypt_print_error(log_crypt_err); - *err = DB_ERROR; - return (TRUE); - } - - ib::error() << "Log block " << no << - " at lsn " << scanned_lsn << " has valid" - " header, but checksum field contains " - << log_block_get_checksum(log_block) - << ", should be " - << log_block_calc_checksum(log_block); - /* Garbage or an incompletely written log block. - - This could be the result of killing the server - while it was writing this log block. We treat - this as an abrupt end of the redo log. */ - finished = true; - break; - } if (log_block_get_flush_bit(log_block)) { /* This block was a start of a log flush operation: @@ -3757,7 +2850,6 @@ recv_scan_log_recs( of startup type, we must initiate crash recovery environment before parsing these log records. */ -#ifndef UNIV_HOTBACKUP if (!recv_needed_recovery) { if (!srv_read_only_mode) { @@ -3767,14 +2859,12 @@ recv_scan_log_recs( recv_init_crash_recovery(); } else { - - ib::warn() << "Recovery skipped," - " --innodb-read-only set!"; - + ib::warn() << "innodb_read_only" + " prevents crash recovery"; + recv_needed_recovery = true; return(true); } } -#endif /* !UNIV_HOTBACKUP */ /* We were able to find more log data: add it to the parsing buffer if parse_start_lsn is already @@ -3787,16 +2877,12 @@ recv_scan_log_recs( recv_sys->found_corrupt_log = true; -#ifndef UNIV_HOTBACKUP if (!srv_force_recovery) { ib::error() << "Set innodb_force_recovery" " to ignore this error."; - *err = DB_ERROR; return(true); } -#endif /* !UNIV_HOTBACKUP */ - } else if (!recv_sys->found_corrupt_log) { more_data = recv_sys_add_to_parsing_buf( log_block, scanned_lsn); @@ -3818,8 +2904,7 @@ recv_scan_log_recs( *group_scanned_lsn = scanned_lsn; - if (recv_needed_recovery - || (recv_is_from_backup && !recv_is_making_a_backup)) { + if (recv_needed_recovery) { recv_scan_print_counter++; if (finished || (recv_scan_print_counter % 80 == 0)) { @@ -3831,25 +2916,13 @@ recv_scan_log_recs( if (more_data && !recv_sys->found_corrupt_log) { /* Try to parse more log records */ - bool parse_finished = false; - dberr_t parse_err = DB_SUCCESS; - parse_finished = recv_parse_log_recs(checkpoint_lsn, - *store_to_hash, apply, - &parse_err); - - if (parse_err != DB_SUCCESS) { - ib::info() << "Parsing more log records failed checkpoint_lsn " - << checkpoint_lsn << " error " << parse_err; - parse_finished = true; - } - - if (parse_finished) { + if (recv_parse_log_recs(checkpoint_lsn, + *store_to_hash, apply)) { ut_ad(recv_sys->found_corrupt_log || recv_sys->found_corrupt_fs || recv_sys->mlog_checkpoint_lsn == recv_sys->recovered_lsn); - *err = parse_err; return(true); } @@ -3868,7 +2941,6 @@ recv_scan_log_recs( return(finished); } -#ifndef UNIV_HOTBACKUP /** Scans log from a buffer and stores new log data to the parsing buffer. Parses and hashes the log records if new data found. @param[in,out] group log group @@ -3876,15 +2948,13 @@ Parses and hashes the log records if new data found. until which all redo log has been scanned @param[in] last_phase whether changes can be applied to the tablespaces -@param[out] err DB_SUCCESS or error code @return whether rescan is needed (not everything was stored) */ static bool recv_group_scan_log_recs( log_group_t* group, lsn_t* contiguous_lsn, - bool last_phase, - dberr_t* err) + bool last_phase) { DBUG_ENTER("recv_group_scan_log_recs"); DBUG_ASSERT(!last_phase || recv_sys->mlog_checkpoint_lsn > 0); @@ -3915,38 +2985,33 @@ recv_group_scan_log_recs( * (buf_pool_get_n_pages() - (recv_n_pool_free_frames * srv_buf_pool_instances)); - *err = DB_SUCCESS; - end_lsn = *contiguous_lsn = ut_uint64_align_down( + group->scanned_lsn = end_lsn = *contiguous_lsn = ut_uint64_align_down( *contiguous_lsn, OS_FILE_LOG_BLOCK_SIZE); do { - if (*err != DB_SUCCESS) { - DBUG_RETURN(false); - } - if (last_phase && store_to_hash == STORE_NO) { store_to_hash = STORE_IF_EXISTS; /* We must not allow change buffer merge here, because it would generate redo log records before we have finished the redo log scan. */ - *err = recv_apply_hashed_log_recs(FALSE); + if (recv_apply_hashed_log_recs(FALSE) != DB_SUCCESS) { + DBUG_RETURN(false); + } } start_lsn = end_lsn; - end_lsn += RECV_SCAN_SIZE; - - log_group_read_log_seg( - log_sys->buf, group, start_lsn, end_lsn); - } while (!recv_scan_log_recs( + end_lsn = log_group_read_log_seg( + log_sys->buf, group, start_lsn, + start_lsn + RECV_SCAN_SIZE); + } while (end_lsn != start_lsn + && !recv_scan_log_recs( available_mem, &store_to_hash, log_sys->buf, - RECV_SCAN_SIZE, + end_lsn - start_lsn, checkpoint_lsn, - start_lsn, contiguous_lsn, &group->scanned_lsn, err)); + start_lsn, contiguous_lsn, &group->scanned_lsn)); if (recv_sys->found_corrupt_log || recv_sys->found_corrupt_fs) { - ib::error() << "Found corrupted log when looking checkpoint lsn: " - << contiguous_lsn << " error = " << *err; DBUG_RETURN(false); } @@ -4097,6 +3162,7 @@ recv_init_crash_recovery_spaces(void) if (srv_force_recovery < SRV_FORCE_NO_LOG_REDO) { /* Spawn the background thread to flush dirty pages from the buffer pools. */ + recv_writer_thread_active = true; os_thread_create(recv_writer_thread, 0, 0); } @@ -4120,7 +3186,6 @@ recv_recovery_from_checkpoint_start( ib_uint64_t checkpoint_no; lsn_t contiguous_lsn; byte* buf; - byte log_hdr_buf[LOG_FILE_HDR_SIZE]; dberr_t err = DB_SUCCESS; /* Initialize red-black tree for fast insertions into the @@ -4129,8 +3194,7 @@ recv_recovery_from_checkpoint_start( if (srv_force_recovery >= SRV_FORCE_NO_LOG_REDO) { - ib::info() << "The user has set SRV_FORCE_NO_LOG_REDO on," - " skipping log redo"; + ib::info() << "innodb_force_recovery=6 skips redo log apply"; return(DB_SUCCESS); } @@ -4157,49 +3221,6 @@ recv_recovery_from_checkpoint_start( checkpoint_lsn = mach_read_from_8(buf + LOG_CHECKPOINT_LSN); checkpoint_no = mach_read_from_8(buf + LOG_CHECKPOINT_NO); - /* Read the first log file header to print a note if this is - a recovery from a restored InnoDB Hot Backup */ - - const page_id_t page_id(max_cp_group->space_id, 0); - - fil_io(IORequestLogRead, true, page_id, univ_page_size, 0, - LOG_FILE_HDR_SIZE, log_hdr_buf, max_cp_group, NULL); - - if (0 == ut_memcmp(log_hdr_buf + LOG_HEADER_CREATOR, - (byte*)"ibbackup", (sizeof "ibbackup") - 1)) { - - if (srv_read_only_mode) { - log_mutex_exit(); - - ib::error() << "Cannot restore from mysqlbackup," - " InnoDB running in read-only mode!"; - - return(DB_ERROR); - } - - /* This log file was created by mysqlbackup --restore: print - a note to the user about it */ - - ib::info() << "The log file was created by mysqlbackup" - " --apply-log at " - << log_hdr_buf + LOG_HEADER_CREATOR - << ". The following crash recovery is part of a" - " normal restore."; - - /* Replace the label. */ - ut_ad(LOG_HEADER_CREATOR_END - LOG_HEADER_CREATOR - >= sizeof LOG_HEADER_CREATOR_CURRENT); - memset(log_hdr_buf + LOG_HEADER_CREATOR, 0, - LOG_HEADER_CREATOR_END - LOG_HEADER_CREATOR); - strcpy(reinterpret_cast<char*>(log_hdr_buf) - + LOG_HEADER_CREATOR, LOG_HEADER_CREATOR_CURRENT); - - /* Write to the log file to wipe over the label */ - fil_io(IORequestLogWrite, true, page_id, - univ_page_size, 0, OS_FILE_LOG_BLOCK_SIZE, log_hdr_buf, - max_cp_group, NULL); - } - /* Start reading the log groups from the checkpoint lsn up. The variable contiguous_lsn contains an lsn up to which the log is known to be contiguously written to all log groups. */ @@ -4218,6 +3239,7 @@ recv_recovery_from_checkpoint_start( log_mutex_exit(); return(recv_log_format_0_recover(checkpoint_lsn)); case LOG_HEADER_FORMAT_CURRENT: + case LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED: break; default: ut_ad(0); @@ -4227,29 +3249,31 @@ recv_recovery_from_checkpoint_start( } /* Look for MLOG_CHECKPOINT. */ - recv_group_scan_log_recs(group, &contiguous_lsn, false, &err); + recv_group_scan_log_recs(group, &contiguous_lsn, false); /* The first scan should not have stored or applied any records. */ ut_ad(recv_sys->n_addrs == 0); ut_ad(!recv_sys->found_corrupt_fs); + if (srv_read_only_mode && recv_needed_recovery) { + log_mutex_exit(); + return(DB_READ_ONLY); + } + if (recv_sys->found_corrupt_log && !srv_force_recovery) { log_mutex_exit(); - ib::error() << "Found corrupted log when looking checkpoint lsn: " - << contiguous_lsn << " error = " << err; + ib::warn() << "Log scan aborted at LSN " << contiguous_lsn; return(DB_ERROR); } if (recv_sys->mlog_checkpoint_lsn == 0) { if (!srv_read_only_mode && group->scanned_lsn != checkpoint_lsn) { - ib::error() << "Ignoring the redo log due to missing" + ib::error() << "Missing" " MLOG_CHECKPOINT between the checkpoint " << checkpoint_lsn << " and the end " << group->scanned_lsn << "."; - if (srv_force_recovery < SRV_FORCE_NO_LOG_REDO) { - log_mutex_exit(); - return(DB_ERROR); - } + log_mutex_exit(); + return(DB_ERROR); } group->scanned_lsn = checkpoint_lsn; @@ -4257,13 +3281,11 @@ recv_recovery_from_checkpoint_start( } else { contiguous_lsn = checkpoint_lsn; rescan = recv_group_scan_log_recs( - group, &contiguous_lsn, false, &err); + group, &contiguous_lsn, false); if ((recv_sys->found_corrupt_log && !srv_force_recovery) || recv_sys->found_corrupt_fs) { log_mutex_exit(); - ib::error() << "Found corrupted log in lsn:" - << contiguous_lsn << " err = " << err; return(DB_ERROR); } } @@ -4314,15 +3336,12 @@ recv_recovery_from_checkpoint_start( if (rescan) { contiguous_lsn = checkpoint_lsn; - recv_group_scan_log_recs(group, &contiguous_lsn, true, &err); + recv_group_scan_log_recs(group, &contiguous_lsn, true); if ((recv_sys->found_corrupt_log && !srv_force_recovery) || recv_sys->found_corrupt_fs) { log_mutex_exit(); - ib::error() << "Found corrupted log in lsn:" - << contiguous_lsn << " err = " << err; - return(DB_ERROR); } } @@ -4362,10 +3381,6 @@ recv_recovery_from_checkpoint_start( log_sys->next_checkpoint_lsn = checkpoint_lsn; log_sys->next_checkpoint_no = checkpoint_no + 1; - /* here the checkpoint info is written without any redo logging ongoing - * and next_checkpoint_no is updated directly hence no +1 */ - log_crypt_set_ver_and_key(log_sys->next_checkpoint_no); - recv_synchronize_groups(); if (!recv_needed_recovery) { @@ -4374,8 +3389,6 @@ recv_recovery_from_checkpoint_start( srv_start_lsn = recv_sys->recovered_lsn; } - ut_memcpy(log_sys->buf, recv_sys->last_block, OS_FILE_LOG_BLOCK_SIZE); - log_sys->buf_free = (ulint) log_sys->lsn % OS_FILE_LOG_BLOCK_SIZE; log_sys->buf_next_to_write = log_sys->buf_free; log_sys->write_lsn = log_sys->lsn; @@ -4391,8 +3404,7 @@ recv_recovery_from_checkpoint_start( MONITOR_SET(MONITOR_LSN_CHECKPOINT_AGE, log_sys->lsn - log_sys->last_checkpoint_lsn); - log_sys->next_checkpoint_no = checkpoint_no + 1; - log_crypt_set_ver_and_key(log_sys->next_checkpoint_no); + log_sys->next_checkpoint_no = ++checkpoint_no; mutex_enter(&recv_sys->mutex); @@ -4446,42 +3458,6 @@ recv_recovery_from_checkpoint_finish(void) /* Free up the flush_rbt. */ buf_flush_free_flush_rbt(); - - /* Validate a few system page types that were left uninitialized - by older versions of MySQL. */ - mtr_t mtr; - buf_block_t* block; - mtr.start(); - mtr.set_sys_modified(); - /* Bitmap page types will be reset in buf_dblwr_check_block() - without redo logging. */ - block = buf_page_get( - page_id_t(IBUF_SPACE_ID, FSP_IBUF_HEADER_PAGE_NO), - univ_page_size, RW_X_LATCH, &mtr); - fil_block_check_type(block, FIL_PAGE_TYPE_SYS, &mtr); - /* Already MySQL 3.23.53 initialized FSP_IBUF_TREE_ROOT_PAGE_NO - to FIL_PAGE_INDEX. No need to reset that one. */ - block = buf_page_get( - page_id_t(TRX_SYS_SPACE, TRX_SYS_PAGE_NO), - univ_page_size, RW_X_LATCH, &mtr); - fil_block_check_type(block, FIL_PAGE_TYPE_TRX_SYS, &mtr); - block = buf_page_get( - page_id_t(TRX_SYS_SPACE, FSP_FIRST_RSEG_PAGE_NO), - univ_page_size, RW_X_LATCH, &mtr); - fil_block_check_type(block, FIL_PAGE_TYPE_SYS, &mtr); - block = buf_page_get( - page_id_t(TRX_SYS_SPACE, FSP_DICT_HDR_PAGE_NO), - univ_page_size, RW_X_LATCH, &mtr); - fil_block_check_type(block, FIL_PAGE_TYPE_SYS, &mtr); - mtr.commit(); - - /* Roll back any recovered data dictionary transactions, so - that the data dictionary tables will be free of any locks. - The data dictionary latch should guarantee that there is at - most one data dictionary transaction active at a time. */ - if (srv_force_recovery < SRV_FORCE_NO_TRX_UNDO) { - trx_rollback_or_clean_recovered(FALSE); - } } /********************************************************//** @@ -4504,8 +3480,6 @@ recv_recovery_rollback_active(void) /* Drop partially created indexes. */ row_merge_drop_temp_indexes(); - /* Drop temporary tables. */ - row_mysql_drop_temp_tables(); /* Drop any auxiliary tables that were not dropped when the parent table was dropped. This can happen if the parent table @@ -4532,18 +3506,14 @@ recv_reset_logs( which we add LOG_BLOCK_HDR_SIZE */ { - log_group_t* group; - ut_ad(log_mutex_own()); log_sys->lsn = ut_uint64_align_up(lsn, OS_FILE_LOG_BLOCK_SIZE); - group = UT_LIST_GET_FIRST(log_sys->log_groups); - - while (group) { + for (log_group_t* group = UT_LIST_GET_FIRST(log_sys->log_groups); + group; group = UT_LIST_GET_NEXT(log_groups, group)) { group->lsn = log_sys->lsn; group->lsn_offset = LOG_FILE_HDR_SIZE; - group = UT_LIST_GET_NEXT(log_groups, group); } log_sys->buf_next_to_write = 0; @@ -4569,100 +3539,6 @@ recv_reset_logs( log_mutex_enter(); } -#endif /* !UNIV_HOTBACKUP */ - -#ifdef UNIV_HOTBACKUP -/******************************************************//** -Creates new log files after a backup has been restored. */ -void -recv_reset_log_files_for_backup( -/*============================*/ - const char* log_dir, /*!< in: log file directory path */ - ulint n_log_files, /*!< in: number of log files */ - lsn_t log_file_size, /*!< in: log file size */ - lsn_t lsn) /*!< in: new start lsn, must be - divisible by OS_FILE_LOG_BLOCK_SIZE */ -{ - os_file_t log_file; - bool success; - byte* buf; - ulint i; - ulint log_dir_len; - char name[5000]; - static const char ib_logfile_basename[] = "ib_logfile"; - - log_dir_len = strlen(log_dir); - /* full path name of ib_logfile consists of log dir path + basename - + number. This must fit in the name buffer. - */ - ut_a(log_dir_len + strlen(ib_logfile_basename) + 11 < sizeof(name)); - - buf = (byte*)ut_zalloc_nokey(LOG_FILE_HDR_SIZE + - OS_FILE_LOG_BLOCK_SIZE); - - for (i = 0; i < n_log_files; i++) { - - sprintf(name, "%s%s%lu", log_dir, - ib_logfile_basename, (ulint) i); - - log_file = os_file_create_simple(innodb_log_file_key, - name, OS_FILE_CREATE, - OS_FILE_READ_WRITE, - srv_read_only_mode, &success); - if (!success) { - ib::fatal() << "Cannot create " << name << ". Check that" - " the file does not exist yet."; - } - - ib::info() << "Setting log file size to " << log_file_size; - - success = os_file_set_size( - name, log_file, log_file_size, srv_read_only_mode); - - if (!success) { - ib::fatal() << "Cannot set " << name << " size to " - << (long long unsigned)log_file_size; - } - - os_file_flush(log_file); - os_file_close(log_file); - } - - /* We pretend there is a checkpoint at lsn + LOG_BLOCK_HDR_SIZE */ - - log_reset_first_header_and_checkpoint(buf, lsn); - - log_block_init(buf + LOG_FILE_HDR_SIZE, lsn); - log_block_set_first_rec_group(buf + LOG_FILE_HDR_SIZE, - LOG_BLOCK_HDR_SIZE); - log_block_set_checksum(buf + LOG_FILE_HDR_SIZE, - log_block_calc_checksum_crc32(buf + LOG_FILE_HDR_SIZE)); - - log_block_set_checksum(buf, log_block_calc_checksum_crc32(buf)); - sprintf(name, "%s%s%lu", log_dir, ib_logfile_basename, (ulint)0); - - log_file = os_file_create_simple(innodb_log_file_key, - name, OS_FILE_OPEN, - OS_FILE_READ_WRITE, - srv_read_only_mode, &success); - if (!success) { - ib::fatal() << "Cannot open " << name << "."; - } - - IORequest request(IORequest::WRITE); - - dberr_t err = os_file_write( - request, name, log_file, buf, 0, - LOG_FILE_HDR_SIZE + OS_FILE_LOG_BLOCK_SIZE); - - ut_a(err == DB_SUCCESS); - - os_file_flush(log_file); - os_file_close(log_file); - - ut_free(buf); -} -#endif /* UNIV_HOTBACKUP */ /** Find a doublewrite copy of a page. @param[in] space_id tablespace identifier diff --git a/storage/innobase/mem/mem0mem.cc b/storage/innobase/mem/mem0mem.cc index 41292ccf842..ff793821895 100644 --- a/storage/innobase/mem/mem0mem.cc +++ b/storage/innobase/mem/mem0mem.cc @@ -277,9 +277,7 @@ mem_heap_create_block_func( ulint type) /*!< in: type of heap: MEM_HEAP_DYNAMIC or MEM_HEAP_BUFFER */ { -#ifndef UNIV_HOTBACKUP buf_block_t* buf_block = NULL; -#endif /* !UNIV_HOTBACKUP */ mem_block_t* block; ulint len; @@ -294,7 +292,6 @@ mem_heap_create_block_func( /* In dynamic allocation, calculate the size: block header + data. */ len = MEM_BLOCK_HEADER_SIZE + MEM_SPACE_NEEDED(n); -#ifndef UNIV_HOTBACKUP if (type == MEM_HEAP_DYNAMIC || len < UNIV_PAGE_SIZE / 2) { ut_ad(type == MEM_HEAP_DYNAMIC || n <= MEM_MAX_ALLOC_IN_BUF); @@ -329,11 +326,6 @@ mem_heap_create_block_func( block->buf_block = buf_block; block->free_block = NULL; -#else /* !UNIV_HOTBACKUP */ - len = MEM_BLOCK_HEADER_SIZE + MEM_SPACE_NEEDED(n); - block = ut_malloc_nokey(len); - ut_ad(block); -#endif /* !UNIV_HOTBACKUP */ block->magic_n = MEM_BLOCK_MAGIC_N; ut_d(ut_strlcpy_rev(block->file_name, file_name, @@ -428,11 +420,9 @@ mem_heap_block_free( { ulint type; ulint len; -#ifndef UNIV_HOTBACKUP buf_block_t* buf_block; buf_block = static_cast<buf_block_t*>(block->buf_block); -#endif /* !UNIV_HOTBACKUP */ mem_block_validate(block); @@ -447,7 +437,6 @@ mem_heap_block_free( UNIV_MEM_ASSERT_W(block, len); -#ifndef UNIV_HOTBACKUP if (type == MEM_HEAP_DYNAMIC || len < UNIV_PAGE_SIZE / 2) { ut_ad(!buf_block); @@ -457,12 +446,8 @@ mem_heap_block_free( buf_block_free(buf_block); } -#else /* !UNIV_HOTBACKUP */ - ut_free(block); -#endif /* !UNIV_HOTBACKUP */ } -#ifndef UNIV_HOTBACKUP /******************************************************************//** Frees the free_block field from a memory heap. */ void @@ -477,4 +462,3 @@ mem_heap_free_block_free( heap->free_block = NULL; } } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/mtr/mtr0log.cc b/storage/innobase/mtr/mtr0log.cc index d653b3851c3..a63ad40a3b0 100644 --- a/storage/innobase/mtr/mtr0log.cc +++ b/storage/innobase/mtr/mtr0log.cc @@ -34,9 +34,7 @@ Created 12/7/1995 Heikki Tuuri #include "log0recv.h" #include "page0page.h" #include "buf0dblwr.h" - -#ifndef UNIV_HOTBACKUP -# include "dict0boot.h" +#include "dict0boot.h" /********************************************************//** Catenates n bytes to the mtr log. */ @@ -85,7 +83,6 @@ mlog_write_initial_log_record( mlog_close(mtr, log_ptr); } -#endif /* !UNIV_HOTBACKUP */ /********************************************************//** Parses an initial log record written by mlog_write_initial_log_record. @@ -307,7 +304,6 @@ mlog_write_ull( } } -#ifndef UNIV_HOTBACKUP /********************************************************//** Writes a string to a file page buffered in the buffer pool. Writes the corresponding log record to the mini-transaction log. */ @@ -362,7 +358,6 @@ mlog_log_string( mlog_catenate_string(mtr, ptr, len); } -#endif /* !UNIV_HOTBACKUP */ /********************************************************//** Parses a log record written by mlog_write_string. @@ -414,7 +409,6 @@ mlog_parse_string( return(ptr + len); } -#ifndef UNIV_HOTBACKUP /********************************************************//** Opens a buffer for mlog, writes the initial log record and, if needed, the field lengths of an index. @@ -534,7 +528,6 @@ mlog_open_and_write_index( } return(log_ptr); } -#endif /* !UNIV_HOTBACKUP */ /********************************************************//** Parses a log record written by mlog_open_and_write_index. diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index e8ed8adb483..418cd7042c6 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -120,6 +121,8 @@ struct FindPage FindPage(const void* ptr, ulint flags) : m_ptr(ptr), m_flags(flags), m_slot(NULL) { + /* There must be some flags to look for. */ + ut_ad(flags); /* We can only look for page-related flags. */ ut_ad(!(flags & ~(MTR_MEMO_PAGE_S_FIX | MTR_MEMO_PAGE_X_FIX @@ -148,6 +151,11 @@ struct FindPage return(true); } + ut_ad(!(m_flags & (MTR_MEMO_PAGE_S_FIX + | MTR_MEMO_PAGE_SX_FIX + | MTR_MEMO_PAGE_X_FIX)) + || rw_lock_own_flagged(&block->lock, m_flags)); + m_slot = slot; return(false); } @@ -358,16 +366,6 @@ struct ReleaseBlocks { || slot->type == MTR_MEMO_PAGE_SX_FIX) { add_dirty_page_to_flush_list(slot); - - } else if (slot->type == MTR_MEMO_BUF_FIX) { - - buf_block_t* block; - block = reinterpret_cast<buf_block_t*>( - slot->object); - if (block->made_dirty_with_no_latch) { - add_dirty_page_to_flush_list(slot); - block->made_dirty_with_no_latch = false; - } } } @@ -1019,14 +1017,30 @@ mtr_t::release_free_extents(ulint n_reserved) @return true if contains */ bool mtr_t::memo_contains( - mtr_buf_t* memo, - const void* object, - ulint type) + const mtr_buf_t* memo, + const void* object, + ulint type) { Find find(object, type); Iterate<Find> iterator(find); - return(!memo->for_each_block_in_reverse(iterator)); + if (memo->for_each_block_in_reverse(iterator)) { + return(false); + } + + switch (type) { + case MTR_MEMO_X_LOCK: + ut_ad(rw_lock_own((rw_lock_t*) object, RW_LOCK_X)); + break; + case MTR_MEMO_SX_LOCK: + ut_ad(rw_lock_own((rw_lock_t*) object, RW_LOCK_SX)); + break; + case MTR_MEMO_S_LOCK: + ut_ad(rw_lock_own((rw_lock_t*) object, RW_LOCK_S)); + break; + } + + return(true); } /** Debug check for flags */ @@ -1036,20 +1050,56 @@ struct FlaggedCheck { m_ptr(ptr), m_flags(flags) { - // Do nothing + /* There must be some flags to look for. */ + ut_ad(flags); + /* Look for rw-lock-related and page-related flags. */ + ut_ad(!(flags & ~(MTR_MEMO_PAGE_S_FIX + | MTR_MEMO_PAGE_X_FIX + | MTR_MEMO_PAGE_SX_FIX + | MTR_MEMO_BUF_FIX + | MTR_MEMO_MODIFY + | MTR_MEMO_X_LOCK + | MTR_MEMO_SX_LOCK + | MTR_MEMO_S_LOCK))); + /* Either some rw-lock-related or page-related flags + must be specified, but not both at the same time. */ + ut_ad(!(flags & (MTR_MEMO_PAGE_S_FIX + | MTR_MEMO_PAGE_X_FIX + | MTR_MEMO_PAGE_SX_FIX + | MTR_MEMO_BUF_FIX + | MTR_MEMO_MODIFY)) + == !!(flags & (MTR_MEMO_X_LOCK + | MTR_MEMO_SX_LOCK + | MTR_MEMO_S_LOCK))); } + /** Visit a memo entry. + @param[in] slot memo entry to visit + @retval false if m_ptr was found + @retval true if the iteration should continue */ bool operator()(const mtr_memo_slot_t* slot) const { - if (m_ptr == slot->object && (m_flags & slot->type)) { - return(false); + if (m_ptr != slot->object || !(m_flags & slot->type)) { + return(true); } - return(true); + if (ulint flags = m_flags & (MTR_MEMO_PAGE_S_FIX + | MTR_MEMO_PAGE_SX_FIX + | MTR_MEMO_PAGE_X_FIX)) { + rw_lock_t* lock = &static_cast<buf_block_t*>( + const_cast<void*>(m_ptr))->lock; + ut_ad(rw_lock_own_flagged(lock, flags)); + } else { + rw_lock_t* lock = static_cast<rw_lock_t*>( + const_cast<void*>(m_ptr)); + ut_ad(rw_lock_own_flagged(lock, m_flags >> 5)); + } + + return(false); } - const void* m_ptr; - ulint m_flags; + const void*const m_ptr; + const ulint m_flags; }; /** Check if memo contains the given item. diff --git a/storage/innobase/mysql-test/storage_engine/define_engine.inc b/storage/innobase/mysql-test/storage_engine/define_engine.inc index 77e384d2351..7d7b0c7407a 100644 --- a/storage/innobase/mysql-test/storage_engine/define_engine.inc +++ b/storage/innobase/mysql-test/storage_engine/define_engine.inc @@ -41,9 +41,5 @@ let $default_char_type = CHAR(8); # e.g. creation of an additional schema or table, etc. # The cleanup part should be defined in cleanup_engine.inc -CALL mtr.add_suppression("InnoDB: Resizing redo log from .* to .* pages, LSN=.*"); -CALL mtr.add_suppression("InnoDB: Starting to delete and rewrite log files."); -CALL mtr.add_suppression("InnoDB: New log files created, LSN=.*"); - --enable_query_log --enable_result_log diff --git a/storage/innobase/mysql-test/storage_engine/tbl_opt_key_block_size.opt b/storage/innobase/mysql-test/storage_engine/tbl_opt_key_block_size.opt deleted file mode 100644 index 7cd737b2b87..00000000000 --- a/storage/innobase/mysql-test/storage_engine/tbl_opt_key_block_size.opt +++ /dev/null @@ -1,3 +0,0 @@ ---innodb-file-per-table=1 ---innodb-file-format=Barracuda - diff --git a/storage/innobase/mysql-test/storage_engine/tbl_opt_row_format.opt b/storage/innobase/mysql-test/storage_engine/tbl_opt_row_format.opt deleted file mode 100644 index 7cd737b2b87..00000000000 --- a/storage/innobase/mysql-test/storage_engine/tbl_opt_row_format.opt +++ /dev/null @@ -1,3 +0,0 @@ ---innodb-file-per-table=1 ---innodb-file-format=Barracuda - diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index de9b347b4ec..270bfa2887e 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Percona Inc.. Those modifications are @@ -44,6 +44,11 @@ Created 10/21/1995 Heikki Tuuri #include "os0file.ic" #endif +#ifdef UNIV_LINUX +#include <sys/types.h> +#include <sys/stat.h> +#endif + #include "srv0srv.h" #include "srv0start.h" #include "fil0fil.h" @@ -54,17 +59,8 @@ Created 10/21/1995 Heikki Tuuri #ifdef HAVE_LINUX_UNISTD_H #include "unistd.h" #endif -#ifndef UNIV_HOTBACKUP -# include "os0event.h" -# include "os0thread.h" -#else /* !UNIV_HOTBACKUP */ -# ifdef _WIN32 -/* Add includes for the _stat() call to compile on Windows */ -# include <sys/types.h> -# include <sys/stat.h> -# include <errno.h> -# endif /* _WIN32 */ -#endif /* !UNIV_HOTBACKUP */ +#include "os0event.h" +#include "os0thread.h" #include <vector> @@ -77,12 +73,6 @@ Created 10/21/1995 Heikki Tuuri # include <linux/falloc.h> #endif /* HAVE_FALLOC_PUNCH_HOLE_AND_KEEP_SIZE */ -#ifdef HAVE_LZ4 -#include <lz4.h> -#endif - -#include <zlib.h> - #ifdef UNIV_DEBUG /** Set when InnoDB has invoked exit(). */ bool innodb_calling_exit; @@ -103,17 +93,8 @@ bool innodb_calling_exit; #include <linux/falloc.h> #endif -#ifdef HAVE_FALLOC_PUNCH_HOLE_AND_KEEP_SIZE -# include <fcntl.h> -# include <linux/falloc.h> -#endif /* HAVE_FALLOC_PUNCH_HOLE_AND_KEEP_SIZE */ - -#ifdef HAVE_LZO -#include "lzo/lzo1x.h" -#endif - -#ifdef HAVE_SNAPPY -#include "snappy-c.h" +#ifdef _WIN32 +#include <winioctl.h> #endif /** Insert buffer segment id */ @@ -125,30 +106,6 @@ static const ulint IO_LOG_SEGMENT = 1; /** Number of retries for partial I/O's */ static const ulint NUM_RETRIES_ON_PARTIAL_IO = 10; -/** Blocks for doing IO, used in the transparent compression -and encryption code. */ -struct Block { - /** Default constructor */ - Block() : m_ptr(), m_in_use() { } - - byte* m_ptr; - - byte pad[CACHE_LINE_SIZE - sizeof(ulint)]; - int32 m_in_use; -}; - -/** For storing the allocated blocks */ -typedef std::vector<Block> Blocks; - -/** Block collection */ -static Blocks* block_cache; - -/** Number of blocks to allocate for sync read/writes */ -static const size_t MAX_BLOCKS = 128; - -/** Block buffer size */ -#define BUFFER_BLOCK_SIZE ((ulint)(UNIV_PAGE_SIZE * 1.3)) - /* This specifies the file permissions InnoDB uses when it creates files in Unix; the value of os_innodb_umask is initialized in ha_innodb.cc to my_umask */ @@ -168,8 +125,6 @@ static DWORD fls_sync_io = FLS_OUT_OF_INDEXES; #define IOCP_SHUTDOWN_KEY (ULONG_PTR)-1 #endif /* _WIN32 */ -#ifndef UNIV_HOTBACKUP - /** In simulated aio, merge at most this many consecutive i/os */ static const ulint OS_AIO_MERGE_N_CONSECUTIVE = 64; @@ -268,8 +223,6 @@ struct Slot { /** buffer used in i/o */ byte* buf; - ulint is_log; /*!< 1 if OS_FILE_LOG or 0 */ - ulint page_size; /*!< UNIV_PAGE_SIZE or zip_size */ /** Buffer pointer used for actual IO. We advance this when partial IO is required and not buf */ @@ -338,19 +291,6 @@ struct Slot { /** Length of the block before it was compressed */ uint32 original_len; - /** Buffer block for compressed pages or encrypted pages */ - Block* buf_block; - - /** Unaligned buffer for compressed pages */ - byte* compressed_ptr; - - /** Compressed data page, aligned and derived from compressed_ptr */ - byte* compressed_page; - - /** true, if we shouldn't punch a hole after writing the page */ - bool skip_punch_hole; - - ulint* write_size; }; /** The asynchronous i/o array structure */ @@ -392,8 +332,7 @@ public: const char* name, void* buf, os_offset_t offset, - ulint len, - ulint* write_size) + ulint len) MY_ATTRIBUTE((warn_unused_result)); /** @return number of reserved slots */ @@ -530,8 +469,6 @@ public: must call os_aio_simulated_wake_handler_threads later to ensure the threads are not left sleeping! */ static void simulated_put_read_threads_to_sleep(); - - #endif /* _WIN32 */ /** Create an instance using new(std::nothrow) @@ -677,11 +614,13 @@ private: ulint m_n_segments; /** The event which is set to the signaled state when - there is space in the aio outside the ibuf segment */ + there is space in the aio outside the ibuf segment; + os_event_set() and os_event_reset() are protected by AIO::m_mutex */ os_event_t m_not_full; /** The event which is set to the signaled state when - there are no pending i/os in this array */ + there are no pending i/os in this array; + os_event_set() and os_event_reset() are protected by AIO::m_mutex */ os_event_t m_is_empty; /** Number of reserved slots in the AIO array outside @@ -741,7 +680,7 @@ static const int OS_AIO_IO_SETUP_RETRY_ATTEMPTS = 5; #endif /* LINUX_NATIVE_AIO */ /** Array of events used in simulated AIO */ -static os_event_t* os_aio_segment_wait_events = NULL; +static os_event_t* os_aio_segment_wait_events; /** Number of asynchronous I/O segments. Set by os_aio_init(). */ static ulint os_aio_n_segments = ULINT_UNDEFINED; @@ -749,7 +688,6 @@ static ulint os_aio_n_segments = ULINT_UNDEFINED; /** If the following is true, read i/o handler threads try to wait until a batch of new read requests have been posted */ static bool os_aio_recommend_sleep_for_read_threads = false; -#endif /* !UNIV_HOTBACKUP */ ulint os_n_file_reads = 0; ulint os_bytes_read_since_printout = 0; @@ -802,26 +740,6 @@ os_file_handle_error_no_exit( const char* operation, bool silent); -/** Decompress after a read and punch a hole in the file if it was a write -@param[in] type IO context -@param[in] fh Open file handle -@param[in,out] buf Buffer to transform -@param[in,out] scratch Scratch area for read decompression -@param[in] src_len Length of the buffer before compression -@param[in] len Compressed buffer length for write and size - of buf len for read -@return DB_SUCCESS or error code */ -static -dberr_t -os_file_io_complete( - const IORequest&type, - os_file_t fh, - byte* buf, - byte* scratch, - ulint src_len, - ulint offset, - ulint len); - /** Does simulated AIO. This function should be called by an i/o-handler thread. @@ -844,6 +762,142 @@ os_aio_simulated_handler( void** m2, IORequest* type); +#ifdef _WIN32 +static HANDLE win_get_syncio_event(); +#endif + +#ifdef _WIN32 +/** + Wrapper around Windows DeviceIoControl() function. + + Works synchronously, also in case for handle opened + for async access (i.e with FILE_FLAG_OVERLAPPED). + + Accepts the same parameters as DeviceIoControl(),except + last parameter (OVERLAPPED). +*/ +static +BOOL +os_win32_device_io_control( + HANDLE handle, + DWORD code, + LPVOID inbuf, + DWORD inbuf_size, + LPVOID outbuf, + DWORD outbuf_size, + LPDWORD bytes_returned +) +{ + OVERLAPPED overlapped = { 0 }; + overlapped.hEvent = win_get_syncio_event(); + BOOL result = DeviceIoControl(handle, code, inbuf, inbuf_size, outbuf, + outbuf_size, bytes_returned, &overlapped); + + if (!result && (GetLastError() == ERROR_IO_PENDING)) { + /* Wait for async io to complete */ + result = GetOverlappedResult(handle, &overlapped, bytes_returned, TRUE); + } + + return result; +} + +#endif + +/***********************************************************************//** +Try to get number of bytes per sector from file system. +@return file block size */ +UNIV_INTERN +ulint +os_file_get_block_size( +/*===================*/ + os_file_t file, /*!< in: handle to a file */ + const char* name) /*!< in: file name */ +{ + ulint fblock_size = 512; + +#if defined(UNIV_LINUX) + struct stat local_stat; + int err; + + err = fstat((int)file, &local_stat); + + if (err != 0) { + os_file_handle_error_no_exit(name, "fstat()", FALSE); + } else { + fblock_size = local_stat.st_blksize; + } +#endif /* UNIV_LINUX */ +#ifdef _WIN32 + + fblock_size = 0; + + // Open volume for this file, find out it "physical bytes per sector" + + HANDLE volume_handle = INVALID_HANDLE_VALUE; + char volume[MAX_PATH + 4]="\\\\.\\"; // Special prefix required for volume names. + if (!GetVolumePathName(name , volume + 4, MAX_PATH)) { + os_file_handle_error_no_exit(name, + "GetVolumePathName()", FALSE); + goto end; + } + + size_t len = strlen(volume); + if (volume[len - 1] == '\\') { + // Trim trailing backslash from volume name. + volume[len - 1] = 0; + } + + volume_handle = CreateFile(volume, FILE_READ_ATTRIBUTES, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + 0, OPEN_EXISTING, 0, 0); + + if (volume_handle == INVALID_HANDLE_VALUE) { + os_file_handle_error_no_exit(volume, + "CreateFile()", FALSE); + goto end; + } + + DWORD tmp; + STORAGE_ACCESS_ALIGNMENT_DESCRIPTOR disk_alignment; + + STORAGE_PROPERTY_QUERY storage_query; + memset(&storage_query, 0, sizeof(storage_query)); + storage_query.PropertyId = StorageAccessAlignmentProperty; + storage_query.QueryType = PropertyStandardQuery; + + BOOL result = os_win32_device_io_control(volume_handle, + IOCTL_STORAGE_QUERY_PROPERTY, + &storage_query, + sizeof(storage_query), + &disk_alignment, + sizeof(disk_alignment), + &tmp); + + CloseHandle(volume_handle); + + if (!result) { + os_file_handle_error_no_exit(volume, + "DeviceIoControl(IOCTL_STORAGE_QUERY_PROPERTY)", FALSE); + goto end; + } + + fblock_size = disk_alignment.BytesPerPhysicalSector; + +end: +#endif /* _WIN32 */ + + /* Currently we support file block size up to 4Kb */ + if (fblock_size > 4096 || fblock_size < 512) { + if (fblock_size < 512) { + fblock_size = 512; + } else { + fblock_size = 4096; + } + } + + return fblock_size; +} + #ifdef WIN_ASYNC_IO /** This function is only used in Windows asynchronous i/o. Waits for an aio operation to complete. This function is used to wait the @@ -878,166 +932,12 @@ os_aio_windows_handler( IORequest* type); #endif /* WIN_ASYNC_IO */ -#ifdef MYSQL_COMPRESSION -/** Allocate a page for sync IO -@return pointer to page */ -static -Block* -os_alloc_block() -{ - size_t pos; - Blocks& blocks = *block_cache; - size_t i = static_cast<size_t>(my_timer_cycles()); - const size_t size = blocks.size(); - ulint retry = 0; - Block* block; - - DBUG_EXECUTE_IF("os_block_cache_busy", retry = MAX_BLOCKS * 3;); - - for (;;) { - - /* After go through the block cache for 3 times, - allocate a new temporary block. */ - if (retry == MAX_BLOCKS * 3) { - byte* ptr; - - ptr = static_cast<byte*>( - ut_malloc_nokey(sizeof(*block) - + BUFFER_BLOCK_SIZE)); - - block = new (ptr) Block(); - block->m_ptr = static_cast<byte*>( - ptr + sizeof(*block)); - block->m_in_use = 1; - - break; - } - - pos = i++ % size; - - if (my_atomic_fas32_explicit(&blocks[pos].m_in_use, 1, - MY_MEMORY_ORDER_ACQUIRE) == 0) { - block = &blocks[pos]; - break; - } - - os_thread_yield(); - - ++retry; - } - - ut_a(block->m_in_use != 0); - - return(block); -} - -/** Free a page after sync IO -@param[in,own] block The block to free/release */ -static -void -os_free_block(Block* block) -{ - ut_ad(block->m_in_use == 1); - - my_atomic_store32_explicit(&block->m_in_use, 0, MY_MEMORY_ORDER_RELEASE); - - /* When this block is not in the block cache, and it's - a temporary block, we need to free it directly. */ - if (std::less<Block*>()(block, &block_cache->front()) - || std::greater<Block*>()(block, &block_cache->back())) { - ut_free(block); - } -} -#endif /* MYSQL_COMPRESSION */ - /** Generic AIO Handler methods. Currently handles IO post processing. */ class AIOHandler { public: /** Do any post processing after a read/write @return DB_SUCCESS or error code. */ static dberr_t post_io_processing(Slot* slot); - - /** Decompress after a read and punch a hole in the file if - it was a write */ - static dberr_t io_complete(const Slot* slot) - { - ut_a(slot->offset > 0); - ut_a(slot->type.is_read() || !slot->skip_punch_hole); - - return(os_file_io_complete( - slot->type, slot->file, slot->buf, - slot->compressed_page, slot->original_len, - static_cast<ulint>(slot->offset), - slot->len)); - } - -private: - /** Check whether the page was encrypted. - @param[in] slot The slot that contains the IO request - @return true if it was an encyrpted page */ - static bool is_encrypted_page(const Slot* slot) - { -#ifdef MYSQL_ENCRYPTION - return(Encryption::is_encrypted_page(slot->buf)); -#else - return (false); -#endif - } - - /** Check whether the page was compressed. - @param[in] slot The slot that contains the IO request - @return true if it was a compressed page */ - static bool is_compressed_page(const Slot* slot) - { - const byte* src = slot->buf; - - ulint page_type = mach_read_from_2(src + FIL_PAGE_TYPE); - - return(page_type == FIL_PAGE_COMPRESSED); - } - - /** Get the compressed page size. - @param[in] slot The slot that contains the IO request - @return number of bytes to read for a successful decompress */ - static ulint compressed_page_size(const Slot* slot) - { - ut_ad(slot->type.is_read()); - ut_ad(is_compressed_page(slot)); - - ulint size; - const byte* src = slot->buf; - - size = mach_read_from_2(src + FIL_PAGE_COMPRESS_SIZE_V1); - - return(size + FIL_PAGE_DATA); - } - - /** Check if the page contents can be decompressed. - @param[in] slot The slot that contains the IO request - @return true if the data read has all the compressed data */ - static bool can_decompress(const Slot* slot) - { - ut_ad(slot->type.is_read()); - ut_ad(is_compressed_page(slot)); - - ulint version; - const byte* src = slot->buf; - - version = mach_read_from_1(src + FIL_PAGE_VERSION); - - ut_a(version == 1); - - /* Includes the page header size too */ - ulint size = compressed_page_size(slot); - - return(size <= (slot->ptr - slot->buf) + (ulint) slot->n_bytes); - } - - /** Check if we need to read some more data. - @param[in] slot The slot that contains the IO request - @param[in] n_bytes Total bytes read so far - @return DB_SUCCESS or error code */ - static dberr_t check_read(Slot* slot, ulint n_bytes); }; /** Helper class for doing synchronous file IO. Currently, the objective @@ -1103,171 +1003,17 @@ private: os_offset_t m_offset; }; -/** If it is a compressed page return the compressed page data + footer size -@param[in] buf Buffer to check, must include header + 10 bytes -@return ULINT_UNDEFINED if the page is not a compressed page or length - of the compressed data (including footer) if it is a compressed page */ -ulint -os_file_compressed_page_size(const byte* buf) -{ - ulint type = mach_read_from_2(buf + FIL_PAGE_TYPE); - - if (type == FIL_PAGE_COMPRESSED) { - ulint version = mach_read_from_1(buf + FIL_PAGE_VERSION); - ut_a(version == 1); - return(mach_read_from_2(buf + FIL_PAGE_COMPRESS_SIZE_V1)); - } - - return(ULINT_UNDEFINED); -} - -/** If it is a compressed page return the original page data + footer size -@param[in] buf Buffer to check, must include header + 10 bytes -@return ULINT_UNDEFINED if the page is not a compressed page or length - of the original data + footer if it is a compressed page */ -ulint -os_file_original_page_size(const byte* buf) -{ - ulint type = mach_read_from_2(buf + FIL_PAGE_TYPE); - - if (type == FIL_PAGE_COMPRESSED) { - - ulint version = mach_read_from_1(buf + FIL_PAGE_VERSION); - ut_a(version == 1); - - return(mach_read_from_2(buf + FIL_PAGE_ORIGINAL_SIZE_V1)); - } - - return(ULINT_UNDEFINED); -} - -/** Check if we need to read some more data. -@param[in] slot The slot that contains the IO request -@param[in] n_bytes Total bytes read so far -@return DB_SUCCESS or error code */ -dberr_t -AIOHandler::check_read(Slot* slot, ulint n_bytes) -{ - dberr_t err=DB_SUCCESS; - - ut_ad(slot->type.is_read()); - ut_ad(slot->original_len > slot->len); - - if (is_compressed_page(slot)) { - - if (can_decompress(slot)) { - - ut_a(slot->offset > 0); - - slot->len = slot->original_len; -#ifdef _WIN32 - slot->n_bytes = static_cast<DWORD>(n_bytes); -#else - slot->n_bytes = static_cast<ulint>(n_bytes); -#endif /* _WIN32 */ - - err = io_complete(slot); - ut_a(err == DB_SUCCESS); - - } else { - /* Read the next block in */ - ut_ad(compressed_page_size(slot) >= n_bytes); - - err = DB_FAIL; - } - } else if (is_encrypted_page(slot)) { - ut_a(slot->offset > 0); - - slot->len = slot->original_len; -#ifdef _WIN32 - slot->n_bytes = static_cast<DWORD>(n_bytes); -#else - slot->n_bytes = static_cast<ulint>(n_bytes); -#endif /* _WIN32 */ - - err = io_complete(slot); - ut_a(err == DB_SUCCESS); - - } else { - err = DB_FAIL; - } - -#ifdef MYSQL_COMPRESSION - if (slot->buf_block != NULL) { - os_free_block(slot->buf_block); - slot->buf_block = NULL; - } -#endif - return(err); -} - /** Do any post processing after a read/write @return DB_SUCCESS or error code. */ dberr_t AIOHandler::post_io_processing(Slot* slot) { - dberr_t err=DB_SUCCESS; - ut_ad(slot->is_reserved); /* Total bytes read so far */ ulint n_bytes = (slot->ptr - slot->buf) + slot->n_bytes; - /* Compressed writes can be smaller than the original length. - Therefore they can be processed without further IO. */ - if (n_bytes == slot->original_len - || (slot->type.is_write() - && slot->type.is_compressed() - && slot->len == static_cast<ulint>(slot->n_bytes))) { - -#ifdef MYSQL_COMPRESSION - if (!slot->type.is_log() - && (is_compressed_page(slot) - || is_encrypted_page(slot))) { - - ut_a(slot->offset > 0); - - if (slot->type.is_read()) { - slot->len = slot->original_len; - } - - /* The punch hole has been done on collect() */ - - if (slot->type.is_read()) { - err = io_complete(slot); - } else { - err = DB_SUCCESS; - } - - ut_ad(err == DB_SUCCESS - || err == DB_UNSUPPORTED - || err == DB_CORRUPTION - || err == DB_IO_DECOMPRESS_FAIL); - } else { - - err = DB_SUCCESS; - } - - if (slot->buf_block != NULL) { - os_free_block(slot->buf_block); - slot->buf_block = NULL; - } -#endif /* MYSQL_COMPRESSION */ - } else if ((ulint) slot->n_bytes == (ulint) slot->len) { - - /* It *must* be a partial read. */ - ut_ad(slot->len < slot->original_len); - - /* Has to be a read request, if it is less than - the original length. */ - ut_ad(slot->type.is_read()); - err = check_read(slot, n_bytes); - - } else { - err = DB_FAIL; - } - - return(err); + return(n_bytes == slot->original_len ? DB_SUCCESS : DB_FAIL); } /** Count the number of free slots @@ -1303,157 +1049,7 @@ AIO::pending_io_count() const return(reserved); } -#ifdef MYSQL_COMPRESSION -/** Compress a data page -#param[in] block_size File system block size -@param[in] src Source contents to compress -@param[in] src_len Length in bytes of the source -@param[out] dst Compressed page contents -@param[out] dst_len Length in bytes of dst contents -@return buffer data, dst_len will have the length of the data */ -static -byte* -os_file_compress_page( - Compression compression, - ulint block_size, - byte* src, - ulint src_len, - byte* dst, - ulint* dst_len) -{ - ulint len = 0; - ulint compression_level = page_zip_level; - ulint page_type = mach_read_from_2(src + FIL_PAGE_TYPE); - - /* The page size must be a multiple of the OS punch hole size. */ - ut_ad(!(src_len % block_size)); - - /* Shouldn't compress an already compressed page. */ - ut_ad(page_type != FIL_PAGE_COMPRESSED); - - /* The page must be at least twice as large as the file system - block size if we are to save any space. Ignore R-Tree pages for now, - they repurpose the same 8 bytes in the page header. No point in - compressing if the file system block size >= our page size. */ - - if (page_type == FIL_PAGE_RTREE - || block_size == ULINT_UNDEFINED - || compression.m_type == Compression::NONE - || src_len < block_size * 2) { - - *dst_len = src_len; - - return(src); - } - - /* Leave the header alone when compressing. */ - ut_ad(block_size >= FIL_PAGE_DATA * 2); - - ut_ad(src_len > FIL_PAGE_DATA + block_size); - - /* Must compress to <= N-1 FS blocks. */ - ulint out_len = src_len - (FIL_PAGE_DATA + block_size); - - /* This is the original data page size - the page header. */ - ulint content_len = src_len - FIL_PAGE_DATA; - - ut_ad(out_len >= block_size - FIL_PAGE_DATA); - ut_ad(out_len <= src_len - (block_size + FIL_PAGE_DATA)); - - /* Only compress the data + trailer, leave the header alone */ - - switch (compression.m_type) { - case Compression::NONE: - ut_error; - - case Compression::ZLIB: { - - uLongf zlen = static_cast<uLongf>(out_len); - - if (compress2( - dst + FIL_PAGE_DATA, - &zlen, - src + FIL_PAGE_DATA, - static_cast<uLong>(content_len), - static_cast<int>(compression_level)) != Z_OK) { - - *dst_len = src_len; - - return(src); - } - - len = static_cast<ulint>(zlen); - - break; - } - -#ifdef HAVE_LZ4 - case Compression::LZ4: - - len = LZ4_compress_limitedOutput( - reinterpret_cast<char*>(src) + FIL_PAGE_DATA, - reinterpret_cast<char*>(dst) + FIL_PAGE_DATA, - static_cast<int>(content_len), - static_cast<int>(out_len)); - - ut_a(len <= src_len - FIL_PAGE_DATA); - - if (len == 0 || len >= out_len) { - - *dst_len = src_len; - - return(src); - } - - break; -#endif - - default: - *dst_len = src_len; - return(src); - } - - ut_a(len <= out_len); - - ut_ad(memcmp(src + FIL_PAGE_LSN + 4, - src + src_len - FIL_PAGE_END_LSN_OLD_CHKSUM + 4, 4) - == 0); - - /* Copy the header as is. */ - memmove(dst, src, FIL_PAGE_DATA); - - /* Add compression control information. Required for decompressing. */ - mach_write_to_2(dst + FIL_PAGE_TYPE, FIL_PAGE_COMPRESSED); - - mach_write_to_1(dst + FIL_PAGE_VERSION, 1); - - mach_write_to_1(dst + FIL_PAGE_ALGORITHM_V1, compression.m_type); - - mach_write_to_2(dst + FIL_PAGE_ORIGINAL_TYPE_V1, page_type); - - mach_write_to_2(dst + FIL_PAGE_ORIGINAL_SIZE_V1, content_len); - - mach_write_to_2(dst + FIL_PAGE_COMPRESS_SIZE_V1, len); - - /* Round to the next full block size */ - - len += FIL_PAGE_DATA; - - *dst_len = ut_calc_align(len, block_size); - - ut_ad(*dst_len >= len && *dst_len <= out_len + FIL_PAGE_DATA); - - /* Clear out the unused portion of the page. */ - if (len % block_size) { - memset(dst + len, 0x0, block_size - (len % block_size)); - } - - return(dst); -} -#endif /* MYSQL_COMPRESSION */ - #ifdef UNIV_DEBUG -# ifndef UNIV_HOTBACKUP /** Validates the consistency the aio system some of the time. @return true if ok or the check was skipped */ bool @@ -1479,16 +1075,12 @@ os_aio_validate_skip() os_aio_validate_count = OS_AIO_VALIDATE_SKIP; return(os_aio_validate()); } -# endif /* !UNIV_HOTBACKUP */ #endif /* UNIV_DEBUG */ #undef USE_FILE_LOCK -#define USE_FILE_LOCK -#if defined(UNIV_HOTBACKUP) || defined(_WIN32) -/* InnoDB Hot Backup does not lock the data files. - * On Windows, mandatory locking is used. - */ -# undef USE_FILE_LOCK +#ifndef _WIN32 +/* On Windows, mandatory locking is used */ +# define USE_FILE_LOCK #endif #ifdef USE_FILE_LOCK /** Obtain an exclusive lock on a file. @@ -1528,8 +1120,6 @@ os_file_lock( } #endif /* USE_FILE_LOCK */ -#ifndef UNIV_HOTBACKUP - /** Calculates local segment number and aio array from global segment number. @param[out] array aio wait array @param[in] segment global segment number @@ -1637,6 +1227,7 @@ os_file_create_tmpfile( const char* path) { FILE* file = NULL; + WAIT_ALLOW_WRITES(); int fd = innobase_mysql_tmpfile(path); if (fd >= 0) { @@ -1678,89 +1269,6 @@ os_file_read_string( } } -/** Decompress after a read and punch a hole in the file if it was a write -@param[in] type IO context -@param[in] fh Open file handle -@param[in,out] buf Buffer to transform -@param[in,out] scratch Scratch area for read decompression -@param[in] src_len Length of the buffer before compression -@param[in] len Used buffer length for write and output - buf len for read -@return DB_SUCCESS or error code */ -static -dberr_t -os_file_io_complete( - const IORequest&type, - os_file_t fh, - byte* buf, - byte* scratch, - ulint src_len, - ulint offset, - ulint len) -{ -#ifdef MYSQL_ENCRYPTION - /* We never compress/decompress the first page */ - ut_a(offset > 0); - ut_ad(type.validate()); - - if (!type.is_compression_enabled()) { - - return(DB_SUCCESS); - - } else if (type.is_read()) { - dberr_t ret = DB_SUCCESS; - Encryption encryption(type.encryption_algorithm()); - - ut_ad(!type.is_log()); - - ret = encryption.decrypt(type, buf, src_len, scratch, len); - if (ret == DB_SUCCESS) { - return(os_file_decompress_page( - type.is_dblwr_recover(), - buf, scratch, len)); - } else { - return(ret); - } - - } else if (type.punch_hole()) { - - ut_ad(len <= src_len); - ut_ad(!type.is_log()); - ut_ad(type.is_write()); - ut_ad(type.is_compressed()); - - /* Nothing to do. */ - if (len == src_len) { - return(DB_SUCCESS); - } - -#ifdef UNIV_DEBUG - const ulint block_size = type.block_size(); -#endif /* UNIV_DEBUG */ - - /* We don't support multiple page sizes in the server - at the moment. */ - ut_ad(src_len == srv_page_size); - - /* Must be a multiple of the compression unit size. */ - ut_ad((len % block_size) == 0); - ut_ad((offset % block_size) == 0); - - ut_ad(len + block_size <= src_len); - - offset += len; - - return(os_file_punch_hole(fh, offset, src_len - len)); - } - - ut_ad(!type.is_log()); -#endif /* MYSQL_ENCRYPTION */ - - return(DB_SUCCESS); -} - -#endif /* !UNIV_HOTBACKUP */ - /** This function returns a new path name after replacing the basename in an old path with a new basename. The old_path is a full path name including the extension. The tablename is in the normal @@ -2056,132 +1564,6 @@ os_file_create_subdirs_if_needed( return(success ? DB_SUCCESS : DB_ERROR); } -#ifdef MYSQL_COMPRESSION -/** Allocate the buffer for IO on a transparently compressed table. -@param[in] type IO flags -@param[out] buf buffer to read or write -@param[in,out] n number of bytes to read/write, starting from - offset -@return pointer to allocated page, compressed data is written to the offset - that is aligned on UNIV_SECTOR_SIZE of Block.m_ptr */ -static -Block* -os_file_compress_page( - IORequest& type, - void*& buf, - ulint* n) -{ - ut_ad(!type.is_log()); - ut_ad(type.is_write()); - ut_ad(type.is_compressed()); - - ulint n_alloc = *n * 2; - - ut_a(n_alloc <= UNIV_PAGE_SIZE_MAX * 2); -#ifdef HAVE_LZ4 - ut_a(type.compression_algorithm().m_type != Compression::LZ4 - || static_cast<ulint>(LZ4_COMPRESSBOUND(*n)) < n_alloc); -#endif - - Block* ptr = reinterpret_cast<Block*>(ut_malloc_nokey(n_alloc)); - - if (ptr == NULL) { - return(NULL); - } - - ulint old_compressed_len; - ulint compressed_len = *n; - - old_compressed_len = mach_read_from_2( - reinterpret_cast<byte*>(buf) - + FIL_PAGE_COMPRESS_SIZE_V1); - - if (old_compressed_len > 0) { - old_compressed_len = ut_calc_align( - old_compressed_len + FIL_PAGE_DATA, - type.block_size()); - } - - byte* compressed_page; - - compressed_page = static_cast<byte*>( - ut_align(block->m_ptr, UNIV_SECTOR_SIZE)); - - byte* buf_ptr; - - buf_ptr = os_file_compress_page( - type.compression_algorithm(), - type.block_size(), - reinterpret_cast<byte*>(buf), - *n, - compressed_page, - &compressed_len); - - if (buf_ptr != buf) { - /* Set new compressed size to uncompressed page. */ - memcpy(reinterpret_cast<byte*>(buf) + FIL_PAGE_COMPRESS_SIZE_V1, - buf_ptr + FIL_PAGE_COMPRESS_SIZE_V1, 2); - - buf = buf_ptr; - *n = compressed_len; - - if (compressed_len >= old_compressed_len) { - - ut_ad(old_compressed_len <= UNIV_PAGE_SIZE); - - type.clear_punch_hole(); - } - } - - return(block); -} -#endif /* MYSQL_COMPRESSION */ - -#ifdef MYSQL_ENCRYPTION -/** Encrypt a page content when write it to disk. -@param[in] type IO flags -@param[out] buf buffer to read or write -@param[in,out] n number of bytes to read/write, starting from - offset -@return pointer to the encrypted page */ -static -Block* -os_file_encrypt_page( - const IORequest& type, - void*& buf, - ulint* n) -{ - - byte* encrypted_page; - ulint encrypted_len = *n; - byte* buf_ptr; - Encryption encryption(type.encryption_algorithm()); - - ut_ad(!type.is_log()); - ut_ad(type.is_write()); - ut_ad(type.is_encrypted()); - - Block* block = os_alloc_block(); - - encrypted_page = static_cast<byte*>( - ut_align(block->m_ptr, UNIV_SECTOR_SIZE)); - - buf_ptr = encryption.encrypt(type, - reinterpret_cast<byte*>(buf), *n, - encrypted_page, &encrypted_len); - - bool encrypted = buf_ptr != buf; - - if (encrypted) { - - buf = buf_ptr; - *n = encrypted_len; - } - - return(block); -} -#endif /* MYSQL_ENCRYPTION */ - #ifndef _WIN32 /** Do the read/write @@ -2201,7 +1583,6 @@ SyncFileIO::execute(const IORequest& request) return(n_bytes); } - /** Free storage space associated with a section of the file. @param[in] fh Open file handle @param[in] off Starting offset (SEEK_SET) @@ -2224,14 +1605,12 @@ os_file_punch_hole_posix( return(DB_SUCCESS); } - ut_a(ret == -1); - if (errno == ENOTSUP) { return(DB_IO_NO_PUNCH_HOLE); } ib::warn() - << "fallocate(" << fh + << "fallocate(" <<", FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, " << off << ", " << len << ") returned errno: " << errno; @@ -2537,17 +1916,17 @@ LinuxAIOHandler::collect() /* We have not overstepped to next segment. */ ut_a(slot->pos < end_pos); - /* We never compress/decompress the first page */ - + /* Deallocate unused blocks from file system. + This is newer done to page 0 or to log files.*/ if (slot->offset > 0 - && !slot->skip_punch_hole - && slot->type.is_compression_enabled() && !slot->type.is_log() && slot->type.is_write() - && slot->type.is_compressed() && slot->type.punch_hole()) { - slot->err = AIOHandler::io_complete(slot); + slot->err = slot->type.punch_hole( + slot->file, + slot->offset, + static_cast<os_offset_t>(slot->len)); } else { slot->err = DB_SUCCESS; } @@ -2716,16 +2095,7 @@ os_aio_linux_handler( void** m2, IORequest* request) { - LinuxAIOHandler handler(global_segment); - - dberr_t err = handler.poll(m1, m2, request); - - if (err == DB_IO_NO_PUNCH_HOLE) { - fil_no_punch_hole(*m1); - err = DB_SUCCESS; - } - - return(err); + return LinuxAIOHandler(global_segment).poll(m1, m2, request); } /** Dispatch an AIO request to the kernel. @@ -3183,6 +2553,7 @@ os_file_flush_func( { int ret; + WAIT_ALLOW_WRITES(); ret = os_file_fsync_posix(file); if (ret == 0) { @@ -3234,6 +2605,10 @@ os_file_create_simple_func( int create_flag; const char* mode_str = NULL; + if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW) { + WAIT_ALLOW_WRITES(); + } + ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); @@ -3331,26 +2706,6 @@ os_file_create_simple_func( } #endif /* USE_FILE_LOCK */ - /* If we have proper file handle and atomic writes should be used, - try to set atomic writes and if that fails when creating a new - table, produce a error. If atomic writes are used on existing - file, ignore error and use traditional writes for that file */ - /* JAN: TODO: ATOMIC WRITES - if (file != -1 - && (awrites == ATOMIC_WRITES_ON || - (srv_use_atomic_writes && awrites == ATOMIC_WRITES_DEFAULT)) - && !os_file_set_atomic_writes(name, file)) { - if (create_mode == OS_FILE_CREATE) { - fprintf(stderr, "InnoDB: Error: Can't create file using atomic writes\n"); - close(file); - os_file_delete_if_exists_func(name); - *success = FALSE; - file = -1; - } - } - */ - - return(file); } @@ -3369,7 +2724,10 @@ os_file_create_directory( const char* pathname, bool fail_if_exists) { - int rcode = mkdir(pathname, 0770); + int rcode; + + WAIT_ALLOW_WRITES(); + rcode = mkdir(pathname, 0770); if (!(rcode == 0 || (errno == EEXIST && !fail_if_exists))) { /* failure */ @@ -3441,43 +2799,15 @@ os_file_readdir_next_file( int ret; struct stat statinfo; -#ifdef HAVE_READDIR_R - char dirent_buf[sizeof(struct dirent) - + _POSIX_PATH_MAX + 100]; - /* In /mysys/my_lib.c, _POSIX_PATH_MAX + 1 is used as - the max file name len; but in most standards, the - length is NAME_MAX; we add 100 to be even safer */ -#endif /* HAVE_READDIR_R */ - next_file: -#ifdef HAVE_READDIR_R - ret = readdir_r(dir, (struct dirent*) dirent_buf, &ent); - - if (ret != 0) { - - ib::error() - << "Cannot read directory " << dirname - << " error: " << ret; - - return(-1); - } - - if (ent == NULL) { - /* End of directory */ - - return(1); - } - - ut_a(strlen(ent->d_name) < _POSIX_PATH_MAX + 100 - 1); -#else ent = readdir(dir); if (ent == NULL) { return(1); } -#endif /* HAVE_READDIR_R */ + ut_a(strlen(ent->d_name) < OS_FILE_MAX_PATH); if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) { @@ -3703,24 +3033,6 @@ os_file_create_func( } #endif /* USE_FILE_LOCK */ - /* If we have proper file handle and atomic writes should be used, - try to set atomic writes and if that fails when creating a new - table, produce a error. If atomic writes are used on existing - file, ignore error and use traditional writes for that file */ - /* JAN: TODO: ATOMIC WRITES - if (file != -1 && type == OS_DATA_FILE - && (awrites == ATOMIC_WRITES_ON || - (srv_use_atomic_writes && awrites == ATOMIC_WRITES_DEFAULT)) - && !os_file_set_atomic_writes(name, file)) { - if (create_mode == OS_FILE_CREATE) { - fprintf(stderr, "InnoDB: Error: Can't create file using atomic writes\n"); - close(file); - os_file_delete_if_exists_func(name); - *success = FALSE; - file = -1; - } - } - */ return(file); } @@ -3748,6 +3060,10 @@ os_file_create_simple_no_error_handling_func( os_file_t file; int create_flag; + if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW) { + WAIT_ALLOW_WRITES(); + } + ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); @@ -3821,7 +3137,10 @@ os_file_delete_if_exists_func( *exist = true; } - int ret = unlink(name); + int ret; + WAIT_ALLOW_WRITES(); + + ret = unlink(name); if (ret != 0 && errno == ENOENT) { if (exist != NULL) { @@ -3843,7 +3162,10 @@ bool os_file_delete_func( const char* name) { - int ret = unlink(name); + int ret; + WAIT_ALLOW_WRITES(); + + ret = unlink(name); if (ret != 0) { os_file_handle_error_no_exit(name, "delete", FALSE); @@ -3879,7 +3201,10 @@ os_file_rename_func( ut_ad(exists); #endif /* UNIV_DEBUG */ - int ret = rename(oldpath, newpath); + int ret; + WAIT_ALLOW_WRITES(); + + ret = rename(oldpath, newpath); if (ret != 0) { os_file_handle_error_no_exit(oldpath, "rename", FALSE); @@ -4064,31 +3389,10 @@ bool os_file_set_eof( FILE* file) /*!< in: file to be truncated */ { + WAIT_ALLOW_WRITES(); return(!ftruncate(fileno(file), ftell(file))); } -#ifdef UNIV_HOTBACKUP -/** Closes a file handle. -@param[in] file Handle to a file -@return true if success */ -bool -os_file_close_no_error_handling( - os_file_t file) -{ - return(close(file) != -1); -} -#endif /* UNIV_HOTBACKUP */ - -/** This function can be called if one wants to post a batch of reads and -prefers an i/o-handler thread to handle them all at once later. You must -call os_aio_simulated_wake_handler_threads later to ensure the threads -are not left sleeping! */ -void -os_aio_simulated_put_read_threads_to_sleep() -{ - /* No op on non Windows */ -} - #else /* !_WIN32 */ #include <WinIoCtl.h> @@ -4251,10 +3555,21 @@ os_is_sparse_file_supported_win32(const char* filename) DWORD flags; - GetVolumeInformation( + result = GetVolumeInformation( volname, NULL, MAX_PATH, NULL, NULL, &flags, NULL, MAX_PATH); + + if (!result) { + ib::error() + << "os_is_sparse_file_supported: " + << "Failed to get the volume info for: " + << volname + << "- OS error number " << GetLastError(); + + return(false); + } + return(flags & FILE_SUPPORTS_SPARSE_FILES) ? true : false; } @@ -4280,12 +3595,11 @@ os_file_punch_hole_win32( /* If lpOverlapped is NULL, lpBytesReturned cannot be NULL, therefore we pass a dummy parameter. */ DWORD temp; - - BOOL result = DeviceIoControl( + BOOL success = os_win32_device_io_control( fh, FSCTL_SET_ZERO_DATA, &punch, sizeof(punch), - NULL, 0, &temp, NULL); + NULL, 0, &temp); - return(!result ? DB_IO_NO_PUNCH_HOLE : DB_SUCCESS); + return(success ? DB_SUCCESS: DB_IO_NO_PUNCH_HOLE); } /** Check the existence and type of the given file. @@ -4596,9 +3910,9 @@ os_file_create_simple_func( /* This is a best effort use case, if it fails then we will find out when we try and punch the hole. */ - DeviceIoControl( + os_win32_device_io_control( file, FSCTL_SET_SPARSE, NULL, 0, NULL, 0, - &temp, NULL); + &temp); } } while (retry); @@ -4827,6 +4141,10 @@ os_file_create_func( DWORD create_flag; DWORD share_mode = FILE_SHARE_READ; + if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW) { + WAIT_ALLOW_WRITES(); + } + on_error_no_exit = create_mode & OS_FILE_ON_ERROR_NO_EXIT ? true : false; @@ -4875,9 +4193,6 @@ os_file_create_func( DWORD attributes = 0; -#ifdef UNIV_HOTBACKUP - attributes |= FILE_FLAG_NO_BUFFERING; -#else if (purpose == OS_FILE_AIO) { #ifdef WIN_ASYNC_IO @@ -4917,7 +4232,6 @@ os_file_create_func( } #endif /* UNIV_NON_BUFFERED_IO */ -#endif /* UNIV_HOTBACKUP */ DWORD access = GENERIC_READ; if (!read_only) { @@ -4959,9 +4273,9 @@ os_file_create_func( /* This is a best effort use case, if it fails then we will find out when we try and punch the hole. */ - DeviceIoControl( + os_win32_device_io_control( file, FSCTL_SET_SPARSE, NULL, 0, NULL, 0, - &temp, NULL); + &temp); } } while (retry); @@ -5084,8 +4398,9 @@ os_file_delete_if_exists_func( } for (;;) { - /* In Windows, deleting an .ibd file may fail if ibbackup - is copying it */ + /* In Windows, deleting an .ibd file may fail if + the file is being accessed by an external program, + such as a backup tool. */ bool ret = DeleteFile((LPCTSTR) name); @@ -5136,8 +4451,9 @@ os_file_delete_func( ulint count = 0; for (;;) { - /* In Windows, deleting an .ibd file may fail if ibbackup - is copying it */ + /* In Windows, deleting an .ibd file may fail if + the file is being accessed by an external program, + such as a backup tool. */ BOOL ret = DeleteFile((LPCTSTR) name); @@ -5160,8 +4476,8 @@ os_file_delete_func( os_file_get_last_error(true); ib::warn() - << "Cannot delete file '" << name << "'. Are " - << "you running ibbackup to back up the file?"; + << "Cannot delete file '" << name << "'. Is " + << "another program accessing it?"; } /* sleep for a second */ @@ -5396,28 +4712,6 @@ os_file_get_status_win32( } stat_info->block_size = bytesPerSector * sectorsPerCluster; - - /* On Windows the block size is not used as the allocation - unit for sparse files. The underlying infra-structure for - sparse files is based on NTFS compression. The punch hole - is done on a "compression unit". This compression unit - is based on the cluster size. You cannot punch a hole if - the cluster size >= 8K. For smaller sizes the table is - as follows: - - Cluster Size Compression Unit - 512 Bytes 8 KB - 1 KB 16 KB - 2 KB 32 KB - 4 KB 64 KB - - Default NTFS cluster size is 4K, compression unit size of 64K. - Therefore unless the user has created the file system with - a smaller cluster size and used larger page sizes there is - little benefit from compression out of the box. */ - - stat_info->block_size = (stat_info->block_size <= 4096) - ? stat_info->block_size * 16 : ULINT_UNDEFINED; } else { stat_info->type = OS_FILE_TYPE_UNKNOWN; } @@ -5470,18 +4764,6 @@ os_file_set_eof( return(SetEndOfFile(h)); } -#ifdef UNIV_HOTBACKUP -/** Closes a file handle. -@param[in] file Handle to close -@return true if success */ -bool -os_file_close_no_error_handling( - os_file_t file) -{ - return(CloseHandle(file) ? true : false); -} -#endif /* UNIV_HOTBACKUP */ - /** This function can be called if one wants to post a batch of reads and prefers an i/o-handler thread to handle them all at once later. You must call os_aio_simulated_wake_handler_threads later to ensure the threads @@ -5526,30 +4808,6 @@ AIO::simulated_put_read_threads_to_sleep() #endif /* !_WIN32*/ -#ifdef MYSQL_COMPRESSION -/** Validate the type, offset and number of bytes to read * -@param[in] type IO flags -@param[in] offset Offset from start of the file -@param[in] n Number of bytes to read from offset */ -static -void -os_file_check_args(const IORequest& type, os_offset_t offset, ulint n) -{ - ut_ad(type.validate()); - - ut_ad(n > 0); - - /* If off_t is > 4 bytes in size, then we assume we can pass a - 64-bit address */ - off_t offs = static_cast<off_t>(offset); - - if (sizeof(off_t) <= 4 && offset != (os_offset_t) offs) { - - ib::error() << "file write at offset > 4 GB."; - } -} -#endif /* MYSQL_COMPRESSION */ - /** Does a syncronous read or write depending upon the type specified In case of partial reads/writes the function tries NUM_RETRIES_ON_PARTIAL_IO times to read/write the complete data. @@ -5572,44 +4830,8 @@ os_file_io( { ulint original_n = n; IORequest type = in_type; - byte* compressed_page=NULL; ssize_t bytes_returned = 0; -#ifdef MYSQL_COMPRESSION - Block* block=NULL; - if (type.is_compressed()) { - - /* We don't compress the first page of any file. */ - ut_ad(offset > 0); - - block = os_file_compress_page(type, buf, &n); - - compressed_page = static_cast<byte*>( - ut_align(block->m_ptr, UNIV_SECTOR_SIZE)); - - } else { - block = NULL; - compressed_page = NULL; - } -#endif /* MYSQL_COMPRESSION */ - -#ifdef MYSQL_ENCRYPTION - /* We do encryption after compression, since if we do encryption - before compression, the encrypted data will cause compression fail - or low compression rate. */ - if (type.is_encrypted() && type.is_write()) { - /* We don't encrypt the first page of any file. */ - Block* compressed_block = block; - ut_ad(offset > 0); - - block = os_file_encrypt_page(type, buf, &n); - - if (compressed_block != NULL) { - os_free_block(compressed_block); - } - } -#endif /* MYSQL_ENCRYPTION */ - SyncFileIO sync_file_io(file, buf, n, offset); for (ulint i = 0; i < NUM_RETRIES_ON_PARTIAL_IO; ++i) { @@ -5626,23 +4848,16 @@ os_file_io( bytes_returned += n_bytes; if (offset > 0 - && (type.is_compressed() || type.is_read())) { - - *err = os_file_io_complete( - type, file, - reinterpret_cast<byte*>(buf), - compressed_page, original_n, - static_cast<ulint>(offset), n); + && !type.is_log() + && type.is_write() + && type.punch_hole()) { + *err = type.punch_hole(file, + offset, + static_cast<os_offset_t>(n)); } else { - *err = DB_SUCCESS; } -#ifdef MYSQL_COMPRESSION - if (block != NULL) { - os_free_block(block); - } -#endif return(original_n); } @@ -5670,12 +4885,6 @@ os_file_io( sync_file_io.advance(n_bytes); } -#ifdef MYSQL_COMPRESSION - if (block != NULL) { - os_free_block(block); - } -#endif - *err = DB_IO_ERROR; if (!type.is_partial_io_warning_disabled()) { @@ -5713,7 +4922,8 @@ os_file_pwrite( (void) my_atomic_addlint(&os_n_pending_writes, 1); MONITOR_ATOMIC_INC(MONITOR_OS_PENDING_WRITES); - ssize_t n_bytes = os_file_io(type, file, (void*) buf, n, offset, err); + ssize_t n_bytes = os_file_io(type, file, const_cast<byte*>(buf), + n, offset, err); (void) my_atomic_addlint(&os_n_pending_writes, -1); MONITOR_ATOMIC_DEC(MONITOR_OS_PENDING_WRITES); @@ -5734,16 +4944,19 @@ os_file_write_page( IORequest& type, const char* name, os_file_t file, - const byte* buf, + const void* buf, os_offset_t offset, ulint n) { dberr_t err; + ut_ad(type.is_write()); ut_ad(type.validate()); ut_ad(n > 0); - - ssize_t n_bytes = os_file_pwrite(type, file, buf, n, offset, &err); + + WAIT_ALLOW_WRITES(); + + ssize_t n_bytes = os_file_pwrite(type, file, (byte*)buf, n, offset, &err); if ((ulint) n_bytes != n && !os_has_said_disk_full) { @@ -5846,23 +5059,7 @@ os_file_read_page( return(err); } else if ((ulint) n_bytes == n) { - -#ifdef MYSQL_COMPRESSION - /** The read will succeed but decompress can fail - for various reasons. */ - - if (type.is_compression_enabled() - && !Compression::is_compressed_page( - static_cast<byte*>(buf))) { - - return(DB_SUCCESS); - - } else { - return(err); - } -#else return(DB_SUCCESS); -#endif /* MYSQL_COMPRESSION */ } ib::error() << "Tried to read " << n @@ -5910,22 +5107,18 @@ os_file_get_last_error( return(os_file_get_last_error_low(report_all_errors, false)); } -/** Does error handling when a file operation fails. -Conditionally exits (calling srv_fatal_error()) based on should_exit value -and the error type, if should_exit is true then on_error_silent is ignored. +/** Handle errors for file operations. @param[in] name name of a file or NULL @param[in] operation operation -@param[in] should_exit call srv_fatal_error() on an unknown error, - if this parameter is true -@param[in] on_error_silent if true then don't print any message to the log - iff it is an unknown non-fatal error +@param[in] should_abort whether to abort on an unknown error +@param[in] on_error_silent whether to suppress reports of non-fatal errors @return true if we should retry the operation */ static MY_ATTRIBUTE((warn_unused_result)) bool os_file_handle_error_cond_exit( const char* name, const char* operation, - bool should_exit, + bool should_abort, bool on_error_silent) { ulint err; @@ -5986,17 +5179,17 @@ os_file_handle_error_cond_exit( is better to ignore on_error_silent and print an error message to the log. */ - if (should_exit || !on_error_silent) { + if (should_abort || !on_error_silent) { ib::error() << "File " << (name != NULL ? name : "(unknown)") << ": '" << operation << "'" " returned OS error " << err << "." - << (should_exit + << (should_abort ? " Cannot continue operation" : ""); } - if (should_exit) { - srv_fatal_error(); + if (should_abort) { + abort(); } } @@ -6140,21 +5333,8 @@ os_file_set_size( dberr_t err; IORequest request(IORequest::WRITE); -#ifdef UNIV_HOTBACKUP - err = os_file_write( request, name, file, buf, current_size, n_bytes); -#else - /* Using OS_AIO_SYNC mode on POSIX systems will result in - fall back to os_file_write/read. On Windows it will use - special mechanism to wait before it returns back. */ - - err = os_aio( - request, - OS_AIO_SYNC, name, - file, buf, current_size, n_bytes, - read_only, NULL, NULL, NULL); -#endif /* UNIV_HOTBACKUP */ if (err != DB_SUCCESS) { @@ -6280,12 +5460,6 @@ os_file_write_func( ut_ad(type.validate()); ut_ad(type.is_write()); - /* We never compress the first page. - Note: This assumes we always do block IO. */ - if (offset == 0) { - type.clear_compressed(); - } - const byte* ptr = reinterpret_cast<const byte*>(buf); return(os_file_write_page(type, name, file, ptr, offset, n)); @@ -6320,6 +5494,28 @@ os_file_punch_hole( os_offset_t off, os_offset_t len) { + dberr_t err; + +#ifdef _WIN32 + err = os_file_punch_hole_win32(fh, off, len); +#else + err = os_file_punch_hole_posix(fh, off, len); +#endif /* _WIN32 */ + + return (err); +} + +/** Free storage space associated with a section of the file. +@param[in] fh Open file handle +@param[in] off Starting offset (SEEK_SET) +@param[in] len Size of the hole +@return DB_SUCCESS or error code */ +dberr_t +IORequest::punch_hole( + os_file_t fh, + os_offset_t off, + os_offset_t len) +{ /* In this debugging mode, we act as if punch hole is supported, and then skip any calls to actually punch a hole here. In this way, Transparent Page Compression is still being tested. */ @@ -6327,11 +5523,34 @@ os_file_punch_hole( return(DB_SUCCESS); ); -#ifdef _WIN32 - return(os_file_punch_hole_win32(fh, off, len)); -#else - return(os_file_punch_hole_posix(fh, off, len)); -#endif /* _WIN32 */ + os_offset_t trim_len = static_cast<os_offset_t>(get_trim_length(len)); + + if (trim_len == 0) { + return(DB_SUCCESS); + } + + off += len; + + /* Check does file system support punching holes for this + tablespace. */ + if (!should_punch_hole() || !srv_use_trim) { + return DB_IO_NO_PUNCH_HOLE; + } + + dberr_t err = os_file_punch_hole(fh, off, trim_len); + + if (err == DB_SUCCESS) { + srv_stats.page_compressed_trim_op.inc(); + } else { + /* If punch hole is not supported, + set space so that it is not used. */ + if (err == DB_IO_NO_PUNCH_HOLE) { + space_no_punch_hole(); + err = DB_SUCCESS; + } + } + + return (err); } /** Check if the file system supports sparse files. @@ -6362,7 +5581,7 @@ os_is_sparse_file_supported(const char* path, os_file_t fh) /* We don't know the FS block size, use the sector size. The FS will do the magic. */ - err = os_file_punch_hole(fh, 0, UNIV_PAGE_SIZE); + err = os_file_punch_hole_posix(fh, 0, UNIV_PAGE_SIZE); return(err == DB_SUCCESS); #endif /* _WIN32 */ @@ -6522,16 +5741,6 @@ AIO::init_slots() memset(&slot.control, 0x0, sizeof(slot.control)); #endif /* WIN_ASYNC_IO */ - - slot.compressed_ptr = reinterpret_cast<byte*>( - ut_zalloc_nokey(UNIV_PAGE_SIZE_MAX * 2)); - - if (slot.compressed_ptr == NULL) { - return(DB_OUT_OF_MEMORY); - } - - slot.compressed_page = static_cast<byte *>( - ut_align(slot.compressed_ptr, UNIV_PAGE_SIZE)); } return(DB_SUCCESS); @@ -6658,16 +5867,6 @@ AIO::~AIO() } #endif /* LINUX_NATIVE_AIO */ - for (ulint i = 0; i < m_slots.size(); ++i) { - Slot& slot = m_slots[i]; - - if (slot.compressed_ptr != NULL) { - ut_free(slot.compressed_ptr); - slot.compressed_ptr = NULL; - slot.compressed_page = NULL; - } - } - m_slots.clear(); } @@ -6773,6 +5972,12 @@ AIO::start( os_aio_validate(); + os_last_printout = ut_time(); + + if (srv_use_native_aio) { + return(true); + } + os_aio_segment_wait_events = static_cast<os_event_t*>( ut_zalloc_nokey( n_segments * sizeof *os_aio_segment_wait_events)); @@ -6786,8 +5991,6 @@ AIO::start( os_aio_segment_wait_events[i] = os_event_create(0); } - os_last_printout = ut_time(); - return(true); } @@ -6829,27 +6032,6 @@ os_aio_init( /* Maximum number of pending aio operations allowed per segment */ ulint limit = 8 * OS_AIO_N_PENDING_IOS_PER_THREAD; - - ut_a(block_cache == NULL); - - block_cache = UT_NEW_NOKEY(Blocks(MAX_BLOCKS)); - - for (Blocks::iterator it = block_cache->begin(); - it != block_cache->end(); - ++it) { - - ut_a(it->m_in_use == 0); - ut_a(it->m_ptr == NULL); - - /* Allocate double of max page size memory, since - compress could generate more bytes than orgininal - data. */ - it->m_ptr = static_cast<byte*>( - ut_malloc_nokey(BUFFER_BLOCK_SIZE)); - - ut_a(it->m_ptr != NULL); - } - return(AIO::start(limit, n_readers, n_writers, n_slots_sync)); } @@ -6859,25 +6041,15 @@ os_aio_free() { AIO::shutdown(); - for (ulint i = 0; i < os_aio_n_segments; i++) { - os_event_destroy(os_aio_segment_wait_events[i]); - } - - ut_free(os_aio_segment_wait_events); - os_aio_segment_wait_events = 0; - os_aio_n_segments = 0; - - for (Blocks::iterator it = block_cache->begin(); - it != block_cache->end(); - ++it) { + if (!srv_use_native_aio) { + for (ulint i = 0; i < os_aio_n_segments; i++) { + os_event_destroy(os_aio_segment_wait_events[i]); + } - ut_a(it->m_in_use == 0); - ut_free(it->m_ptr); + ut_free(os_aio_segment_wait_events); + os_aio_segment_wait_events = 0; } - - UT_DELETE(block_cache); - - block_cache = NULL; + os_aio_n_segments = 0; } /** Wakes up all async i/o threads so that they know to exit themselves in @@ -6890,21 +6062,16 @@ os_aio_wake_all_threads_at_shutdown() AIO::wake_at_shutdown(); #elif defined(LINUX_NATIVE_AIO) - /* When using native AIO interface the io helper threads wait on io_getevents with a timeout value of 500ms. At each wake up these threads check the server status. No need to do anything to wake them up. */ +#endif /* !WIN_ASYNC_AIO */ if (srv_use_native_aio) { return; } -#endif /* !WIN_ASYNC_AIO */ - - /* Fall through to simulated AIO handler wakeup if we are - not using native AIO. */ - /* This loop wakes up all simulated ai/o threads */ for (ulint i = 0; i < os_aio_n_segments; ++i) { @@ -6984,12 +6151,7 @@ AIO::reserve_slot( const char* name, void* buf, os_offset_t offset, - ulint len, - ulint* write_size)/*!< in/out: Actual write size initialized - after fist successfull trim - operation for this page and if - initialized we do not trim again if - actual page size does not decrease. */ + ulint len) { #ifdef WIN_ASYNC_IO ut_a((len & 0xFFFFFFFFUL) == len); @@ -7079,83 +6241,10 @@ AIO::reserve_slot( slot->ptr = slot->buf; slot->offset = offset; slot->err = DB_SUCCESS; - slot->write_size = write_size; - slot->is_log = type.is_log(); slot->original_len = static_cast<uint32>(len); slot->io_already_done = false; - slot->buf_block = NULL; slot->buf = static_cast<byte*>(buf); -#ifdef MYSQL_COMPRESSION - if (srv_use_native_aio - && offset > 0 - && type.is_write() - && type.is_compressed()) { - ulint compressed_len = len; - - ut_ad(!type.is_log()); - - release(); - - void* src_buf = slot->buf; - - slot->buf_block = os_file_compress_page( - type, - src_buf, - &compressed_len); - - slot->buf = static_cast<byte*>(src_buf); - slot->ptr = slot->buf; -#ifdef _WIN32 - slot->len = static_cast<DWORD>(compressed_len); -#else - slot->len = static_cast<ulint>(compressed_len); -#endif /* _WIN32 */ - slot->skip_punch_hole = type.punch_hole(); - - acquire(); - } -#endif /* MYSQL_COMPRESSION */ - -#ifdef MYSQL_ENCRYPTION - /* We do encryption after compression, since if we do encryption - before compression, the encrypted data will cause compression fail - or low compression rate. */ - if (srv_use_native_aio - && offset > 0 - && type.is_write() - && type.is_encrypted()) { - ulint encrypted_len = slot->len; - Block* encrypted_block; - - ut_ad(!type.is_log()); - - release(); - - void* src_buf = slot->buf; - encrypted_block = os_file_encrypt_page( - type, - src_buf, - &encrypted_len); - - if (slot->buf_block != NULL) { - os_free_block(slot->buf_block); - } - - slot->buf_block = encrypted_block; - slot->buf = static_cast<byte*>(src_buf); - slot->ptr = slot->buf; - -#ifdef _WIN32 - slot->len = static_cast<DWORD>(encrypted_len); -#else - slot->len = static_cast<ulint>(encrypted_len); -#endif /* _WIN32 */ - - acquire(); - } -#endif /* MYSQL_ENCRYPTION */ - #ifdef WIN_ASYNC_IO { OVERLAPPED* control; @@ -7504,6 +6593,7 @@ Requests an asynchronous i/o operation. @param[in,out] m2 message for the AIO handler (can be used to identify a completed AIO operation); ignored if mode is OS_AIO_SYNC + @return DB_SUCCESS or error code */ dberr_t os_aio_func( @@ -7516,12 +6606,7 @@ os_aio_func( ulint n, bool read_only, fil_node_t* m1, - void* m2, - ulint* write_size)/*!< in/out: Actual write size initialized - after fist successfull trim - operation for this page and if - initialized we do not trim again if - actual page size does not decrease. */ + void* m2) { #ifdef WIN_ASYNC_IO BOOL ret = TRUE; @@ -7557,7 +6642,7 @@ try_again: Slot* slot; - slot = array->reserve_slot(type, m1, m2, file, name, buf, offset, n, write_size); + slot = array->reserve_slot(type, m1, m2, file, name, buf, offset, n); if (type.is_read()) { @@ -7822,13 +6907,6 @@ public: } } - /** Do the decompression of the pages read in */ - void io_complete() - { - // Note: For non-compressed tables. Not required - // for correctness. - } - /** Mark the i/os done in slots */ void done() { @@ -7882,7 +6960,7 @@ private: slot->offset, slot->len); - ut_a(err == DB_SUCCESS || err == DB_IO_NO_PUNCH_HOLE); + ut_a(err == DB_SUCCESS); } /** @return true if the slots are adjacent and can be merged */ @@ -8173,8 +7251,6 @@ os_aio_simulated_handler( srv_set_io_thread_op_info(global_segment, "file i/o done"); - handler.io_complete(); - array->acquire(); handler.done(); @@ -8348,7 +7424,8 @@ os_aio_print(FILE* file) srv_io_thread_function[i]); #ifndef _WIN32 - if (os_event_is_set(os_aio_segment_wait_events[i])) { + if (!srv_use_native_aio + && os_event_is_set(os_aio_segment_wait_events[i])) { fprintf(file, " ev set"); } #endif /* _WIN32 */ @@ -8512,920 +7589,9 @@ os_file_set_umask(ulint umask) } #else - #include "univ.i" -#include "db0err.h" -#include "mach0data.h" -#include "fil0fil.h" -#include "os0file.h" - -#ifdef HAVE_LZ4 -#include <lz4.h> -#endif - -#include <zlib.h> -#ifndef UNIV_INNOCHECKSUM -#include <my_aes.h> -#include <my_rnd.h> -#include <mysqld.h> -#include <mysql/service_mysql_keyring.h> -#endif - -typedef byte Block; - -#ifdef MYSQL_COMPRESSION -/** Allocate a page for sync IO -@return pointer to page */ -static -Block* -os_alloc_block() -{ - return(reinterpret_cast<byte*>(malloc(UNIV_PAGE_SIZE_MAX * 2))); -} - -/** Free a page after sync IO -@param[in,own] block The block to free/release */ -static -void -os_free_block(Block* block) -{ - ut_free(block); -} -#endif -#endif /* !UNIV_INNOCHECKSUM */ - -#ifdef MYSQL_COMPRESSION - -/** -@param[in] type The compression type -@return the string representation */ -const char* -Compression::to_string(Type type) -{ - switch(type) { - case NONE: - return("None"); - case ZLIB: - return("Zlib"); - case LZ4: - return("LZ4"); - } - - ut_ad(0); - - return("<UNKNOWN>"); -} - -/** -@param[in] meta Page Meta data -@return the string representation */ -std::string Compression::to_string(const Compression::meta_t& meta) -{ - std::ostringstream stream; - - stream << "version: " << int(meta.m_version) << " " - << "algorithm: " << meta.m_algorithm << " " - << "(" << to_string(meta.m_algorithm) << ") " - << "orginal_type: " << meta.m_original_type << " " - << "original_size: " << meta.m_original_size << " " - << "compressed_size: " << meta.m_compressed_size; - - return(stream.str()); -} - -/** @return true if it is a compressed page */ -bool -Compression::is_compressed_page(const byte* page) -{ - return(mach_read_from_2(page + FIL_PAGE_TYPE) == FIL_PAGE_COMPRESSED); -} - -/** Deserizlise the page header compression meta-data -@param[in] page Pointer to the page header -@param[out] control Deserialised data */ -void -Compression::deserialize_header( - const byte* page, - Compression::meta_t* control) -{ - ut_ad(is_compressed_page(page)); - - control->m_version = static_cast<uint8_t>( - mach_read_from_1(page + FIL_PAGE_VERSION)); - - control->m_original_type = static_cast<uint16_t>( - mach_read_from_2(page + FIL_PAGE_ORIGINAL_TYPE_V1)); - - control->m_compressed_size = static_cast<uint16_t>( - mach_read_from_2(page + FIL_PAGE_COMPRESS_SIZE_V1)); - - control->m_original_size = static_cast<uint16_t>( - mach_read_from_2(page + FIL_PAGE_ORIGINAL_SIZE_V1)); - - control->m_algorithm = static_cast<Type>( - mach_read_from_1(page + FIL_PAGE_ALGORITHM_V1)); -} - -/** Decompress the page data contents. Page type must be FIL_PAGE_COMPRESSED, if -not then the source contents are left unchanged and DB_SUCCESS is returned. -@param[in] dblwr_recover true of double write recovery in progress -@param[in,out] src Data read from disk, decompressed data will be - copied to this page -@param[in,out] dst Scratch area to use for decompression -@param[in] dst_len Size of the scratch area in bytes -@return DB_SUCCESS or error code */ -dberr_t -Compression::deserialize( - bool dblwr_recover, - byte* src, - byte* dst, - ulint dst_len) -{ - if (!is_compressed_page(src)) { - /* There is nothing we can do. */ - return(DB_SUCCESS); - } - - meta_t header; - - deserialize_header(src, &header); - - byte* ptr = src + FIL_PAGE_DATA; - - ut_ad(header.m_version == 1); - - if (header.m_version != 1 - || header.m_original_size < UNIV_PAGE_SIZE_MIN - (FIL_PAGE_DATA + 8) - || header.m_original_size > UNIV_PAGE_SIZE_MAX - FIL_PAGE_DATA - || dst_len < header.m_original_size + FIL_PAGE_DATA) { - - /* The last check could potentially return DB_OVERFLOW, - the caller should be able to retry with a larger buffer. */ - - return(DB_CORRUPTION); - } - - Block* block; - - /* The caller doesn't know what to expect */ - if (dst == NULL) { - - block = os_alloc_block(); - -#ifdef UNIV_INNOCHECKSUM - dst = block; -#else - dst = block->m_ptr; -#endif /* UNIV_INNOCHECKSUM */ - - } else { - block = NULL; - } - - int ret; - Compression compression; - ulint len = header.m_original_size; - - compression.m_type = static_cast<Compression::Type>(header.m_algorithm); - - switch(compression.m_type) { - case Compression::ZLIB: { - - uLongf zlen = header.m_original_size; - - if (uncompress(dst, &zlen, ptr, header.m_compressed_size) - != Z_OK) { - - if (block != NULL) { - os_free_block(block); - } - - return(DB_IO_DECOMPRESS_FAIL); - } - - len = static_cast<ulint>(zlen); - - break; - } -#ifdef HAVE_LZ4 - case Compression::LZ4: { - int ret; - - if (dblwr_recover) { - - ret = LZ4_decompress_safe( - reinterpret_cast<char*>(ptr), - reinterpret_cast<char*>(dst), - header.m_compressed_size, - header.m_original_size); - - } else { - - /* This can potentially read beyond the input - buffer if the data is malformed. According to - the LZ4 documentation it is a little faster - than the above function. When recovering from - the double write buffer we can afford to us the - slower function above. */ - - ret = LZ4_decompress_fast( - reinterpret_cast<char*>(ptr), - reinterpret_cast<char*>(dst), - header.m_original_size); - } - - if (ret < 0) { - - if (block != NULL) { - os_free_block(block); - } - - return(DB_IO_DECOMPRESS_FAIL); - } - - break; - } -#endif - default: -#if !defined(UNIV_INNOCHECKSUM) - ib::error() - << "Compression algorithm support missing: " - << Compression::to_string(compression.m_type); -#else - fprintf(stderr, "Compression algorithm support missing: %s\n", - Compression::to_string(compression.m_type)); #endif /* !UNIV_INNOCHECKSUM */ - if (block != NULL) { - os_free_block(block); - } - - return(DB_UNSUPPORTED); - } - /* Leave the header alone */ - memmove(src + FIL_PAGE_DATA, dst, len); - - mach_write_to_2(src + FIL_PAGE_TYPE, header.m_original_type); - - ut_ad(dblwr_recover - || memcmp(src + FIL_PAGE_LSN + 4, - src + (header.m_original_size + FIL_PAGE_DATA) - - FIL_PAGE_END_LSN_OLD_CHKSUM + 4, 4) == 0); - - if (block != NULL) { - os_free_block(block); - } - - return(DB_SUCCESS); -} - -/** Decompress the page data contents. Page type must be FIL_PAGE_COMPRESSED, if -not then the source contents are left unchanged and DB_SUCCESS is returned. -@param[in] dblwr_recover true of double write recovery in progress -@param[in,out] src Data read from disk, decompressed data will be - copied to this page -@param[in,out] dst Scratch area to use for decompression -@param[in] dst_len Size of the scratch area in bytes -@return DB_SUCCESS or error code */ -dberr_t -os_file_decompress_page( - bool dblwr_recover, - byte* src, - byte* dst, - ulint dst_len) -{ - return(Compression::deserialize(dblwr_recover, src, dst, dst_len)); -} -#endif /* MYSQL_COMPRESSION */ - -#ifdef MYSQL_ENCRYPTION - -/** -@param[in] type The encryption type -@return the string representation */ -const char* -Encryption::to_string(Type type) -{ - switch(type) { - case NONE: - return("N"); - case AES: - return("Y"); - } - - ut_ad(0); - - return("<UNKNOWN>"); -} - -/** Generate random encryption value for key and iv. -@param[in,out] value Encryption value */ -void Encryption::random_value(byte* value) -{ - ut_ad(value != NULL); - - my_rand_buffer(value, ENCRYPTION_KEY_LEN); -} - -/** Create new master key for key rotation. -@param[in,out] master_key master key */ -void -Encryption::create_master_key(byte** master_key) -{ -#ifndef UNIV_INNOCHECKSUM - char* key_type = NULL; - size_t key_len; - char key_name[ENCRYPTION_MASTER_KEY_NAME_MAX_LEN]; - int ret; - - /* If uuid does not match with current server uuid, - set uuid as current server uuid. */ - if (strcmp(uuid, server_uuid) != 0) { - memcpy(uuid, server_uuid, ENCRYPTION_SERVER_UUID_LEN); - } - memset(key_name, 0, ENCRYPTION_MASTER_KEY_NAME_MAX_LEN); - - /* Generate new master key */ - ut_snprintf(key_name, ENCRYPTION_MASTER_KEY_NAME_MAX_LEN, - "%s-%s-%lu", ENCRYPTION_MASTER_KEY_PRIFIX, - uuid, master_key_id + 1); - - /* We call key ring API to generate master key here. */ - ret = my_key_generate(key_name, "AES", - NULL, ENCRYPTION_KEY_LEN); - - /* We call key ring API to get master key here. */ - ret = my_key_fetch(key_name, &key_type, NULL, - reinterpret_cast<void**>(master_key), - &key_len); - - if (ret || *master_key == NULL) { - ib::error() << "Encryption can't find master key, please check" - " the keyring plugin is loaded."; - *master_key = NULL; - } else { - master_key_id++; - } - - if (key_type) { - my_free(key_type); - } -#endif -} - -/** Get master key by key id. -@param[in] master_key_id master key id -@param[in] srv_uuid uuid of server instance -@param[in,out] master_key master key */ -void -Encryption::get_master_key(ulint master_key_id, - char* srv_uuid, - byte** master_key) -{ -#ifndef UNIV_INNOCHECKSUM - char* key_type = NULL; - size_t key_len; - char key_name[ENCRYPTION_MASTER_KEY_NAME_MAX_LEN]; - int ret; - - memset(key_name, 0, ENCRYPTION_MASTER_KEY_NAME_MAX_LEN); - - if (srv_uuid != NULL) { - ut_snprintf(key_name, ENCRYPTION_MASTER_KEY_NAME_MAX_LEN, - "%s-%s-%lu", ENCRYPTION_MASTER_KEY_PRIFIX, - srv_uuid, master_key_id); - } else { - /* For compitable with 5.7.11, we need to get master key with - server id. */ - memset(key_name, 0, ENCRYPTION_MASTER_KEY_NAME_MAX_LEN); - ut_snprintf(key_name, ENCRYPTION_MASTER_KEY_NAME_MAX_LEN, - "%s-%lu-%lu", ENCRYPTION_MASTER_KEY_PRIFIX, - server_id, master_key_id); - } - - /* We call key ring API to get master key here. */ - ret = my_key_fetch(key_name, &key_type, NULL, - reinterpret_cast<void**>(master_key), &key_len); - - if (key_type) { - my_free(key_type); - } - - if (ret) { - *master_key = NULL; - ib::error() << "Encryption can't find master key, please check" - " the keyring plugin is loaded."; - } - -#ifdef UNIV_ENCRYPT_DEBUG - if (!ret && *master_key) { - fprintf(stderr, "Fetched master key:%lu ", master_key_id); - ut_print_buf(stderr, *master_key, key_len); - fprintf(stderr, "\n"); - } -#endif /* DEBUG_TDE */ - -#endif -} - -/** Current master key id */ -ulint Encryption::master_key_id = 0; - -/** Current uuid of server instance */ -char Encryption::uuid[ENCRYPTION_SERVER_UUID_LEN + 1] = {0}; - -/** Get current master key and master key id -@param[in,out] master_key_id master key id -@param[in,out] master_key master key -@param[in,out] version encryption information version */ -void -Encryption::get_master_key(ulint* master_key_id, - byte** master_key, - Encryption::Version* version) -{ -#ifndef UNIV_INNOCHECKSUM - char* key_type = NULL; - size_t key_len; - char key_name[ENCRYPTION_MASTER_KEY_NAME_MAX_LEN]; - int ret; - - memset(key_name, 0, ENCRYPTION_KEY_LEN); - *version = Encryption::ENCRYPTION_VERSION_2; - - if (Encryption::master_key_id == 0) { - /* If m_master_key is 0, means there's no encrypted - tablespace, we need to generate the first master key, - and store it to key ring. */ - memset(uuid, 0, ENCRYPTION_SERVER_UUID_LEN + 1); - memcpy(uuid, server_uuid, ENCRYPTION_SERVER_UUID_LEN); - - /* Prepare the server uuid. */ - ut_snprintf(key_name, ENCRYPTION_MASTER_KEY_NAME_MAX_LEN, - "%s-%s-1", ENCRYPTION_MASTER_KEY_PRIFIX, - uuid); - - /* We call key ring API to generate master key here. */ - ret = my_key_generate(key_name, "AES", - NULL, ENCRYPTION_KEY_LEN); - - /* We call key ring API to get master key here. */ - ret = my_key_fetch(key_name, &key_type, NULL, - reinterpret_cast<void**>(master_key), - &key_len); - - if (!ret && *master_key != NULL) { - Encryption::master_key_id++; - *master_key_id = Encryption::master_key_id; - } -#ifdef UNIV_ENCRYPT_DEBUG - if (!ret && *master_key) { - fprintf(stderr, "Generated new master key:"); - ut_print_buf(stderr, *master_key, key_len); - fprintf(stderr, "\n"); - } -#endif - } else { - *master_key_id = Encryption::master_key_id; - - ut_snprintf(key_name, ENCRYPTION_MASTER_KEY_NAME_MAX_LEN, - "%s-%s-%lu", ENCRYPTION_MASTER_KEY_PRIFIX, - uuid, *master_key_id); - - /* We call key ring API to get master key here. */ - ret = my_key_fetch(key_name, &key_type, NULL, - reinterpret_cast<void**>(master_key), - &key_len); - - /* For compitable with 5.7.11, we need to try to get master key with - server id when get master key with server uuid failure. */ - if (ret || *master_key == NULL) { - if (key_type) { - my_free(key_type); - } - - memset(key_name, 0, - ENCRYPTION_MASTER_KEY_NAME_MAX_LEN); - ut_snprintf(key_name, ENCRYPTION_MASTER_KEY_NAME_MAX_LEN, - "%s-%lu-%lu", ENCRYPTION_MASTER_KEY_PRIFIX, - server_id, *master_key_id); - - ret = my_key_fetch(key_name, &key_type, NULL, - reinterpret_cast<void**>(master_key), - &key_len); - *version = Encryption::ENCRYPTION_VERSION_1; - } -#ifdef UNIV_ENCRYPT_DEBUG - if (!ret && *master_key) { - fprintf(stderr, "Fetched master key:%lu ", - *master_key_id); - ut_print_buf(stderr, *master_key, key_len); - fprintf(stderr, "\n"); - } -#endif - } - - if (ret) { - *master_key = NULL; - ib::error() << "Encryption can't find master key, please check" - " the keyring plugin is loaded."; - } - - if (key_type) { - my_free(key_type); - } -#endif -} - -/** Check if page is encrypted page or not -@param[in] page page which need to check -@return true if it is a encrypted page */ -bool -Encryption::is_encrypted_page(const byte* page) -{ - ulint page_type = mach_read_from_2(page + FIL_PAGE_TYPE); - - return(page_type == FIL_PAGE_ENCRYPTED - || page_type == FIL_PAGE_COMPRESSED_AND_ENCRYPTED - || page_type == FIL_PAGE_ENCRYPTED_RTREE); -} - -/** Encrypt the page data contents. Page type can't be -FIL_PAGE_ENCRYPTED, FIL_PAGE_COMPRESSED_AND_ENCRYPTED, -FIL_PAGE_ENCRYPTED_RTREE. -@param[in] type IORequest -@param[in,out] src page data which need to encrypt -@param[in] src_len Size of the source in bytes -@param[in,out] dst destination area -@param[in,out] dst_len Size of the destination in bytes -@return buffer data, dst_len will have the length of the data */ -byte* -Encryption::encrypt( - const IORequest& type, - byte* src, - ulint src_len, - byte* dst, - ulint* dst_len) -{ - ulint len = 0; - ulint page_type = mach_read_from_2(src + FIL_PAGE_TYPE); - ulint data_len; - ulint main_len; - ulint remain_len; - byte remain_buf[MY_AES_BLOCK_SIZE * 2]; - -#ifdef UNIV_ENCRYPT_DEBUG - ulint space_id = - mach_read_from_4(src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - ulint page_no = mach_read_from_4(src + FIL_PAGE_OFFSET); - - fprintf(stderr, "Encrypting page:%lu.%lu len:%lu\n", - space_id, page_no, src_len); -#endif - - /* Shouldn't encrypte an already encrypted page. */ - ut_ad(page_type != FIL_PAGE_ENCRYPTED - && page_type != FIL_PAGE_COMPRESSED_AND_ENCRYPTED - && page_type != FIL_PAGE_ENCRYPTED_RTREE); - - ut_ad(m_type != Encryption::NONE); - - /* This is data size which need to encrypt. */ - data_len = src_len - FIL_PAGE_DATA; - main_len = (data_len / MY_AES_BLOCK_SIZE) * MY_AES_BLOCK_SIZE; - remain_len = data_len - main_len; - - /* Only encrypt the data + trailer, leave the header alone */ - - switch (m_type) { - case Encryption::NONE: - ut_error; - - case Encryption::AES: { - lint elen; - - ut_ad(m_klen == ENCRYPTION_KEY_LEN); - - elen = my_aes_encrypt( - src + FIL_PAGE_DATA, - static_cast<uint32>(main_len), - dst + FIL_PAGE_DATA, - reinterpret_cast<unsigned char*>(m_key), - static_cast<uint32>(m_klen), - my_aes_256_cbc, - reinterpret_cast<unsigned char*>(m_iv), - false); - - if (elen == MY_AES_BAD_DATA) { - ulint page_no =mach_read_from_4( - src + FIL_PAGE_OFFSET); - ulint space_id = mach_read_from_4( - src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - *dst_len = src_len; -#ifndef UNIV_INNOCHECKSUM - ib::warn() - << " Can't encrypt data of page," - << " page no:" << page_no - << " space id:" << space_id; -#else - fprintf(stderr, " Can't encrypt data of page," - " page no:" ULINTPF - " space id:" ULINTPF, - page_no, space_id); -#endif /* !UNIV_INNOCHECKSUM */ - return(src); - } - - len = static_cast<ulint>(elen); - ut_ad(len == main_len); - - /* Copy remain bytes and page tailer. */ - memcpy(dst + FIL_PAGE_DATA + len, - src + FIL_PAGE_DATA + len, - src_len - FIL_PAGE_DATA - len); - - /* Encrypt the remain bytes. */ - if (remain_len != 0) { - remain_len = MY_AES_BLOCK_SIZE * 2; - - elen = my_aes_encrypt( - dst + FIL_PAGE_DATA + data_len - remain_len, - static_cast<uint32>(remain_len), - remain_buf, - reinterpret_cast<unsigned char*>(m_key), - static_cast<uint32>(m_klen), - my_aes_256_cbc, - reinterpret_cast<unsigned char*>(m_iv), - false); - - if (elen == MY_AES_BAD_DATA) { - ulint page_no =mach_read_from_4( - src + FIL_PAGE_OFFSET); - ulint space_id = mach_read_from_4( - src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); -#ifndef UNIV_INNOCHECKSUM - ib::warn() - << " Can't encrypt data of page," - << " page no:" << page_no - << " space id:" << space_id; -#else - fprintf(stderr, " Can't encrypt data of page," - " page no:" ULINTPF - " space id:" ULINTPF, - page_no, space_id); -#endif /* !UNIV_INNOCHECKSUM */ - *dst_len = src_len; - return(src); - } - - memcpy(dst + FIL_PAGE_DATA + data_len - remain_len, - remain_buf, remain_len); - } - - - break; - } - - default: - ut_error; - } - - /* Copy the header as is. */ - memmove(dst, src, FIL_PAGE_DATA); - ut_ad(memcmp(src, dst, FIL_PAGE_DATA) == 0); - - /* Add encryption control information. Required for decrypting. */ - if (page_type == FIL_PAGE_COMPRESSED) { - /* If the page is compressed, we don't need to save the - original type, since it is done in compression already. */ - mach_write_to_2(dst + FIL_PAGE_TYPE, - FIL_PAGE_COMPRESSED_AND_ENCRYPTED); - ut_ad(memcmp(src+FIL_PAGE_TYPE+2, - dst+FIL_PAGE_TYPE+2, - FIL_PAGE_DATA-FIL_PAGE_TYPE-2) == 0); - } else if (page_type == FIL_PAGE_RTREE) { - /* If the page is R-tree page, we need to save original - type. */ - mach_write_to_2(dst + FIL_PAGE_TYPE, FIL_PAGE_ENCRYPTED_RTREE); - } else{ - mach_write_to_2(dst + FIL_PAGE_TYPE, FIL_PAGE_ENCRYPTED); - mach_write_to_2(dst + FIL_PAGE_ORIGINAL_TYPE_V1, page_type); - } - -#ifdef UNIV_ENCRYPT_DEBUG -#ifndef UNIV_INNOCHECKSUM -#if 0 - byte* check_buf = static_cast<byte*>(ut_malloc_nokey(src_len)); - byte* buf2 = static_cast<byte*>(ut_malloc_nokey(src_len)); - - memcpy(check_buf, dst, src_len); - - dberr_t err = decrypt(type, check_buf, src_len, buf2, src_len); - if (err != DB_SUCCESS || memcmp(src + FIL_PAGE_DATA, - check_buf + FIL_PAGE_DATA, - src_len - FIL_PAGE_DATA) != 0) { - ut_print_buf(stderr, src, src_len); - ut_print_buf(stderr, check_buf, src_len); - ut_ad(0); - } - ut_free(buf2); - ut_free(check_buf); -#endif - fprintf(stderr, "Encrypted page:%lu.%lu\n", space_id, page_no); -#endif -#endif - *dst_len = src_len; - - - return(dst); -} - -/** Decrypt the page data contents. Page type must be FIL_PAGE_ENCRYPTED, -if not then the source contents are left unchanged and DB_SUCCESS is returned. -@param[in] type IORequest -@param[in,out] src Data read from disk, decrypted data will be - copied to this page -@param[in] src_len source data length -@param[in,out] dst Scratch area to use for decryption -@param[in] dst_len Size of the scratch area in bytes -@return DB_SUCCESS or error code */ -dberr_t -Encryption::decrypt( - const IORequest& type, - byte* src, - ulint src_len, - byte* dst, - ulint dst_len) -{ - ulint data_len; - ulint main_len; - ulint remain_len; - ulint original_type; - ulint page_type; - byte remain_buf[MY_AES_BLOCK_SIZE * 2]; - Block* block; - - /* Do nothing if it's not an encrypted table. */ - if (!is_encrypted_page(src)) { - return(DB_SUCCESS); - } - - /* For compressed page, we need to get the compressed size - for decryption */ - page_type = mach_read_from_2(src + FIL_PAGE_TYPE); - if (page_type == FIL_PAGE_COMPRESSED_AND_ENCRYPTED) { - src_len = static_cast<uint16_t>( - mach_read_from_2(src + FIL_PAGE_COMPRESS_SIZE_V1)) - + FIL_PAGE_DATA; -#ifndef UNIV_INNOCHECKSUM - src_len = ut_calc_align(src_len, type.block_size()); -#endif - } -#ifdef UNIV_ENCRYPT_DEBUG - ulint space_id = - mach_read_from_4(src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - ulint page_no = mach_read_from_4(src + FIL_PAGE_OFFSET); - - fprintf(stderr, "Decrypting page:%lu.%lu len:%lu\n", - space_id, page_no, src_len); -#endif - - original_type = static_cast<uint16_t>( - mach_read_from_2(src + FIL_PAGE_ORIGINAL_TYPE_V1)); - - byte* ptr = src + FIL_PAGE_DATA; - - /* The caller doesn't know what to expect */ - if (dst == NULL) { - - block = os_alloc_block(); -#ifdef UNIV_INNOCHECKSUM - dst = block; -#else - dst = block->m_ptr; -#endif /* UNIV_INNOCHECKSUM */ - - } else { - block = NULL; - } - - data_len = src_len - FIL_PAGE_DATA; - main_len = (data_len / MY_AES_BLOCK_SIZE) * MY_AES_BLOCK_SIZE; - remain_len = data_len - main_len; - - switch(m_type) { - case Encryption::AES: { - lint elen; - - /* First decrypt the last 2 blocks data of data, since - data is no block aligned. */ - if (remain_len != 0) { - ut_ad(m_klen == ENCRYPTION_KEY_LEN); - - remain_len = MY_AES_BLOCK_SIZE * 2; - - /* Copy the last 2 blocks. */ - memcpy(remain_buf, - ptr + data_len - remain_len, - remain_len); - - elen = my_aes_decrypt( - remain_buf, - static_cast<uint32>(remain_len), - dst + data_len - remain_len, - reinterpret_cast<unsigned char*>(m_key), - static_cast<uint32>(m_klen), - my_aes_256_cbc, - reinterpret_cast<unsigned char*>(m_iv), - false); - if (elen == MY_AES_BAD_DATA) { - if (block != NULL) { - os_free_block(block); - } - - return(DB_IO_DECRYPT_FAIL); - } - - /* Copy the other data bytes to temp area. */ - memcpy(dst, ptr, data_len - remain_len); - } else { - ut_ad(data_len == main_len); - - /* Copy the data bytes to temp area. */ - memcpy(dst, ptr, data_len); - } - - /* Then decrypt the main data */ - elen = my_aes_decrypt( - dst, - static_cast<uint32>(main_len), - ptr, - reinterpret_cast<unsigned char*>(m_key), - static_cast<uint32>(m_klen), - my_aes_256_cbc, - reinterpret_cast<unsigned char*>(m_iv), - false); - if (elen == MY_AES_BAD_DATA) { - - if (block != NULL) { - os_free_block(block); - } - - return(DB_IO_DECRYPT_FAIL); - } - - ut_ad(static_cast<ulint>(elen) == main_len); - - /* Copy the remain bytes. */ - memcpy(ptr + main_len, dst + main_len, data_len - main_len); - - break; - } - - default: -#if !defined(UNIV_INNOCHECKSUM) - ib::error() - << "Encryption algorithm support missing: " - << Encryption::to_string(m_type); -#else - fprintf(stderr, "Encryption algorithm support missing: %s\n", - Encryption::to_string(m_type)); -#endif /* !UNIV_INNOCHECKSUM */ - - if (block != NULL) { - os_free_block(block); - } - - return(DB_UNSUPPORTED); - } - - /* Restore the original page type. If it's a compressed and - encrypted page, just reset it as compressed page type, since - we will do uncompress later. */ - - if (page_type == FIL_PAGE_ENCRYPTED) { - mach_write_to_2(src + FIL_PAGE_TYPE, original_type); - mach_write_to_2(src + FIL_PAGE_ORIGINAL_TYPE_V1, 0); - } else if (page_type == FIL_PAGE_ENCRYPTED_RTREE) { - mach_write_to_2(src + FIL_PAGE_TYPE, FIL_PAGE_RTREE); - } else { - ut_ad(page_type == FIL_PAGE_COMPRESSED_AND_ENCRYPTED); - mach_write_to_2(src + FIL_PAGE_TYPE, FIL_PAGE_COMPRESSED); - } - - if (block != NULL) { - os_free_block(block); - } - -#ifdef UNIV_ENCRYPT_DEBUG - fprintf(stderr, "Decrypted page:%lu.%lu\n", space_id, page_no); -#endif - - DBUG_EXECUTE_IF("ib_crash_during_decrypt_page", DBUG_SUICIDE();); - - return(DB_SUCCESS); -} -#endif /* MYSQL_ENCRYPTION */ - /** Normalizes a directory path for the current OS: On Windows, we convert '/' to '\', else we convert '\' to '/'. @param[in,out] str A null-terminated directory and file path */ diff --git a/storage/innobase/os/os0thread.cc b/storage/innobase/os/os0thread.cc index edc9c8e9406..8039c2e4f46 100644 --- a/storage/innobase/os/os0thread.cc +++ b/storage/innobase/os/os0thread.cc @@ -32,10 +32,8 @@ Created 9/8/1995 Heikki Tuuri #include "os0thread.ic" #endif -#ifndef UNIV_HOTBACKUP #include "srv0srv.h" #include "os0event.h" - #include <map> /** Mutex that tracks the thread count. Used by innorwlocktest.cc @@ -153,9 +151,7 @@ os_thread_create_func( int ret = pthread_create(&new_thread_id, &attr, func, arg); - if (ret != 0) { - ib::fatal() << "pthread_create returned " << ret; - } + ut_a(ret == 0); pthread_attr_destroy(&attr); @@ -210,7 +206,6 @@ os_thread_yield(void) sched_yield(); #endif } -#endif /* !UNIV_HOTBACKUP */ /*****************************************************************//** The thread sleeps at least the time given in microseconds. */ @@ -284,4 +279,3 @@ os_thread_free() mutex_destroy(&thread_mutex); } - diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index b0412009b80..b32e8deb6e6 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -35,7 +35,6 @@ Created 10/4/1994 Heikki Tuuri #include "btr0btr.h" #include "mtr0log.h" #include "log0recv.h" -#ifndef UNIV_HOTBACKUP #include "rem0cmp.h" #include "gis0rtree.h" @@ -301,96 +300,6 @@ page_cur_rec_field_extends( } #endif /* PAGE_CUR_LE_OR_EXTENDS */ -/** If key is fixed length then populate offset directly from -cached version. -@param[in] rec B-Tree record for which offset needs to be - populated. -@param[in,out] index index handler -@param[in] tuple data tuple -@param[in,out] offsets default offsets array -@param[in,out] heap heap -@return reference to populate offsets. */ -static -ulint* -populate_offsets( - const rec_t* rec, - const dtuple_t* tuple, - dict_index_t* index, - ulint* offsets, - mem_heap_t** heap) -{ - ut_ad(dict_table_is_intrinsic(index->table)); - - bool rec_has_null_values = false; - - if (index->rec_cache.key_has_null_cols) { - /* Check if record has null value. */ - const byte* nulls = rec - (1 + REC_N_NEW_EXTRA_BYTES); - ulint n_bytes_to_scan - = UT_BITS_IN_BYTES(index->n_nullable); - byte null_mask = 0xff; - ulint bits_examined = 0; - - for (ulint i = 0; i < n_bytes_to_scan - 1; i++) { - if (*nulls & null_mask) { - rec_has_null_values = true; - break; - } - --nulls; - bits_examined += 8; - } - - if (!rec_has_null_values) { - null_mask >>= (8 - (index->n_nullable - bits_examined)); - rec_has_null_values = *nulls & null_mask; - } - - if (rec_has_null_values) { - - offsets = rec_get_offsets( - rec, index, offsets, - dtuple_get_n_fields_cmp(tuple), heap); - - return(offsets); - } - } - - /* Check if offsets are cached else cache them first. - There are queries that will first verify if key is present using index - search and then initiate insert. If offsets are cached during index - search it would be based on key part only but during insert that looks - out for exact location to insert key + db_row_id both columns would - be used and so re-compute offsets in such case. */ - if (!index->rec_cache.offsets_cached - || (rec_offs_n_fields(index->rec_cache.offsets) - < dtuple_get_n_fields_cmp(tuple))) { - - offsets = rec_get_offsets( - rec, index, offsets, - dtuple_get_n_fields_cmp(tuple), heap); - - /* Reallocate if our offset array is not big - enough to hold the needed size. */ - ulint sz1 = index->rec_cache.sz_of_offsets; - ulint sz2 = offsets[0]; - if (sz1 < sz2) { - index->rec_cache.offsets = static_cast<ulint*>( - mem_heap_alloc( - index->heap, sizeof(ulint) * sz2)); - index->rec_cache.sz_of_offsets = - static_cast<uint32_t>(sz2); - } - - memcpy(index->rec_cache.offsets, - offsets, (sizeof(ulint) * sz2)); - index->rec_cache.offsets_cached = true; - } - - ut_ad(index->rec_cache.offsets[2] = (ulint) rec); - - return(index->rec_cache.offsets); -} - /****************************************************************//** Searches the right position for a page cursor. */ void @@ -522,17 +431,9 @@ page_cur_search_with_match( up_matched_fields); offsets = offsets_; - if (index->rec_cache.fixed_len_key) { - offsets = populate_offsets( - mid_rec, tuple, - const_cast<dict_index_t*>(index), - offsets, &heap); - } else { - offsets = rec_get_offsets( - mid_rec, index, offsets, - dtuple_get_n_fields_cmp(tuple), &heap); - - } + offsets = rec_get_offsets( + mid_rec, index, offsets, + dtuple_get_n_fields_cmp(tuple), &heap); cmp = cmp_dtuple_rec_with_match( tuple, mid_rec, offsets, &cur_matched_fields); @@ -584,17 +485,9 @@ up_slot_match: up_matched_fields); offsets = offsets_; - if (index->rec_cache.fixed_len_key) { - offsets = populate_offsets( - mid_rec, tuple, - const_cast<dict_index_t*>(index), - offsets, &heap); - } else { - offsets = rec_get_offsets( - mid_rec, index, offsets, - dtuple_get_n_fields_cmp(tuple), &heap); - - } + offsets = rec_get_offsets( + mid_rec, index, offsets, + dtuple_get_n_fields_cmp(tuple), &heap); cmp = cmp_dtuple_rec_with_match( tuple, mid_rec, offsets, &cur_matched_fields); @@ -1133,9 +1026,6 @@ need_extra_info: mlog_catenate_string(mtr, ins_ptr, rec_size); } } -#else /* !UNIV_HOTBACKUP */ -# define page_cur_insert_rec_write_log(ins_rec,size,cur,index,mtr) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses a log record of a record insert on a page. @@ -1554,214 +1444,6 @@ use_heap: return(insert_rec); } -/** Inserts a record next to page cursor on an uncompressed page. -@param[in] current_rec pointer to current record after which - the new record is inserted. -@param[in] index record descriptor -@param[in] tuple pointer to a data tuple -@param[in] n_ext number of externally stored columns -@param[in] mtr mini-transaction handle, or NULL - -@return pointer to record if succeed, NULL otherwise */ -rec_t* -page_cur_direct_insert_rec_low( - rec_t* current_rec, - dict_index_t* index, - const dtuple_t* tuple, - ulint n_ext, - mtr_t* mtr) -{ - byte* insert_buf; - ulint rec_size; - page_t* page; /*!< the relevant page */ - rec_t* last_insert; /*!< cursor position at previous - insert */ - rec_t* free_rec; /*!< a free record that was reused, - or NULL */ - rec_t* insert_rec; /*!< inserted record */ - ulint heap_no; /*!< heap number of the inserted - record */ - - page = page_align(current_rec); - - ut_ad(dict_table_is_comp(index->table) - == (ibool) !!page_is_comp(page)); - - ut_ad(fil_page_index_page_check(page)); - - ut_ad(mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID) - == index->id); - - ut_ad(!page_rec_is_supremum(current_rec)); - - /* 1. Get the size of the physical record in the page */ - rec_size = index->rec_cache.rec_size; - - /* 2. Try to find suitable space from page memory management */ - free_rec = page_header_get_ptr(page, PAGE_FREE); - if (free_rec) { - /* Try to allocate from the head of the free list. */ - ulint foffsets_[REC_OFFS_NORMAL_SIZE]; - ulint* foffsets = foffsets_; - mem_heap_t* heap = NULL; - - rec_offs_init(foffsets_); - - foffsets = rec_get_offsets( - free_rec, index, foffsets, ULINT_UNDEFINED, &heap); - if (rec_offs_size(foffsets) < rec_size) { - if (heap != NULL) { - mem_heap_free(heap); - heap = NULL; - } - - free_rec = NULL; - insert_buf = page_mem_alloc_heap( - page, NULL, rec_size, &heap_no); - - if (insert_buf == NULL) { - return(NULL); - } - } else { - insert_buf = free_rec - rec_offs_extra_size(foffsets); - - if (page_is_comp(page)) { - heap_no = rec_get_heap_no_new(free_rec); - page_mem_alloc_free( - page, NULL, - rec_get_next_ptr(free_rec, TRUE), - rec_size); - } else { - heap_no = rec_get_heap_no_old(free_rec); - page_mem_alloc_free( - page, NULL, - rec_get_next_ptr(free_rec, FALSE), - rec_size); - } - - if (heap != NULL) { - mem_heap_free(heap); - heap = NULL; - } - } - } else { - free_rec = NULL; - insert_buf = page_mem_alloc_heap(page, NULL, - rec_size, &heap_no); - - if (insert_buf == NULL) { - return(NULL); - } - } - - /* 3. Create the record */ - insert_rec = rec_convert_dtuple_to_rec(insert_buf, index, tuple, n_ext); - - /* 4. Insert the record in the linked list of records */ - ut_ad(current_rec != insert_rec); - - { - /* next record after current before the insertion */ - rec_t* next_rec = page_rec_get_next(current_rec); -#ifdef UNIV_DEBUG - if (page_is_comp(page)) { - ut_ad(rec_get_status(current_rec) - <= REC_STATUS_INFIMUM); - ut_ad(rec_get_status(insert_rec) < REC_STATUS_INFIMUM); - ut_ad(rec_get_status(next_rec) != REC_STATUS_INFIMUM); - } -#endif - page_rec_set_next(insert_rec, next_rec); - page_rec_set_next(current_rec, insert_rec); - } - - page_header_set_field(page, NULL, PAGE_N_RECS, - 1 + page_get_n_recs(page)); - - /* 5. Set the n_owned field in the inserted record to zero, - and set the heap_no field */ - if (page_is_comp(page)) { - rec_set_n_owned_new(insert_rec, NULL, 0); - rec_set_heap_no_new(insert_rec, heap_no); - } else { - rec_set_n_owned_old(insert_rec, 0); - rec_set_heap_no_old(insert_rec, heap_no); - } - - /* 6. Update the last insertion info in page header */ - - last_insert = page_header_get_ptr(page, PAGE_LAST_INSERT); - ut_ad(!last_insert || !page_is_comp(page) - || rec_get_node_ptr_flag(last_insert) - == rec_get_node_ptr_flag(insert_rec)); - - if (last_insert == NULL) { - page_header_set_field(page, NULL, PAGE_DIRECTION, - PAGE_NO_DIRECTION); - page_header_set_field(page, NULL, PAGE_N_DIRECTION, 0); - - } else if ((last_insert == current_rec) - && (page_header_get_field(page, PAGE_DIRECTION) - != PAGE_LEFT)) { - - page_header_set_field(page, NULL, PAGE_DIRECTION, - PAGE_RIGHT); - page_header_set_field(page, NULL, PAGE_N_DIRECTION, - page_header_get_field( - page, PAGE_N_DIRECTION) + 1); - - } else if ((page_rec_get_next(insert_rec) == last_insert) - && (page_header_get_field(page, PAGE_DIRECTION) - != PAGE_RIGHT)) { - - page_header_set_field(page, NULL, PAGE_DIRECTION, - PAGE_LEFT); - page_header_set_field(page, NULL, PAGE_N_DIRECTION, - page_header_get_field( - page, PAGE_N_DIRECTION) + 1); - } else { - page_header_set_field(page, NULL, PAGE_DIRECTION, - PAGE_NO_DIRECTION); - page_header_set_field(page, NULL, PAGE_N_DIRECTION, 0); - } - - page_header_set_ptr(page, NULL, PAGE_LAST_INSERT, insert_rec); - - /* 7. It remains to update the owner record. */ - { - rec_t* owner_rec = page_rec_find_owner_rec(insert_rec); - ulint n_owned; - if (page_is_comp(page)) { - n_owned = rec_get_n_owned_new(owner_rec); - rec_set_n_owned_new(owner_rec, NULL, n_owned + 1); - } else { - n_owned = rec_get_n_owned_old(owner_rec); - rec_set_n_owned_old(owner_rec, n_owned + 1); - } - - /* 8. Now we have incremented the n_owned field of the owner - record. If the number exceeds PAGE_DIR_SLOT_MAX_N_OWNED, - we have to split the corresponding directory slot in two. */ - - if (n_owned == PAGE_DIR_SLOT_MAX_N_OWNED) { - page_dir_split_slot( - page, NULL, - page_dir_find_owner_slot(owner_rec)); - } - } - - /* 8. Open the mtr for name sake to set the modification flag - to true failing which no flush would be done. */ - byte* log_ptr = mlog_open(mtr, 0); - ut_ad(log_ptr == NULL); - if (log_ptr != NULL) { - /* To keep complier happy. */ - mlog_close(mtr, log_ptr); - } - - return(insert_rec); -} - /***********************************************************//** Inserts a record next to page cursor on a compressed and uncompressed page. Returns pointer to inserted record if succeed, i.e., @@ -2233,7 +1915,6 @@ use_heap: return(insert_rec); } -#ifndef UNIV_HOTBACKUP /**********************************************************//** Writes a log record of copying a record list end to a new created page. @return 4-byte field where to write the log data length, or NULL if @@ -2261,7 +1942,6 @@ page_copy_rec_list_to_created_page_write_log( return(log_ptr); } -#endif /* !UNIV_HOTBACKUP */ /**********************************************************//** Parses a log record of copying a record list end to a new created page. @@ -2321,7 +2001,6 @@ page_parse_copy_rec_list_to_created_page( return(rec_end); } -#ifndef UNIV_HOTBACKUP /*************************************************************//** Copies records from page to a newly created page, from a given record onward, including that record. Infimum and supremum records are not copied. @@ -2538,9 +2217,6 @@ page_cur_delete_rec_write_log( mlog_close(mtr, log_ptr + 2); } -#else /* !UNIV_HOTBACKUP */ -# define page_cur_delete_rec_write_log(rec,index,mtr) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses log record of a record delete on a page. diff --git a/storage/innobase/page/page0page.cc b/storage/innobase/page/page0page.cc index 89669d09e89..e2d2dd40fd8 100644 --- a/storage/innobase/page/page0page.cc +++ b/storage/innobase/page/page0page.cc @@ -34,12 +34,10 @@ Created 2/2/1994 Heikki Tuuri #include "buf0buf.h" #include "btr0btr.h" #include "row0trunc.h" -#ifndef UNIV_HOTBACKUP -# include "srv0srv.h" -# include "lock0lock.h" -# include "fut0lst.h" -# include "btr0sea.h" -#endif /* !UNIV_HOTBACKUP */ +#include "srv0srv.h" +#include "lock0lock.h" +#include "fut0lst.h" +#include "btr0sea.h" /* THE INDEX PAGE ============== @@ -205,9 +203,7 @@ page_set_max_trx_id( mtr_t* mtr) /*!< in/out: mini-transaction, or NULL */ { page_t* page = buf_block_get_frame(block); -#ifndef UNIV_HOTBACKUP ut_ad(!mtr || mtr_memo_contains(mtr, block, MTR_MEMO_PAGE_X_FIX)); -#endif /* !UNIV_HOTBACKUP */ /* It is not necessary to write this change to the redo log, as during a database recovery we assume that the max trx id of every @@ -218,16 +214,48 @@ page_set_max_trx_id( page_zip_write_header(page_zip, page + (PAGE_HEADER + PAGE_MAX_TRX_ID), 8, mtr); -#ifndef UNIV_HOTBACKUP } else if (mtr) { mlog_write_ull(page + (PAGE_HEADER + PAGE_MAX_TRX_ID), trx_id, mtr); -#endif /* !UNIV_HOTBACKUP */ } else { mach_write_to_8(page + (PAGE_HEADER + PAGE_MAX_TRX_ID), trx_id); } } +/** Persist the AUTO_INCREMENT value on a clustered index root page. +@param[in,out] block clustered index root page +@param[in] index clustered index +@param[in] autoinc next available AUTO_INCREMENT value +@param[in,out] mtr mini-transaction +@param[in] reset whether to reset the AUTO_INCREMENT + to a possibly smaller value than currently + exists in the page */ +void +page_set_autoinc( + buf_block_t* block, + const dict_index_t* index MY_ATTRIBUTE((unused)), + ib_uint64_t autoinc, + mtr_t* mtr, + bool reset) +{ + ut_ad(mtr_memo_contains_flagged( + mtr, block, MTR_MEMO_PAGE_X_FIX | MTR_MEMO_PAGE_SX_FIX)); + ut_ad(dict_index_is_clust(index)); + ut_ad(index->page == block->page.id.page_no()); + ut_ad(index->space == block->page.id.space()); + + byte* field = PAGE_HEADER + PAGE_ROOT_AUTO_INC + + buf_block_get_frame(block); + if (!reset && mach_read_from_8(field) >= autoinc) { + /* nothing to update */ + } else if (page_zip_des_t* page_zip = buf_block_get_page_zip(block)) { + mach_write_to_8(field, autoinc); + page_zip_write_header(page_zip, field, 8, mtr); + } else { + mlog_write_ull(field, autoinc, mtr); + } +} + /************************************************************//** Allocates a block of memory from the heap of an index page. @return pointer to start of allocated buffer, or NULL if allocation fails */ @@ -265,7 +293,6 @@ page_mem_alloc_heap( return(NULL); } -#ifndef UNIV_HOTBACKUP /**********************************************************//** Writes a log record of page creation. */ UNIV_INLINE @@ -289,9 +316,6 @@ page_create_write_log( mlog_write_initial_log_record(frame, type, mtr); } -#else /* !UNIV_HOTBACKUP */ -# define page_create_write_log(frame,mtr,comp,is_rtree) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ /** The page infimum and supremum of an empty page in ROW_FORMAT=REDUNDANT */ static const byte infimum_supremum_redundant[] = { @@ -454,6 +478,22 @@ page_create_zip( is_spatial = index ? dict_index_is_spatial(index) : page_comp_info->type & DICT_SPATIAL; + /* PAGE_MAX_TRX_ID or PAGE_ROOT_AUTO_INC are always 0 for + temporary tables. */ + ut_ad(max_trx_id == 0 || !dict_table_is_temporary(index->table)); + /* In secondary indexes and the change buffer, PAGE_MAX_TRX_ID + must be zero on non-leaf pages. max_trx_id can be 0 when the + index consists of an empty root (leaf) page. */ + ut_ad(max_trx_id == 0 + || level == 0 + || !dict_index_is_sec_or_ibuf(index) + || dict_table_is_temporary(index->table)); + /* In the clustered index, PAGE_ROOT_AUTOINC or + PAGE_MAX_TRX_ID must be 0 on other pages than the root. */ + ut_ad(level == 0 || max_trx_id == 0 + || !dict_index_is_sec_or_ibuf(index) + || dict_table_is_temporary(index->table)); + page = page_create_low(block, TRUE, is_spatial); mach_write_to_2(PAGE_HEADER + PAGE_LEVEL + page, level); mach_write_to_8(PAGE_HEADER + PAGE_MAX_TRX_ID + page, max_trx_id); @@ -487,8 +527,8 @@ page_create_empty( dict_index_t* index, /*!< in: the index of the page */ mtr_t* mtr) /*!< in/out: mini-transaction */ { - trx_id_t max_trx_id = 0; - const page_t* page = buf_block_get_frame(block); + trx_id_t max_trx_id; + page_t* page = buf_block_get_frame(block); page_zip_des_t* page_zip= buf_block_get_page_zip(block); ut_ad(fil_page_index_page_check(page)); @@ -502,9 +542,15 @@ page_create_empty( && page_is_leaf(page)) { max_trx_id = page_get_max_trx_id(page); ut_ad(max_trx_id); + } else if (page_is_root(page)) { + /* Preserve PAGE_ROOT_AUTO_INC. */ + max_trx_id = page_get_max_trx_id(page); + } else { + max_trx_id = 0; } if (page_zip) { + ut_ad(!dict_table_is_temporary(index->table)); page_create_zip(block, index, page_header_get_field(page, PAGE_LEVEL), max_trx_id, NULL, mtr); @@ -513,8 +559,8 @@ page_create_empty( dict_index_is_spatial(index)); if (max_trx_id) { - page_update_max_trx_id( - block, page_zip, max_trx_id, mtr); + mlog_write_ull(PAGE_HEADER + PAGE_MAX_TRX_ID + page, + max_trx_id, mtr); } } } @@ -583,7 +629,6 @@ page_copy_rec_list_end_no_locks( } } -#ifndef UNIV_HOTBACKUP /*************************************************************//** Copies records from page to new_page, from a given record onward, including that record. Infimum and supremum records are not copied. @@ -923,9 +968,6 @@ page_delete_rec_list_write_log( mlog_close(mtr, log_ptr + 2); } } -#else /* !UNIV_HOTBACKUP */ -# define page_delete_rec_list_write_log(rec,index,type,mtr) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ /**********************************************************//** Parses a log record of a record list end or start deletion. @@ -1269,7 +1311,6 @@ page_delete_rec_list_start( mtr_set_log_mode(mtr, log_mode); } -#ifndef UNIV_HOTBACKUP /*************************************************************//** Moves record list end to another page. Moved records include split_rec. @@ -1360,7 +1401,6 @@ page_move_rec_list_start( return(TRUE); } -#endif /* !UNIV_HOTBACKUP */ /**************************************************************//** Used to delete n slots from the directory. This function updates @@ -1686,7 +1726,6 @@ page_rec_get_n_recs_before( return((ulint) n); } -#ifndef UNIV_HOTBACKUP /************************************************************//** Prints record contents including the data relevant only in the index page context. */ @@ -1712,7 +1751,7 @@ page_rec_print( rec_validate(rec, offsets); } -# ifdef UNIV_BTR_PRINT +#ifdef UNIV_BTR_PRINT /***************************************************************//** This is used to print the contents of the directory for debugging purposes. */ @@ -1869,8 +1908,7 @@ page_print( page_dir_print(page, dn); page_print_list(block, index, rn); } -# endif /* UNIV_BTR_PRINT */ -#endif /* !UNIV_HOTBACKUP */ +#endif /* UNIV_BTR_PRINT */ /***************************************************************//** The following is used to validate a record on a page. This function @@ -1917,7 +1955,6 @@ page_rec_validate( return(TRUE); } -#ifndef UNIV_HOTBACKUP #ifdef UNIV_DEBUG /***************************************************************//** Checks that the first directory slot points to the infimum record and @@ -1950,7 +1987,6 @@ page_check_dir( } } #endif /* UNIV_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ /***************************************************************//** This function checks the consistency of an index page when we do not @@ -2449,7 +2485,6 @@ page_validate( goto func_exit; } -#ifndef UNIV_HOTBACKUP /* Check that the records are in the ascending order */ if (count >= PAGE_HEAP_NO_USER_LOW && !page_rec_is_supremum(rec)) { @@ -2492,7 +2527,6 @@ page_validate( goto func_exit; } } -#endif /* !UNIV_HOTBACKUP */ if (page_rec_is_user_rec(rec)) { @@ -2656,7 +2690,6 @@ func_exit2: return(ret); } -#ifndef UNIV_HOTBACKUP /***************************************************************//** Looks in the page record list for a record with the given heap number. @return record, NULL if not found */ @@ -2702,7 +2735,6 @@ page_find_rec_with_heap_no( } } } -#endif /* !UNIV_HOTBACKUP */ /*******************************************************//** Removes the record from a leaf page. This function does not log diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index d93b80778b5..d49cecdb968 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -49,32 +49,24 @@ const byte field_ref_zero[FIELD_REF_SIZE] = { #include "log0recv.h" #include "row0trunc.h" #include "zlib.h" -#ifndef UNIV_HOTBACKUP -# include "buf0buf.h" +#include "buf0buf.h" #include "buf0types.h" #include "buf0checksum.h" -# include "btr0sea.h" -# include "dict0boot.h" -# include "lock0lock.h" -# include "srv0srv.h" -# include "buf0lru.h" -# include "srv0mon.h" -# include "ut0crc32.h" -#else /* !UNIV_HOTBACKUP */ -# include "buf0checksum.h" -# define lock_move_reorganize_page(block, temp_block) ((void) 0) -# define buf_LRU_stat_inc_unzip() ((void) 0) -#endif /* !UNIV_HOTBACKUP */ +#include "btr0sea.h" +#include "dict0boot.h" +#include "lock0lock.h" +#include "srv0srv.h" +#include "buf0lru.h" +#include "srv0mon.h" +#include "ut0crc32.h" #include <map> #include <algorithm> -#ifndef UNIV_HOTBACKUP /** Statistics on compression, indexed by page_zip_des_t::ssize - 1 */ page_zip_stat_t page_zip_stat[PAGE_ZIP_SSIZE_MAX]; /** Statistics on compression, indexed by index->id */ page_zip_stat_per_index_t page_zip_stat_per_index; -#endif /* !UNIV_HOTBACKUP */ /* Compression level to be used by zlib. Settable by user. */ uint page_zip_level = DEFAULT_COMPRESSION_LEVEL; @@ -156,7 +148,6 @@ page_zip_fail_func( # define page_zip_fail(fmt_args) /* empty */ #endif /* UNIV_DEBUG || UNIV_ZIP_DEBUG */ -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Determine the guaranteed free space on an empty page. @return minimum payload size on the page */ @@ -228,7 +219,6 @@ page_zip_is_too_big( return(false); } -#endif /* !UNIV_HOTBACKUP */ /*************************************************************//** Gets the number of elements in the dense page directory, @@ -384,7 +374,6 @@ page_zip_dir_get( - PAGE_ZIP_DIR_SLOT_SIZE * (slot + 1))); } -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Write a log record of compressing an index page. */ static @@ -448,7 +437,6 @@ page_zip_compress_write_log( mlog_catenate_string(mtr, page_zip->data + page_zip_get_size(page_zip) - trailer_size, trailer_size); } -#endif /* !UNIV_HOTBACKUP */ /******************************************************//** Determine how many externally stored columns are contained @@ -1290,9 +1278,7 @@ page_zip_compress( byte* storage; /* storage of uncompressed columns */ index_id_t ind_id; -#ifndef UNIV_HOTBACKUP uintmax_t usec = ut_time_us(NULL); -#endif /* !UNIV_HOTBACKUP */ #ifdef PAGE_ZIP_COMPRESS_DBG FILE* logfile = NULL; #endif @@ -1375,14 +1361,12 @@ page_zip_compress( } } #endif /* PAGE_ZIP_COMPRESS_DBG */ -#ifndef UNIV_HOTBACKUP page_zip_stat[page_zip->ssize - 1].compressed++; if (cmp_per_index_enabled) { mutex_enter(&page_zip_stat_per_index_mutex); page_zip_stat_per_index[ind_id].compressed++; mutex_exit(&page_zip_stat_per_index_mutex); } -#endif /* !UNIV_HOTBACKUP */ if (UNIV_UNLIKELY(n_dense * PAGE_ZIP_DIR_SLOT_SIZE >= page_zip_get_size(page_zip))) { @@ -1404,9 +1388,8 @@ page_zip_compress( #ifdef UNIV_DEBUG ib::error() - << "InnoDB: Simulating a compression failure" - << " for table " - << (index->table->name.m_name) + << "Simulating a compression failure" + << " for table " << index->table->name << " index " << index->name() << " page " @@ -1578,7 +1561,6 @@ err_exit: fclose(logfile); } #endif /* PAGE_ZIP_COMPRESS_DBG */ -#ifndef UNIV_HOTBACKUP if (page_is_leaf(page) && index) { dict_index_zip_failure(index); } @@ -1592,7 +1574,6 @@ err_exit: += time_diff; mutex_exit(&page_zip_stat_per_index_mutex); } -#endif /* !UNIV_HOTBACKUP */ return(FALSE); } @@ -1633,9 +1614,7 @@ err_exit: #endif /* UNIV_ZIP_DEBUG */ if (mtr) { -#ifndef UNIV_HOTBACKUP page_zip_compress_write_log(page_zip, page, index, mtr); -#endif /* !UNIV_HOTBACKUP */ } UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); @@ -1652,7 +1631,6 @@ err_exit: fclose(logfile); } #endif /* PAGE_ZIP_COMPRESS_DBG */ -#ifndef UNIV_HOTBACKUP uintmax_t time_diff = ut_time_us(NULL) - usec; page_zip_stat[page_zip->ssize - 1].compressed_ok++; page_zip_stat[page_zip->ssize - 1].compressed_usec += time_diff; @@ -1666,7 +1644,6 @@ err_exit: if (page_is_leaf(page) && !truncate_t::s_fix_up_active) { dict_index_zip_success(index); } -#endif /* !UNIV_HOTBACKUP */ return(TRUE); } @@ -3277,15 +3254,12 @@ page_zip_decompress( page header fields that should not change after page creation */ { -#ifndef UNIV_HOTBACKUP uintmax_t usec = ut_time_us(NULL); -#endif /* !UNIV_HOTBACKUP */ if (!page_zip_decompress_low(page_zip, page, all)) { return(FALSE); } -#ifndef UNIV_HOTBACKUP uintmax_t time_diff = ut_time_us(NULL) - usec; page_zip_stat[page_zip->ssize - 1].decompressed++; page_zip_stat[page_zip->ssize - 1].decompressed_usec += time_diff; @@ -3298,7 +3272,6 @@ page_zip_decompress( page_zip_stat_per_index[index_id].decompressed_usec += time_diff; mutex_exit(&page_zip_stat_per_index_mutex); } -#endif /* !UNIV_HOTBACKUP */ /* Update the stat counter for LRU policy. */ buf_LRU_stat_inc_unzip(); @@ -4013,7 +3986,6 @@ page_zip_write_blob_ptr( #endif /* UNIV_ZIP_DEBUG */ if (mtr) { -#ifndef UNIV_HOTBACKUP byte* log_ptr = mlog_open( mtr, 11 + 2 + 2 + BTR_EXTERN_FIELD_REF_SIZE); if (UNIV_UNLIKELY(!log_ptr)) { @@ -4029,7 +4001,6 @@ page_zip_write_blob_ptr( memcpy(log_ptr, externs, BTR_EXTERN_FIELD_REF_SIZE); log_ptr += BTR_EXTERN_FIELD_REF_SIZE; mlog_close(mtr, log_ptr); -#endif /* !UNIV_HOTBACKUP */ } } @@ -4153,7 +4124,6 @@ page_zip_write_node_ptr( memcpy(storage, field, REC_NODE_PTR_SIZE); if (mtr) { -#ifndef UNIV_HOTBACKUP byte* log_ptr = mlog_open(mtr, 11 + 2 + 2 + REC_NODE_PTR_SIZE); if (UNIV_UNLIKELY(!log_ptr)) { @@ -4169,7 +4139,6 @@ page_zip_write_node_ptr( memcpy(log_ptr, field, REC_NODE_PTR_SIZE); log_ptr += REC_NODE_PTR_SIZE; mlog_close(mtr, log_ptr); -#endif /* !UNIV_HOTBACKUP */ } } @@ -4654,7 +4623,6 @@ corrupt: return(ptr + len); } -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Write a log record of writing to the uncompressed header portion of a page. */ void @@ -4688,7 +4656,6 @@ page_zip_write_header_log( mlog_catenate_string(mtr, data, length); } -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Reorganize and compress a page. This is a low-level operation for @@ -4711,9 +4678,7 @@ page_zip_reorganize( dict_index_t* index, /*!< in: index of the B-tree node */ mtr_t* mtr) /*!< in: mini-transaction */ { -#ifndef UNIV_HOTBACKUP buf_pool_t* buf_pool = buf_pool_from_block(block); -#endif /* !UNIV_HOTBACKUP */ page_zip_des_t* page_zip = buf_block_get_page_zip(block); page_t* page = buf_block_get_frame(block); buf_block_t* temp_block; @@ -4722,6 +4687,7 @@ page_zip_reorganize( ut_ad(mtr_memo_contains(mtr, block, MTR_MEMO_PAGE_X_FIX)); ut_ad(page_is_comp(page)); ut_ad(!dict_index_is_ibuf(index)); + ut_ad(!dict_table_is_temporary(index->table)); /* Note that page_zip_validate(page_zip, page, index) may fail here. */ UNIV_MEM_ASSERT_RW(page, UNIV_PAGE_SIZE); UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); @@ -4729,13 +4695,8 @@ page_zip_reorganize( /* Disable logging */ mtr_log_t log_mode = mtr_set_log_mode(mtr, MTR_LOG_NONE); -#ifndef UNIV_HOTBACKUP temp_block = buf_block_alloc(buf_pool); btr_search_drop_page_hash_index(block); -#else /* !UNIV_HOTBACKUP */ - ut_ad(block == back_block1); - temp_block = back_block2; -#endif /* !UNIV_HOTBACKUP */ temp_page = temp_block->frame; /* Copy the old page to temporary space */ @@ -4753,19 +4714,18 @@ page_zip_reorganize( page_get_infimum_rec(temp_page), index, mtr); - /* Temp-Tables are not shared across connection and so we avoid - locking of temp-tables as there would be no 2 trx trying to - operate on same temp-table in parallel. - max_trx_id is use to track which all trxs wrote to the page - in parallel but in case of temp-table this can is not needed. */ - if (!dict_index_is_clust(index) - && !dict_table_is_temporary(index->table) - && page_is_leaf(temp_page)) { - /* Copy max trx id to recreated page */ - trx_id_t max_trx_id = page_get_max_trx_id(temp_page); - page_set_max_trx_id(block, NULL, max_trx_id, NULL); - ut_ad(max_trx_id != 0); - } + /* Copy the PAGE_MAX_TRX_ID or PAGE_ROOT_AUTO_INC. */ + memcpy(page + (PAGE_HEADER + PAGE_MAX_TRX_ID), + temp_page + (PAGE_HEADER + PAGE_MAX_TRX_ID), 8); + /* PAGE_MAX_TRX_ID must be set on secondary index leaf pages. */ + ut_ad(dict_index_is_clust(index) || !page_is_leaf(temp_page) + || page_get_max_trx_id(page) != 0); + /* PAGE_MAX_TRX_ID must be zero on non-leaf pages other than + clustered index root pages. */ + ut_ad(page_get_max_trx_id(page) == 0 + || (dict_index_is_clust(index) + ? page_is_root(temp_page) + : page_is_leaf(temp_page))); /* Restore logging. */ mtr_set_log_mode(mtr, log_mode); @@ -4773,21 +4733,16 @@ page_zip_reorganize( if (!page_zip_compress(page_zip, page, index, page_zip_level, NULL, mtr)) { -#ifndef UNIV_HOTBACKUP buf_block_free(temp_block); -#endif /* !UNIV_HOTBACKUP */ return(FALSE); } lock_move_reorganize_page(block, temp_block); -#ifndef UNIV_HOTBACKUP buf_block_free(temp_block); -#endif /* !UNIV_HOTBACKUP */ return(TRUE); } -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Copy the records of a page byte for byte. Do not copy the page header or trailer, except those B-tree header fields that are directly @@ -4805,11 +4760,10 @@ page_zip_copy_recs( dict_index_t* index, /*!< in: index of the B-tree */ mtr_t* mtr) /*!< in: mini-transaction */ { - ut_ad(mtr_memo_contains_page(mtr, page, MTR_MEMO_PAGE_X_FIX) - || dict_table_is_intrinsic(index->table)); - ut_ad(mtr_memo_contains_page(mtr, src, MTR_MEMO_PAGE_X_FIX) - || dict_table_is_intrinsic(index->table)); + ut_ad(mtr_memo_contains_page(mtr, page, MTR_MEMO_PAGE_X_FIX)); + ut_ad(mtr_memo_contains_page(mtr, src, MTR_MEMO_PAGE_X_FIX)); ut_ad(!dict_index_is_ibuf(index)); + ut_ad(!dict_table_is_temporary(index->table)); #ifdef UNIV_ZIP_DEBUG /* The B-tree operations that call this function may set FIL_PAGE_PREV or PAGE_LEVEL, causing a temporary min_rec_flag @@ -4823,13 +4777,6 @@ page_zip_copy_recs( ut_a(dict_index_is_clust(index)); } - /* The PAGE_MAX_TRX_ID must be set on leaf pages of secondary - indexes. It does not matter on other pages. */ - ut_a(dict_index_is_clust(index) - || dict_table_is_temporary(index->table) - || !page_is_leaf(src) - || page_get_max_trx_id(src)); - UNIV_MEM_ASSERT_W(page, UNIV_PAGE_SIZE); UNIV_MEM_ASSERT_W(page_zip->data, page_zip_get_size(page_zip)); UNIV_MEM_ASSERT_RW(src, UNIV_PAGE_SIZE); @@ -4851,6 +4798,18 @@ page_zip_copy_recs( memcpy(PAGE_DATA + page_zip->data, PAGE_DATA + src_zip->data, page_zip_get_size(page_zip) - PAGE_DATA); + if (dict_index_is_clust(index)) { + /* Reset the PAGE_ROOT_AUTO_INC field when copying + from a root page. */ + memset(PAGE_HEADER + PAGE_ROOT_AUTO_INC + page, 0, 8); + memset(PAGE_HEADER + PAGE_ROOT_AUTO_INC + page_zip->data, + 0, 8); + } else { + /* The PAGE_MAX_TRX_ID must be nonzero on leaf pages + of secondary indexes, and 0 on others. */ + ut_ad(!page_is_leaf(src) == !page_get_max_trx_id(src)); + } + /* Copy all fields of src_zip to page_zip, except the pointer to the compressed data page. */ { @@ -4881,7 +4840,6 @@ page_zip_copy_recs( #endif /* UNIV_ZIP_DEBUG */ page_zip_compress_write_log(page_zip, page, index, mtr); } -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Parses a log record of compressing an index page. @@ -5243,13 +5201,15 @@ page_zip_verify_checksum( const uint32_t calculated = page_zip_calc_checksum( data, size, SRV_CHECKSUM_ALGORITHM_CRC32); - const uint32_t calculated1 = page_zip_calc_checksum( - data, size, SRV_CHECKSUM_ALGORITHM_CRC32, true); + uint32_t calculated1; if (stored == calculated - || stored == calculated1 #ifdef UNIV_INNOCHECKSUM || ( encrypted == true && checksum == calculated) +#endif + || stored == (calculated1 = + page_zip_calc_checksum(data, size, SRV_CHECKSUM_ALGORITHM_CRC32, true)) +#ifdef UNIV_INNOCHECKSUM || ( encrypted == true && checksum == calculated1) #endif ) { diff --git a/storage/innobase/que/que0que.cc b/storage/innobase/que/que0que.cc index 8d3e8cfa115..9fd12686f57 100644 --- a/storage/innobase/que/que0que.cc +++ b/storage/innobase/que/que0que.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1248,21 +1249,3 @@ que_eval_sql( DBUG_RETURN(trx->error_state); } - -/*********************************************************************//** -Initialise the query sub-system. */ -void -que_init(void) -/*==========*/ -{ - /* No op */ -} - -/*********************************************************************//** -Close the query sub-system. */ -void -que_close(void) -/*===========*/ -{ - /* No op */ -} diff --git a/storage/innobase/rem/rem0rec.cc b/storage/innobase/rem/rem0rec.cc index d055aa5d84e..920fdb62079 100644 --- a/storage/innobase/rem/rem0rec.cc +++ b/storage/innobase/rem/rem0rec.cc @@ -1503,7 +1503,6 @@ rec_convert_dtuple_to_rec( return(rec); } -#ifndef UNIV_HOTBACKUP /**********************************************************//** Determines the size of a data tuple prefix in ROW_FORMAT=COMPACT. @return total size */ @@ -1763,7 +1762,6 @@ rec_copy_prefix_to_buf( return(*buf + (rec - (lens + 1))); } -#endif /* UNIV_HOTBACKUP */ /***************************************************************//** Validates the consistency of an old-style physical record. @@ -1930,7 +1928,6 @@ rec_print_old( rec_validate_old(rec); } -#ifndef UNIV_HOTBACKUP /***************************************************************//** Prints a physical record in ROW_FORMAT=COMPACT. Ignores the record header. */ @@ -2268,7 +2265,7 @@ operator<<(std::ostream& o, const rec_offsets_print& r) return(o); } -# ifdef UNIV_DEBUG +#ifdef UNIV_DEBUG /************************************************************//** Reads the DB_TRX_ID of a clustered index record. @return the value of DB_TRX_ID */ @@ -2308,8 +2305,7 @@ rec_get_trx_id( return(trx_read_trx_id(trx_id)); } -# endif /* UNIV_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ +#endif /* UNIV_DEBUG */ /** Mark the nth field as externally stored. @param[in] offsets array returned by rec_get_offsets() diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index 7729d4c5c30..3179c63f162 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -63,6 +63,8 @@ tokenized doc string. The index has three "fields": integer value) 3) Word's position in original doc. +@see fts_create_one_index_table() + @return dict_index_t structure for the fts sort index */ dict_index_t* row_merge_create_fts_sort_index( @@ -103,16 +105,12 @@ row_merge_create_fts_sort_index( field->prefix_len = 0; field->col = static_cast<dict_col_t*>( mem_heap_alloc(new_index->heap, sizeof(dict_col_t))); - field->col->len = FTS_MAX_WORD_LEN; - - if (strcmp(charset->name, "latin1_swedish_ci") == 0) { - field->col->mtype = DATA_VARCHAR; - } else { - field->col->mtype = DATA_VARMYSQL; - } - field->col->prtype = idx_field->col->prtype | DATA_NOT_NULL; + field->col->mtype = charset == &my_charset_latin1 + ? DATA_VARCHAR : DATA_VARMYSQL; field->col->mbminmaxlen = idx_field->col->mbminmaxlen; + field->col->len = HA_FT_MAXCHARLEN * DATA_MBMAXLEN(field->col->mbminmaxlen); + field->fixed_len = 0; /* Doc ID */ @@ -229,10 +227,7 @@ row_fts_psort_info_init( common_info->opt_doc_id_size = opt_doc_id_size; crypt_data = fil_space_get_crypt_data(new_table->space); - if ((crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) || - (srv_encrypt_tables && - crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { - + if (crypt_data && crypt_data->should_encrypt()) { common_info->crypt_data = crypt_data; encrypted = true; } else { @@ -487,7 +482,8 @@ row_merge_fts_doc_tokenize_by_parser( /*********************************************************************//** Tokenize incoming text data and add to the sort buffer. -@return TRUE if the record passed, FALSE if out of space */ +@see row_merge_buf_encode() +@return TRUE if the record passed, FALSE if out of space */ static ibool row_merge_fts_doc_tokenize( @@ -495,8 +491,6 @@ row_merge_fts_doc_tokenize( row_merge_buf_t** sort_buf, /*!< in/out: sort buffer */ doc_id_t doc_id, /*!< in: Doc ID */ fts_doc_t* doc, /*!< in: Doc to be tokenized */ - dtype_t* word_dtype, /*!< in: data structure for - word col */ merge_file_t** merge_file, /*!< in/out: merge file */ ibool opt_doc_id_size,/*!< in: whether to use 4 bytes instead of 8 bytes integer to @@ -530,7 +524,7 @@ row_merge_fts_doc_tokenize( while (t_ctx->processed_len < doc->text.f_len) { ulint idx = 0; ib_uint32_t position; - ulint cur_len = 0; + ulint cur_len; doc_id_t write_doc_id; row_fts_token_t* fts_token = NULL; @@ -620,14 +614,34 @@ row_merge_fts_doc_tokenize( dfield_set_data(field, t_str.f_str, t_str.f_len); len = dfield_get_len(field); - field->type.mtype = word_dtype->mtype; - field->type.prtype = word_dtype->prtype | DATA_NOT_NULL; + dict_col_copy_type(dict_index_get_nth_col(buf->index, 0), &field->type); + field->type.prtype |= DATA_NOT_NULL; + ut_ad(len <= field->type.len); - /* Variable length field, set to max size. */ - field->type.len = FTS_MAX_WORD_LEN; - field->type.mbminmaxlen = word_dtype->mbminmaxlen; + /* For the temporary file, row_merge_buf_encode() uses + 1 byte for representing the number of extra_size bytes. + This number will always be 1, because for this 3-field index + consisting of one variable-size column, extra_size will always + be 1 or 2, which can be encoded in one byte. + + The extra_size is 1 byte if the length of the + variable-length column is less than 128 bytes or the + maximum length is less than 256 bytes. */ + + /* One variable length column, word with its lenght less than + fts_max_token_size, add one extra size and one extra byte. + + Since the max length for FTS token now is larger than 255, + so we will need to signify length byte itself, so only 1 to 128 + bytes can be used for 1 bytes, larger than that 2 bytes. */ + if (len < 128 || field->type.len < 256) { + /* Extra size is one byte. */ + cur_len = 2 + len; + } else { + /* Extra size is two bytes. */ + cur_len = 3 + len; + } - cur_len += len; dfield_dup(field, buf->heap); field++; @@ -683,20 +697,6 @@ row_merge_fts_doc_tokenize( cur_len += len; dfield_dup(field, buf->heap); - /* One variable length column, word with its lenght less than - fts_max_token_size, add one extra size and one extra byte. - - Since the max length for FTS token now is larger than 255, - so we will need to signify length byte itself, so only 1 to 128 - bytes can be used for 1 bytes, larger than that 2 bytes. */ - if (t_str.f_len < 128) { - /* Extra size is one byte. */ - cur_len += 2; - } else { - /* Extra size is two bytes. */ - cur_len += 3; - } - /* Reserve one byte for the end marker of row_merge_block_t and we have reserved ROW_MERGE_RESERVE_SIZE (= 4) for encryption key_version in the beginning of the buffer. */ @@ -793,8 +793,6 @@ fts_parallel_tokenization( mem_heap_t* blob_heap = NULL; fts_doc_t doc; dict_table_t* table = psort_info->psort_common->new_table; - dtype_t word_dtype; - dict_field_t* idx_field; fts_tokenize_ctx_t t_ctx; ulint retried = 0; dberr_t error = DB_SUCCESS; @@ -823,13 +821,6 @@ fts_parallel_tokenization( doc.charset = fts_index_get_charset( psort_info->psort_common->dup->index); - idx_field = dict_index_get_nth_field( - psort_info->psort_common->dup->index, 0); - word_dtype.prtype = idx_field->col->prtype; - word_dtype.mbminmaxlen = idx_field->col->mbminmaxlen; - word_dtype.mtype = (strcmp(doc.charset->name, "latin1_swedish_ci") == 0) - ? DATA_VARCHAR : DATA_VARMYSQL; - block = psort_info->merge_block; crypt_block = psort_info->crypt_block; crypt_data = psort_info->psort_common->crypt_data; @@ -881,7 +872,6 @@ loop: processed = row_merge_fts_doc_tokenize( buf, doc_item->doc_id, &doc, - &word_dtype, merge_file, psort_info->psort_common->opt_doc_id_size, &t_ctx); diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index 1ae11204f69..a6f4b221302 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -124,8 +124,7 @@ struct row_import { m_col_names(), m_n_indexes(), m_indexes(), - m_missing(true), - m_cfp_missing(true) { } + m_missing(true) {} ~row_import() UNIV_NOTHROW; @@ -220,9 +219,6 @@ struct row_import { bool m_missing; /*!< true if a .cfg file was found and was readable */ - - bool m_cfp_missing; /*!< true if a .cfp file was - found and was readable */ }; /** Use the page cursor to iterate over records in a block. */ @@ -365,8 +361,7 @@ public: m_space(ULINT_UNDEFINED), m_xdes(), m_xdes_page_no(ULINT_UNDEFINED), - m_space_flags(ULINT_UNDEFINED), - m_table_flags(ULINT_UNDEFINED) UNIV_NOTHROW { } + m_space_flags(ULINT_UNDEFINED) UNIV_NOTHROW { } /** Free any extent descriptor instance */ virtual ~AbstractCallback() @@ -388,6 +383,12 @@ public: return(get_page_size().is_compressed()); } + /** @return the tablespace flags */ + ulint get_space_flags() const + { + return(m_space_flags); + } + protected: /** Get the data page depending on the table type, compressed or not. @param block block read from disk @@ -527,10 +528,6 @@ protected: /** Flags value read from the header page */ ulint m_space_flags; - - /** Derived from m_space_flags and row format type, the row format - type is determined from the page header. */ - ulint m_table_flags; }; /** Determine the page size to use for traversing the tablespace @@ -545,13 +542,19 @@ AbstractCallback::init( const page_t* page = block->frame; m_space_flags = fsp_header_get_flags(page); + if (!fsp_flags_is_valid(m_space_flags)) { + ulint cflags = fsp_flags_convert_from_101(m_space_flags); + if (cflags == ULINT_UNDEFINED) { + ib::error() << "Invalid FSP_SPACE_FLAGS=" + << ib::hex(m_space_flags); + return(DB_CORRUPTION); + } + m_space_flags = cflags; + } - /* Since we don't know whether it is a compressed table - or not, the data is always read into the block->frame. */ - - set_page_size(block->frame); - - /* Set the page size used to traverse the tablespace. */ + /* Clear the DATA_DIR flag, which is basically garbage. */ + m_space_flags &= ~(1U << FSP_FLAGS_POS_RESERVED); + m_page_size.copy_from(page_size_t(m_space_flags)); if (!is_compressed_table() && !m_page_size.equals_to(univ_page_size)) { @@ -618,52 +621,6 @@ struct FetchIndexRootPages : public AbstractCallback { return(m_space); } - /** - @retval the space flags of the tablespace being iterated over */ - virtual ulint get_space_flags() const UNIV_NOTHROW - { - return(m_space_flags); - } - - /** Check if the .ibd file row format is the same as the table's. - @param ibd_table_flags determined from space and page. - @return DB_SUCCESS or error code. */ - dberr_t check_row_format(ulint ibd_table_flags) UNIV_NOTHROW - { - dberr_t err; - rec_format_t ibd_rec_format; - rec_format_t table_rec_format; - - if (!dict_tf_is_valid(ibd_table_flags)) { - - ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - ".ibd file has invalid table flags: %lx", - ibd_table_flags); - - return(DB_CORRUPTION); - } - - ibd_rec_format = dict_tf_get_rec_format(ibd_table_flags); - table_rec_format = dict_tf_get_rec_format(m_table->flags); - - if (table_rec_format != ibd_rec_format) { - - ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Table has %s row format, .ibd" - " file has %s row format.", - dict_tf_to_row_format_string(m_table->flags), - dict_tf_to_row_format_string(ibd_table_flags)); - - err = DB_CORRUPTION; - } else { - err = DB_SUCCESS; - } - - return(err); - } - /** Called for each block as it is read from the file. @param offset physical offset in the file @param block block to convert, it is not from the buffer pool. @@ -724,12 +681,17 @@ FetchIndexRootPages::operator() ( m_indexes.push_back(Index(id, block->page.id.page_no())); if (m_indexes.size() == 1) { - - m_table_flags = fsp_flags_to_dict_tf( - m_space_flags, - page_is_comp(page) ? true : false); - - err = check_row_format(m_table_flags); + /* Check that the tablespace flags match the table flags. */ + ulint expected = dict_tf_to_fsp_flags(m_table->flags); + if (!fsp_flags_match(expected, m_space_flags)) { + ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, + ER_TABLE_SCHEMA_MISMATCH, + "Expected FSP_SPACE_FLAGS=0x%x, .ibd " + "file contains 0x%x.", + unsigned(expected), + unsigned(m_space_flags)); + return(DB_CORRUPTION); + } } } @@ -853,13 +815,6 @@ public: return(m_cfg->m_table->space); } - /** - @retval the space flags of the tablespace being iterated over */ - virtual ulint get_space_flags() const UNIV_NOTHROW - { - return(m_space_flags); - } - /** Called for each block as it is read from the file. @param offset physical offset in the file @param block block to convert, it is not from the buffer pool. @@ -1852,6 +1807,11 @@ PageConverter::update_index_page( row_index_t* index = find_index(id); if (index == 0) { + ib::error() << "Page for tablespace " << m_space + << " is index page with id " << id + << " but that index is not found from" + << " configuration file. Current index name " + << m_index->m_name << " and id " << m_index->m_id; m_index = 0; return(DB_CORRUPTION); } @@ -1909,19 +1869,14 @@ PageConverter::update_header( ib::warn() << "Space id check in the header failed: ignored"; } - ulint space_flags = fsp_header_get_flags(get_frame(block)); - - if (!fsp_flags_is_valid(space_flags)) { - - ib::error() << "Unsupported tablespace format " - << space_flags; - - return(DB_UNSUPPORTED); - } - mach_write_to_8( get_frame(block) + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, m_current_lsn); + + /* Write back the adjusted flags. */ + mach_write_to_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + + get_frame(block), m_space_flags); + /* Write space_id to the tablespace header, page 0. */ mach_write_to_4( get_frame(block) + FSP_HEADER_OFFSET + FSP_SPACE_ID, @@ -2177,9 +2132,6 @@ row_import_cleanup( trx_commit_for_mysql(trx); - prebuilt->table->encryption_key = NULL; - prebuilt->table->encryption_iv = NULL; - row_mysql_unlock_data_dictionary(trx); trx_free_for_mysql(trx); @@ -3163,170 +3115,6 @@ row_import_read_cfg( return(err); } -#ifdef MYSQL_ENCRYPTION -/** Read the contents of the <tablespace>.cfp file. -@param[in] table table -@param[in] file file to read from -@param[in] thd session -@param[in] cfp contents of the .cfp file -@return DB_SUCCESS or error code. */ -static -dberr_t -row_import_read_encryption_data( - dict_table_t* table, - FILE* file, - THD* thd, - row_import& import) -{ - byte row[sizeof(ib_uint32_t)]; - ulint key_size; - byte transfer_key[ENCRYPTION_KEY_LEN]; - byte encryption_key[ENCRYPTION_KEY_LEN]; - byte encryption_iv[ENCRYPTION_KEY_LEN]; - lint elen; - - if (fread(&row, 1, sizeof(row), file) != sizeof(row)) { - ib_senderrf( - thd, IB_LOG_LEVEL_ERROR, ER_IO_READ_ERROR, - errno, strerror(errno), - "while reading encrypton key size."); - - return(DB_IO_ERROR); - } - - key_size = mach_read_from_4(row); - if (key_size != ENCRYPTION_KEY_LEN) { - ib_senderrf( - thd, IB_LOG_LEVEL_ERROR, ER_IO_READ_ERROR, - errno, strerror(errno), - "while parsing encryption key size."); - - return(DB_IO_ERROR); - } - - /* Read the transfer key. */ - if (fread(transfer_key, 1, ENCRYPTION_KEY_LEN, file) - != ENCRYPTION_KEY_LEN) { - ib_senderrf( - thd, IB_LOG_LEVEL_WARN, ER_IO_WRITE_ERROR, - errno, strerror(errno), - "while reading tranfer key."); - - return(DB_IO_ERROR); - } - - /* Read the encrypted key. */ - if (fread(encryption_key, 1, ENCRYPTION_KEY_LEN, file) - != ENCRYPTION_KEY_LEN) { - ib_senderrf( - thd, IB_LOG_LEVEL_WARN, ER_IO_WRITE_ERROR, - errno, strerror(errno), - "while reading encryption key."); - - return(DB_IO_ERROR); - } - - /* Read the encrypted iv. */ - if (fread(encryption_iv, 1, ENCRYPTION_KEY_LEN, file) - != ENCRYPTION_KEY_LEN) { - ib_senderrf( - thd, IB_LOG_LEVEL_WARN, ER_IO_WRITE_ERROR, - errno, strerror(errno), - "while reading encryption iv."); - - return(DB_IO_ERROR); - } - - table->encryption_key = - static_cast<byte*>(mem_heap_alloc(table->heap, - ENCRYPTION_KEY_LEN)); - - table->encryption_iv = - static_cast<byte*>(mem_heap_alloc(table->heap, - ENCRYPTION_KEY_LEN)); - /* Decrypt tablespace key and iv. */ - elen = my_aes_decrypt( - encryption_key, - ENCRYPTION_KEY_LEN, - table->encryption_key, - transfer_key, - ENCRYPTION_KEY_LEN, - my_aes_256_ecb, NULL, false); - - if (elen == MY_AES_BAD_DATA) { - ib_senderrf( - thd, IB_LOG_LEVEL_ERROR, ER_IO_READ_ERROR, - errno, strerror(errno), - "while decrypt encryption key."); - - return(DB_IO_ERROR); - } - - elen = my_aes_decrypt( - encryption_iv, - ENCRYPTION_KEY_LEN, - table->encryption_iv, - transfer_key, - ENCRYPTION_KEY_LEN, - my_aes_256_ecb, NULL, false); - - if (elen == MY_AES_BAD_DATA) { - ib_senderrf( - thd, IB_LOG_LEVEL_ERROR, ER_IO_READ_ERROR, - errno, strerror(errno), - "while decrypt encryption iv."); - - return(DB_IO_ERROR); - } - - return(DB_SUCCESS); -} - -/** Read the contents of the <tablename>.cfp file. -@param[in] table table -@param[in] thd session -@param[in] cfp contents of the .cfp file -@return DB_SUCCESS or error code. */ -static -dberr_t -row_import_read_cfp( - dict_table_t* table, - THD* thd, - row_import& import) -{ - dberr_t err; - char name[OS_FILE_MAX_PATH]; - - /* Clear table encryption information. */ - table->encryption_key = NULL; - table->encryption_iv = NULL; - - srv_get_encryption_data_filename(table, name, sizeof(name)); - - FILE* file = fopen(name, "rb"); - - if (file == NULL) { - import.m_cfp_missing = true; - - /* If there's no cfp file, we assume it's not an - encrpyted table. return directly. */ - - import.m_cfp_missing = true; - - err = DB_SUCCESS; - } else { - - import.m_cfp_missing = false; - - err = row_import_read_encryption_data(table, file, - thd, import); - fclose(file); - } - - return(err); -} -#endif /* MYSQL_ENCRYPTION */ - /*****************************************************************//** Update the <space, root page> of a table's indexes from the values in the data dictionary. @@ -3694,42 +3482,7 @@ row_import_for_mysql( rw_lock_s_unlock_gen(dict_operation_lock, 0); } - /* Try to read encryption information. */ - if (err == DB_SUCCESS) { -#ifdef MYSQL_ENCRYPTION - err = row_import_read_cfp(table, trx->mysql_thd, cfg); - - /* If table is not set to encrypted, but the fsp flag - is not, then return error. */ - if (!dict_table_is_encrypted(table) - && space_flags != 0 - && FSP_FLAGS_GET_ENCRYPTION(space_flags)) { - - ib_errf(trx->mysql_thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Table is not marked as encrypted, but" - " the tablespace is marked as encrypted"); - - err = DB_ERROR; - return(row_import_error(prebuilt, trx, err)); - } - - /* If table is set to encrypted, but can't find - cfp file, then return error. */ - if (cfg.m_cfp_missing== true - && ((space_flags != 0 - && FSP_FLAGS_GET_ENCRYPTION(space_flags)) - || dict_table_is_encrypted(table))) { - ib_errf(trx->mysql_thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Table is in an encrypted tablespace, but" - " can't find the encryption meta-data file" - " in importing"); - err = DB_ERROR; - return(row_import_error(prebuilt, trx, err)); - } -#endif /* MYSQL_ENCRYPTION */ - } else { + if (err != DB_SUCCESS) { return(row_import_error(prebuilt, trx, err)); } @@ -3752,22 +3505,6 @@ row_import_for_mysql( DBUG_EXECUTE_IF("ib_import_reset_space_and_lsn_failure", err = DB_TOO_MANY_CONCURRENT_TRXS;); -#ifdef MYSQL_ENCRYPTION - if (err == DB_IO_NO_ENCRYPT_TABLESPACE) { - char table_name[MAX_FULL_NAME_LEN + 1]; - - innobase_format_name( - table_name, sizeof(table_name), - table->name.m_name); - - ib_errf(trx->mysql_thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Encryption attribute is no matched"); - - return(row_import_cleanup(prebuilt, trx, err)); - } -#endif /* MYSQL_ENCRYPTION */ - if (err != DB_SUCCESS) { char table_name[MAX_FULL_NAME_LEN + 1]; @@ -3821,13 +3558,7 @@ row_import_for_mysql( we will not be writing any redo log for it before we have invoked fil_space_set_imported() to declare it a persistent tablespace. */ - ulint fsp_flags = dict_tf_to_fsp_flags(table->flags, false); - -#ifdef MYSQL_ENCRYPTION - if (table->encryption_key != NULL) { - fsp_flags |= FSP_FLAGS_MASK_ENCRYPTION; - } -#endif /* MYSQL_ENCRYPTION */ + ulint fsp_flags = dict_tf_to_fsp_flags(table->flags); err = fil_ibd_open( true, true, FIL_TYPE_IMPORT, table->space, @@ -3848,17 +3579,6 @@ row_import_for_mysql( return(row_import_cleanup(prebuilt, trx, err)); } -#ifdef MYSQL_ENCRYPTION - /* For encrypted table, set encryption information. */ - if (dict_table_is_encrypted(table)) { - - err = fil_set_encryption(table->space, - Encryption::AES, - table->encryption_key, - table->encryption_iv); - } -#endif /* MYSQL_ENCRYPTION */ - row_mysql_unlock_data_dictionary(trx); ut_free(filepath); @@ -3956,37 +3676,6 @@ row_import_for_mysql( ib::info() << "Phase IV - Flush complete"; fil_space_set_imported(prebuilt->table->space); -#ifdef MYSQL_ENCRYPTION - if (dict_table_is_encrypted(table)) { - fil_space_t* space; - mtr_t mtr; - byte encrypt_info[ENCRYPTION_INFO_SIZE_V2]; - - mtr_start(&mtr); - - mtr.set_named_space(table->space); - space = mtr_x_lock_space(table->space, &mtr); - - memset(encrypt_info, 0, ENCRYPTION_INFO_SIZE_V2); - - if (!fsp_header_rotate_encryption(space, - encrypt_info, - &mtr)) { - mtr_commit(&mtr); - ib_senderrf(trx->mysql_thd, IB_LOG_LEVEL_ERROR, - ER_FILE_NOT_FOUND, - filepath, err, ut_strerr(err)); - - ut_free(filepath); - row_mysql_unlock_data_dictionary(trx); - - return(row_import_cleanup(prebuilt, trx, err)); - } - - mtr_commit(&mtr); - } -#endif /* MYSQL_ENCRYPTION */ - /* The dictionary latches will be released in in row_import_cleanup() after the transaction commit, for both success and error. */ @@ -4009,18 +3698,15 @@ row_import_for_mysql( table->ibd_file_missing = false; table->flags2 &= ~DICT_TF2_DISCARDED; - /* Set autoinc value read from cfg file. The value is set to zero - if the cfg file is missing and is initialized later from table - column value. */ - ib::info() << table->name << " autoinc value set to " - << autoinc; - - dict_table_autoinc_lock(table); - dict_table_autoinc_initialize(table, autoinc); - dict_table_autoinc_unlock(table); + /* Set autoinc value read from .cfg file, if one was specified. + Otherwise, keep the PAGE_ROOT_AUTO_INC as is. */ + if (autoinc) { + ib::info() << table->name << " autoinc value set to " + << autoinc; - ut_a(err == DB_SUCCESS); + table->autoinc = autoinc--; + btr_write_autoinc(dict_table_get_first_index(table), autoinc); + } return(row_import_cleanup(prebuilt, trx, err)); } - diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index 720ad44b4b8..1f884017dd3 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -155,10 +156,7 @@ row_ins_alloc_sys_fields( ut_ad(dtuple_get_n_fields(row) == dict_table_get_n_cols(table)); /* allocate buffer to hold the needed system created hidden columns. */ - uint len = DATA_ROW_ID_LEN + DATA_TRX_ID_LEN; - if (!dict_table_is_intrinsic(table)) { - len += DATA_ROLL_PTR_LEN; - } + const uint len = DATA_ROW_ID_LEN + DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN; ptr = static_cast<byte*>(mem_heap_zalloc(heap, len)); /* 1. Populate row-id */ @@ -183,13 +181,11 @@ row_ins_alloc_sys_fields( ptr += DATA_TRX_ID_LEN; - if (!dict_table_is_intrinsic(table)) { - col = dict_table_get_sys_col(table, DATA_ROLL_PTR); + col = dict_table_get_sys_col(table, DATA_ROLL_PTR); - dfield = dtuple_get_nth_field(row, dict_col_get_no(col)); + dfield = dtuple_get_nth_field(row, dict_col_get_no(col)); - dfield_set_data(dfield, ptr, DATA_ROLL_PTR_LEN); - } + dfield_set_data(dfield, ptr, DATA_ROLL_PTR_LEN); } /*********************************************************************//** @@ -936,7 +932,6 @@ row_ins_invalidate_query_cache( innobase_invalidate_query_cache(thr_get_trx(thr), name, len); } -#ifdef MYSQL_VIRTUAL_COLUMNS /** Fill virtual column information in cascade node for the child table. @param[out] cascade child update node @@ -1047,7 +1042,6 @@ func_exit: mem_heap_free(v_heap); } } -#endif /* MYSQL_VIRTUAL_COLUMNS */ #ifdef WITH_WSREP dberr_t wsrep_append_foreign_key(trx_t *trx, @@ -1343,7 +1337,6 @@ row_ins_foreign_check_on_constraint( cascade->fts_doc_id = doc_id; } -#ifdef MYSQL_VIRTUAL_COLUMNS if (foreign->v_cols != NULL && foreign->v_cols->size() > 0) { row_ins_foreign_fill_virtual( @@ -1354,7 +1347,6 @@ row_ins_foreign_check_on_constraint( goto nonstandard_exit_func; } } -#endif /* MYSQL_VIRTUAL_COLUMNS */ } else if (table->fts && cascade->is_delete) { /* DICT_FOREIGN_ON_DELETE_CASCADE case */ for (i = 0; i < foreign->n_fields; i++) { @@ -1384,7 +1376,6 @@ row_ins_foreign_check_on_constraint( trx, &fts_col_affacted, cascade); -#ifdef MYSQL_VIRTUAL_COLUMNS if (foreign->v_cols != NULL && foreign->v_cols->size() > 0) { row_ins_foreign_fill_virtual( @@ -1395,7 +1386,6 @@ row_ins_foreign_check_on_constraint( goto nonstandard_exit_func; } } -#endif /* MYSQL_VIRTUAL_COLUMNS */ if (n_to_update == ULINT_UNDEFINED) { err = DB_ROW_IS_REFERENCED; @@ -2152,7 +2142,7 @@ row_ins_scan_sec_index_for_duplicate( cmp = cmp_dtuple_rec(entry, rec, offsets); - if (cmp == 0 && !index->allow_duplicates) { + if (cmp == 0) { if (row_ins_dupl_error_with_rec(rec, entry, index, offsets)) { err = DB_DUPLICATE_KEY; @@ -2174,7 +2164,7 @@ row_ins_scan_sec_index_for_duplicate( goto end_scan; } } else { - ut_a(cmp < 0 || index->allow_duplicates); + ut_a(cmp < 0); goto end_scan; } } while (btr_pcur_move_to_next(&pcur, mtr)); @@ -2484,6 +2474,7 @@ row_ins_clust_index_entry_low( dberr_t err = DB_SUCCESS; big_rec_t* big_rec = NULL; mtr_t mtr; + ib_uint64_t auto_inc = 0; mem_heap_t* offsets_heap = NULL; ulint offsets_[REC_OFFS_NORMAL_SIZE]; ulint* offsets = offsets_; @@ -2498,7 +2489,6 @@ row_ins_clust_index_entry_low( ut_ad(!thr_get_trx(thr)->in_rollback); mtr_start(&mtr); - mtr.set_named_space(index->space); if (dict_table_is_temporary(index->table)) { /* Disable REDO logging as the lifetime of temp-tables is @@ -2507,29 +2497,41 @@ row_ins_clust_index_entry_low( Disable locking as temp-tables are local to a connection. */ ut_ad(flags & BTR_NO_LOCKING_FLAG); - ut_ad(!dict_table_is_intrinsic(index->table) - || (flags & BTR_NO_UNDO_LOG_FLAG)); - + ut_ad(!dict_index_is_online_ddl(index)); + ut_ad(!index->table->persistent_autoinc); mtr.set_log_mode(MTR_LOG_NO_REDO); - } + } else { + mtr.set_named_space(index->space); - if (mode == BTR_MODIFY_LEAF && dict_index_is_online_ddl(index)) { - mode = BTR_MODIFY_LEAF | BTR_ALREADY_S_LATCHED; - mtr_s_lock(dict_index_get_lock(index), &mtr); + if (mode == BTR_MODIFY_LEAF + && dict_index_is_online_ddl(index)) { + mode = BTR_MODIFY_LEAF | BTR_ALREADY_S_LATCHED; + mtr_s_lock(dict_index_get_lock(index), &mtr); + } + + if (unsigned ai = index->table->persistent_autoinc) { + /* Prepare to persist the AUTO_INCREMENT value + from the index entry to PAGE_ROOT_AUTO_INC. */ + const dfield_t* dfield = dtuple_get_nth_field( + entry, ai - 1); + auto_inc = dfield_is_null(dfield) + ? 0 + : row_parse_int(static_cast<const byte*>( + dfield->data), + dfield->len, + dfield->type.mtype, + dfield->type.prtype + & DATA_UNSIGNED); + } } /* Note that we use PAGE_CUR_LE as the search mode, because then the function will return in both low_match and up_match of the cursor sensible values */ - btr_pcur_open(index, entry, PAGE_CUR_LE, mode, &pcur, &mtr); + btr_pcur_open_low(index, 0, entry, PAGE_CUR_LE, mode, &pcur, + __FILE__, __LINE__, auto_inc, &mtr); cursor = btr_pcur_get_btr_cur(&pcur); - - if (cursor) { - cursor->thr = thr; - } - - ut_ad(!dict_table_is_intrinsic(index->table) - || cursor->page_cur.block->made_dirty_with_no_latch); + cursor->thr = thr; #ifdef UNIV_DEBUG { @@ -2542,15 +2544,7 @@ row_ins_clust_index_entry_low( } #endif /* UNIV_DEBUG */ - /* Allowing duplicates in clustered index is currently enabled - only for intrinsic table and caller understand the limited - operation that can be done in this case. */ - ut_ad(!index->allow_duplicates - || (index->allow_duplicates - && dict_table_is_intrinsic(index->table))); - - if (!index->allow_duplicates - && n_uniq + if (n_uniq && (cursor->up_match >= n_uniq || cursor->low_match >= n_uniq)) { if (flags @@ -2593,20 +2587,13 @@ err_exit: } /* Note: Allowing duplicates would qualify for modification of - an existing record as the new entry is exactly same as old entry. - Avoid this check if allow duplicates is enabled. */ - if (!index->allow_duplicates && row_ins_must_modify_rec(cursor)) { + an existing record as the new entry is exactly same as old entry. */ + if (row_ins_must_modify_rec(cursor)) { /* There is already an index entry with a long enough common prefix, we must convert the insert into a modify of an existing record */ mem_heap_t* entry_heap = mem_heap_create(1024); - /* If the existing record is being modified and the new record - doesn't fit the provided slot then existing record is added - to free list and new record is inserted. This also means - cursor that we have cached for SELECT is now invalid. */ - index->last_sel_cur->invalid = true; - err = row_ins_clust_index_entry_by_modify( &pcur, flags, mode, &offsets, &offsets_heap, entry_heap, entry, thr, &mtr); @@ -2690,147 +2677,6 @@ func_exit: DBUG_RETURN(err); } -/** This is a specialized function meant for direct insertion to -auto-generated clustered index based on cached position from -last successful insert. To be used when data is sorted. - -@param[in] mode BTR_MODIFY_LEAF or BTR_MODIFY_TREE. - depending on whether we wish optimistic or - pessimistic descent down the index tree -@param[in,out] index clustered index -@param[in,out] entry index entry to insert -@param[in] thr query thread - -@return error code */ -static -dberr_t -row_ins_sorted_clust_index_entry( - ulint mode, - dict_index_t* index, - dtuple_t* entry, - ulint n_ext, - que_thr_t* thr) -{ - dberr_t err = DB_SUCCESS; - mtr_t* mtr; - const bool commit_mtr = mode == BTR_MODIFY_TREE; - - mem_heap_t* offsets_heap = NULL; - ulint offsets_[REC_OFFS_NORMAL_SIZE]; - ulint* offsets = offsets_; - rec_offs_init(offsets_); - - DBUG_ENTER("row_ins_sorted_clust_index_entry"); - - ut_ad(index->last_ins_cur != NULL); - ut_ad(dict_index_is_clust(index)); - ut_ad(dict_table_is_intrinsic(index->table)); - ut_ad(dict_index_is_auto_gen_clust(index)); - - btr_cur_t cursor; - cursor.thr = thr; - mtr = &index->last_ins_cur->mtr; - - /* Search for position if tree needs to be split or if last position - is not cached. */ - if (mode == BTR_MODIFY_TREE - || index->last_ins_cur->rec == NULL - || index->last_ins_cur->disable_caching) { - - /* Commit the previous mtr. */ - index->last_ins_cur->release(); - - mtr_start(mtr); - mtr_set_log_mode(mtr, MTR_LOG_NO_REDO); - - err = btr_cur_search_to_nth_level_with_no_latch( - index, 0, entry, PAGE_CUR_LE, &cursor, - __FILE__, __LINE__, mtr); - ut_ad(cursor.page_cur.block != NULL); - ut_ad(cursor.page_cur.block->made_dirty_with_no_latch); - } else { - cursor.index = index; - - cursor.page_cur.index = index; - - cursor.page_cur.rec = index->last_ins_cur->rec; - - cursor.page_cur.block = index->last_ins_cur->block; - } - - const ulint flags = BTR_NO_LOCKING_FLAG | BTR_NO_UNDO_LOG_FLAG; - - for (;;) { - rec_t* insert_rec; - big_rec_t* big_rec = NULL; - - if (mode != BTR_MODIFY_TREE) { - ut_ad((mode & ~BTR_ALREADY_S_LATCHED) - == BTR_MODIFY_LEAF); - - err = btr_cur_optimistic_insert( - flags, &cursor, &offsets, &offsets_heap, entry, - &insert_rec, &big_rec, n_ext, thr, mtr); - if (err != DB_SUCCESS) { - break; - } - } else { - /* TODO: Check if this is needed for intrinsic table. */ - if (buf_LRU_buf_pool_running_out()) { - err = DB_LOCK_TABLE_FULL; - break; - } - - err = btr_cur_optimistic_insert( - flags, &cursor, &offsets, &offsets_heap, entry, - &insert_rec, &big_rec, n_ext, thr, mtr); - - if (err == DB_FAIL) { - err = btr_cur_pessimistic_insert( - flags, &cursor, &offsets, &offsets_heap, - entry, &insert_rec, &big_rec, n_ext, - thr, mtr); - } - } - - if (big_rec != NULL) { - /* If index involves big-record optimization is - turned-off. */ - index->last_ins_cur->release(); - index->last_ins_cur->disable_caching = true; - - err = row_ins_index_entry_big_rec( - entry, big_rec, offsets, &offsets_heap, index, - thr_get_trx(thr)->mysql_thd, __FILE__, __LINE__); - - dtuple_convert_back_big_rec(index, entry, big_rec); - - } else if (err == DB_SUCCESS ) { - if (!commit_mtr - && !index->last_ins_cur->disable_caching) { - index->last_ins_cur->rec = insert_rec; - - index->last_ins_cur->block - = cursor.page_cur.block; - } else { - index->last_ins_cur->release(); - } - } - - break; - } - - if (err != DB_SUCCESS) { - index->last_ins_cur->release(); - } - - if (offsets_heap != NULL) { - mem_heap_free(offsets_heap); - } - - DBUG_RETURN(err); -} - /** Start a mini-transaction and check if the index will be dropped. @param[in,out] mtr mini-transaction @param[in,out] index secondary index @@ -2923,8 +2769,7 @@ row_ins_sec_index_entry_low( cursor.thr = thr; cursor.rtr_info = NULL; - ut_ad(thr_get_trx(thr)->id != 0 - || dict_table_is_intrinsic(index->table)); + ut_ad(thr_get_trx(thr)->id != 0); mtr_start(&mtr); mtr.set_named_space(index->space); @@ -2936,9 +2781,6 @@ row_ins_sec_index_entry_low( Disable locking as temp-tables are local to a connection. */ ut_ad(flags & BTR_NO_LOCKING_FLAG); - ut_ad(!dict_table_is_intrinsic(index->table) - || (flags & BTR_NO_UNDO_LOG_FLAG)); - mtr.set_log_mode(MTR_LOG_NO_REDO); } else if (!dict_index_is_spatial(index)) { /* Enable insert buffering if it's neither temp-table @@ -3008,18 +2850,10 @@ row_ins_sec_index_entry_low( goto func_exit;}); } else { - if (dict_table_is_intrinsic(index->table)) { - err = btr_cur_search_to_nth_level_with_no_latch( - index, 0, entry, PAGE_CUR_LE, &cursor, - __FILE__, __LINE__, &mtr); - ut_ad(cursor.page_cur.block != NULL); - ut_ad(cursor.page_cur.block->made_dirty_with_no_latch); - } else { - err = btr_cur_search_to_nth_level( - index, 0, entry, PAGE_CUR_LE, - search_mode, - &cursor, 0, __FILE__, __LINE__, &mtr); - } + err = btr_cur_search_to_nth_level( + index, 0, entry, PAGE_CUR_LE, + search_mode, + &cursor, 0, __FILE__, __LINE__, &mtr); } if (err != DB_SUCCESS) { @@ -3111,19 +2945,11 @@ row_ins_sec_index_entry_low( prevent any insertion of a duplicate by another transaction. Let us now reposition the cursor and continue the insertion. */ - if (dict_table_is_intrinsic(index->table)) { - err = btr_cur_search_to_nth_level_with_no_latch( - index, 0, entry, PAGE_CUR_LE, &cursor, - __FILE__, __LINE__, &mtr); - ut_ad(cursor.page_cur.block != NULL); - ut_ad(cursor.page_cur.block->made_dirty_with_no_latch); - } else { - btr_cur_search_to_nth_level( - index, 0, entry, PAGE_CUR_LE, - (search_mode - & ~(BTR_INSERT | BTR_IGNORE_SEC_UNIQUE)), - &cursor, 0, __FILE__, __LINE__, &mtr); - } + btr_cur_search_to_nth_level( + index, 0, entry, PAGE_CUR_LE, + (search_mode + & ~(BTR_INSERT | BTR_IGNORE_SEC_UNIQUE)), + &cursor, 0, __FILE__, __LINE__, &mtr); } if (!(flags & BTR_NO_LOCKING_FLAG) @@ -3169,12 +2995,6 @@ row_ins_sec_index_entry_low( } if (row_ins_must_modify_rec(&cursor)) { - /* If the existing record is being modified and the new record - is doesn't fit the provided slot then existing record is added - to free list and new record is inserted. This also means - cursor that we have cached for SELECT is now invalid. */ - index->last_sel_cur->invalid = true; - /* There is already an index entry with a long enough common prefix, we must convert the insert into a modify of an existing record */ @@ -3339,26 +3159,14 @@ row_ins_clust_index_entry( n_uniq = dict_index_is_unique(index) ? index->n_uniq : 0; /* Try first optimistic descent to the B-tree */ - ulint flags; - - if (!dict_table_is_intrinsic(index->table)) { - log_free_check(); - flags = dict_table_is_temporary(index->table) - ? BTR_NO_LOCKING_FLAG - : 0; - } else { - flags = BTR_NO_LOCKING_FLAG | BTR_NO_UNDO_LOG_FLAG; - } + log_free_check(); + const ulint flags = dict_table_is_temporary(index->table) + ? BTR_NO_LOCKING_FLAG + : 0; - if (dict_table_is_intrinsic(index->table) - && dict_index_is_auto_gen_clust(index)) { - err = row_ins_sorted_clust_index_entry( - BTR_MODIFY_LEAF, index, entry, n_ext, thr); - } else { - err = row_ins_clust_index_entry_low( - flags, BTR_MODIFY_LEAF, index, n_uniq, entry, - n_ext, thr, dup_chk_only); - } + err = row_ins_clust_index_entry_low( + flags, BTR_MODIFY_LEAF, index, n_uniq, entry, + n_ext, thr, dup_chk_only); DEBUG_SYNC_C_IF_THD(thr_get_trx(thr)->mysql_thd, @@ -3370,21 +3178,11 @@ row_ins_clust_index_entry( } /* Try then pessimistic descent to the B-tree */ - if (!dict_table_is_intrinsic(index->table)) { - log_free_check(); - } else { - index->last_sel_cur->invalid = true; - } + log_free_check(); - if (dict_table_is_intrinsic(index->table) - && dict_index_is_auto_gen_clust(index)) { - err = row_ins_sorted_clust_index_entry( - BTR_MODIFY_TREE, index, entry, n_ext, thr); - } else { - err = row_ins_clust_index_entry_low( - flags, BTR_MODIFY_TREE, index, n_uniq, entry, - n_ext, thr, dup_chk_only); - } + err = row_ins_clust_index_entry_low( + flags, BTR_MODIFY_TREE, index, n_uniq, entry, + n_ext, thr, dup_chk_only); DBUG_RETURN(err); } @@ -3422,24 +3220,17 @@ row_ins_sec_index_entry( } } - ut_ad(thr_get_trx(thr)->id != 0 - || dict_table_is_intrinsic(index->table)); + ut_ad(thr_get_trx(thr)->id != 0); offsets_heap = mem_heap_create(1024); heap = mem_heap_create(1024); /* Try first optimistic descent to the B-tree */ - ulint flags; - - if (!dict_table_is_intrinsic(index->table)) { - log_free_check(); - flags = dict_table_is_temporary(index->table) - ? BTR_NO_LOCKING_FLAG - : 0; - } else { - flags = BTR_NO_LOCKING_FLAG | BTR_NO_UNDO_LOG_FLAG; - } + log_free_check(); + const ulint flags = dict_table_is_temporary(index->table) + ? BTR_NO_LOCKING_FLAG + : 0; err = row_ins_sec_index_entry_low( flags, BTR_MODIFY_LEAF, index, offsets_heap, heap, entry, @@ -3448,12 +3239,7 @@ row_ins_sec_index_entry( mem_heap_empty(heap); /* Try then pessimistic descent to the B-tree */ - - if (!dict_table_is_intrinsic(index->table)) { - log_free_check(); - } else { - index->last_sel_cur->invalid = true; - } + log_free_check(); err = row_ins_sec_index_entry_low( flags, BTR_MODIFY_TREE, index, @@ -3892,7 +3678,6 @@ row_ins_step( node = static_cast<ins_node_t*>(thr->run_node); ut_ad(que_node_get_type(node) == QUE_NODE_INSERT); - ut_ad(!dict_table_is_intrinsic(node->table)); parent = que_node_get_parent(node); sel_node = node->select; diff --git a/storage/innobase/row/row0log.cc b/storage/innobase/row/row0log.cc index 9de1277e712..0fc1dd097cc 100644 --- a/storage/innobase/row/row0log.cc +++ b/storage/innobase/row/row0log.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -526,7 +527,7 @@ err_exit: my_atomic_addlint(&onlineddl_rowlog_rows, 1); /* 10000 means 100.00%, 4525 means 45.25% */ - onlineddl_rowlog_pct_used = (log->tail.total * 10000) / srv_online_max_size; + onlineddl_rowlog_pct_used = static_cast<ulint>((log->tail.total * 10000) / srv_online_max_size); } #ifdef UNIV_DEBUG @@ -1641,15 +1642,9 @@ row_log_table_apply_insert_low( ut_ad(dtuple_validate(row)); ut_ad(trx_id); -#ifdef UNIV_DEBUG - { - rec_printer p(row); - DBUG_PRINT("ib_alter_table", - ("insert table " IB_ID_FMT " (index " IB_ID_FMT "): %s", - index->table->id, index->id, - p.str().c_str())); - } -#endif + DBUG_LOG("ib_alter_table", + "insert table " << index->table->id << " (index " + << index->id << "): " << rec_printer(row).str()); static const ulint flags = (BTR_CREATE_FLAG @@ -1777,15 +1772,10 @@ row_log_table_apply_delete_low( ut_ad(dict_index_is_clust(index)); -#ifdef UNIV_DEBUG - { - rec_printer p(btr_pcur_get_rec(pcur), offsets); - DBUG_PRINT("ib_alter_table", - ("delete table " IB_ID_FMT " (index " IB_ID_FMT "): %s", - index->table->id, index->id, - p.str().c_str())); - } -#endif + DBUG_LOG("ib_alter_table", + "delete table " << index->table->id << " (index " + << index->id << "): " + << rec_printer(btr_pcur_get_rec(pcur), offsets).str()); if (dict_table_get_next_index(index)) { /* Build a row template for purging secondary index entries. */ @@ -2266,17 +2256,11 @@ func_exit_committed: cur_offsets, NULL, NULL, NULL, &old_ext, heap); ut_ad(old_row); -#ifdef UNIV_DEBUG - { - rec_printer old(old_row); - rec_printer new_row(row); - DBUG_PRINT("ib_alter_table", - ("update table " IB_ID_FMT " (index " IB_ID_FMT "): %s to %s", - index->table->id, index->id, - old.str().c_str(), - new_row.str().c_str())); - } -#endif + DBUG_LOG("ib_alter_table", + "update table " << index->table->id + << " (index " << index->id + << ": " << rec_printer(old_row).str() + << " to " << rec_printer(row).str()); } else { old_row = NULL; old_ext = NULL; @@ -3298,17 +3282,11 @@ row_log_apply_op_low( ut_ad(!dict_index_is_corrupted(index)); ut_ad(trx_id != 0 || op == ROW_OP_DELETE); -#ifdef UNIV_DEBUG - { - rec_printer p(entry); - DBUG_PRINT("ib_create_index", - ("%s %s index " IB_ID_FMT "," IB_ID_FMT ": %s", - op == ROW_OP_INSERT ? "insert" : "delete", - has_index_lock ? "locked" : "unlocked", - index->id, trx_id, - p.str().c_str())); - } -#endif + DBUG_LOG("ib_create_index", + (op == ROW_OP_INSERT ? "insert " : "delete ") + << (has_index_lock ? "locked index " : "unlocked index ") + << index->id << ',' << ib::hex(trx_id) << ": " + << rec_printer(entry).str()); mtr_start(&mtr); mtr.set_named_space(index->space); diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 0b7c1c36231..8fc7ecf75ac 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -30,10 +31,6 @@ Completed by Sunny Bains and Marko Makela #include "ha_prototypes.h" -#include <math.h> - -#include "ha_prototypes.h" - #include "row0merge.h" #include "row0ext.h" #include "row0log.h" @@ -51,7 +48,6 @@ Completed by Sunny Bains and Marko Makela #include "fsp0sysspace.h" #include "ut0new.h" #include "ut0stage.h" -#include "math.h" /* log() */ #include "fil0crypt.h" float my_log2f(float n) @@ -383,9 +379,9 @@ row_merge_insert_index_tuples( const row_merge_buf_t* row_buf, BtrBulk* btr_bulk, const ib_uint64_t table_total_rows, /*!< in: total rows of old table */ - const float pct_progress, /*!< in: total progress + const double pct_progress, /*!< in: total progress percent until now */ - const float pct_cost, /*!< in: current progress percent + const double pct_cost, /*!< in: current progress percent */ fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ @@ -660,13 +656,10 @@ row_merge_buf_add( const dfield_t* row_field; col = ifield->col; - -#ifdef MYSQL_VIRTUAL_COLUMNS const dict_v_col_t* v_col = NULL; if (dict_col_is_virtual(col)) { v_col = reinterpret_cast<const dict_v_col_t*>(col); } -#endif /* MYSQL_VIRTUAL_COLUMNS */ col_no = dict_col_get_no(col); @@ -692,7 +685,6 @@ row_merge_buf_add( } else { /* Use callback to get the virtual column value */ if (dict_col_is_virtual(col)) { - #ifdef MYSQL_VIRTUAL_COLUMN dict_index_t* clust_index = dict_table_get_first_index(new_table); @@ -706,7 +698,6 @@ row_merge_buf_add( DBUG_RETURN(0); } dfield_copy(field, row_field); -#endif /* MYSQL_VIRTUAL_COLUMN */ } else { row_field = dtuple_get_nth_field(row, col_no); dfield_copy(field, row_field); @@ -1110,16 +1101,11 @@ row_merge_buf_write( row_merge_buf_encode(&b, index, entry, n_fields); ut_ad(b < &block[srv_sort_buf_size]); -#ifdef UNIV_DEBUG - { - rec_printer p(entry->fields, n_fields); - DBUG_PRINT("ib_merge_sort", - ("%p,fd=%d,%lu %lu: %s", - reinterpret_cast<const void*>(b), of->fd, - (ulint)of->offset, (ulint)i, - p.str().c_str())); - } -#endif + DBUG_LOG("ib_merge_sort", + reinterpret_cast<const void*>(b) << ',' + << of->fd << ',' << of->offset << ' ' << + i << ": " << + rec_printer(entry->fields, n_fields).str()); } /* Write an "end-of-chunk" marker. */ @@ -1131,10 +1117,9 @@ row_merge_buf_write( to avoid bogus warnings. */ memset(b, 0xff, &block[srv_sort_buf_size] - b); #endif /* UNIV_DEBUG_VALGRIND */ - DBUG_PRINT("ib_merge_sort", - ("write %p,%d,%lu EOF", - reinterpret_cast<const void*>(b), of->fd, - (ulint)of->offset)); + DBUG_LOG("ib_merge_sort", + "write " << reinterpret_cast<const void*>(b) << ',' + << of->fd << ',' << of->offset << " EOF"); DBUG_VOID_RETURN; } @@ -1187,14 +1172,11 @@ row_merge_read( os_offset_t ofs = ((os_offset_t) offset) * srv_sort_buf_size; DBUG_ENTER("row_merge_read"); - DBUG_PRINT("ib_merge_sort", ("fd=%d ofs=" UINT64PF, fd, ofs)); + DBUG_LOG("ib_merge_sort", "fd=" << fd << " ofs=" << ofs); DBUG_EXECUTE_IF("row_merge_read_failure", DBUG_RETURN(FALSE);); IORequest request; - /* Merge sort pages are never compressed. */ - request.disable_compression(); - dberr_t err = os_file_read_no_error_handling( request, OS_FILE_FROM_FD(fd), buf, ofs, srv_sort_buf_size, NULL); @@ -1237,7 +1219,7 @@ row_merge_write( void* out_buf = (void *)buf; DBUG_ENTER("row_merge_write"); - DBUG_PRINT("ib_merge_sort", ("fd=%d ofs=" UINT64PF, fd, ofs)); + DBUG_LOG("ib_merge_sort", "fd=" << fd << " ofs=" << ofs); DBUG_EXECUTE_IF("row_merge_write_failure", DBUG_RETURN(FALSE);); IORequest request(IORequest::WRITE); @@ -1249,8 +1231,6 @@ row_merge_write( mach_write_to_4((byte *)out_buf, 0); } - request.disable_compression(); - dberr_t err = os_file_write( request, "(merge)", OS_FILE_FROM_FD(fd), out_buf, ofs, buf_len); @@ -1311,11 +1291,10 @@ row_merge_read_rec( if (UNIV_UNLIKELY(!extra_size)) { /* End of list */ *mrec = NULL; - DBUG_PRINT("ib_merge_sort", - ("read %p,%p,%d,%lu EOF\n", - reinterpret_cast<const void*>(b), - reinterpret_cast<const void*>(block), - fd, ulint(*foffs))); + DBUG_LOG("ib_merge_sort", + "read " << reinterpret_cast<const void*>(b) << ',' << + reinterpret_cast<const void*>(block) << ',' << + fd << ',' << *foffs << " EOF"); DBUG_RETURN(NULL); } @@ -1428,18 +1407,11 @@ err_exit: b += extra_size + data_size - avail_size; func_exit: - -#ifdef UNIV_DEBUG - { - rec_printer p(*mrec, 0, offsets); - DBUG_PRINT("ib_merge_sort", - ("%p,%p,fd=%d,%lu: %s", - reinterpret_cast<const void*>(b), - reinterpret_cast<const void*>(block), - fd, ulint(*foffs), - p.str().c_str())); - } -#endif + DBUG_LOG("ib_merge_sort", + reinterpret_cast<const void*>(b) << ',' << + reinterpret_cast<const void*>(block) + << ",fd=" << fd << ',' << *foffs << ": " + << rec_printer(*mrec, 0, offsets).str()); DBUG_RETURN(b); } @@ -1470,15 +1442,9 @@ row_merge_write_rec_low( #endif /* DBUG_OFF */ DBUG_ASSERT(e == rec_offs_extra_size(offsets) + 1); -#ifdef UNIV_DEBUG - { - rec_printer p(mrec, 0, offsets); - DBUG_PRINT("ib_merge_sort", - ("%p,fd=%d,%lu: %s", - reinterpret_cast<const void*>(b), fd, ulint(foffs), - p.str().c_str())); - } -#endif + DBUG_LOG("ib_merge_sort", + reinterpret_cast<const void*>(b) << ",fd=" << fd << ',' + << foffs << ": " << rec_printer(mrec, 0, offsets).str()); if (e < 0x80) { *b++ = (byte) e; @@ -1588,11 +1554,10 @@ row_merge_write_eof( ut_ad(foffs); DBUG_ENTER("row_merge_write_eof"); - DBUG_PRINT("ib_merge_sort", - ("%p,%p,fd=%d,%lu", - reinterpret_cast<const void*>(b), - reinterpret_cast<const void*>(block), - fd, ulint(*foffs))); + DBUG_LOG("ib_merge_sort", + reinterpret_cast<const void*>(b) << ',' << + reinterpret_cast<const void*>(block) << + ",fd=" << fd << ',' << *foffs); if (b == &block[0]) { b+= ROW_MERGE_RESERVE_SIZE; @@ -1835,7 +1800,7 @@ row_merge_read_clustered_index( bool skip_pk_sort, int* tmpfd, ut_stage_alter_t* stage, - float pct_cost, + double pct_cost, fil_space_crypt_t* crypt_data, row_merge_block_t* crypt_block, struct TABLE* eval_table) @@ -1870,7 +1835,7 @@ row_merge_read_clustered_index( mtuple_t prev_mtuple; mem_heap_t* conv_heap = NULL; FlushObserver* observer = trx->flush_observer; - float curr_progress = 0.0; + double curr_progress = 0.0; ib_uint64_t read_rows = 0; ib_uint64_t table_total_rows = 0; @@ -2952,10 +2917,9 @@ row_merge_blocks( ulint* offsets1;/* offsets of mrec1 */ DBUG_ENTER("row_merge_blocks"); - DBUG_PRINT("ib_merge_sort", - ("fd=%d,%lu+%lu to fd=%d,%lu", - file->fd, ulint(*foffs0), ulint(*foffs1), - of->fd, ulint(of->offset))); + DBUG_LOG("ib_merge_sort", + "fd=" << file->fd << ',' << *foffs0 << '+' << *foffs1 + << " to fd=" << of->fd << ',' << of->offset); heap = row_merge_heap_create(dup->index, &buf, &offsets0, &offsets1); @@ -3065,10 +3029,9 @@ row_merge_blocks_copy( ulint* offsets1;/* dummy offsets */ DBUG_ENTER("row_merge_blocks_copy"); - DBUG_PRINT("ib_merge_sort", - ("fd=%d,%lu to fd=%d,%lu", - file->fd, ulint(foffs0), - of->fd, ulint(of->offset))); + DBUG_LOG("ib_merge_sort", + "fd=" << file->fd << ',' << foffs0 + << " to fd=" << of->fd << ',' << of->offset); heap = row_merge_heap_create(index, &buf, &offsets0, &offsets1); @@ -3283,10 +3246,10 @@ row_merge_sort( const bool update_progress, /*!< in: update progress status variable or not */ - const float pct_progress, + const double pct_progress, /*!< in: total progress percent until now */ - const float pct_cost, /*!< in: current progress percent */ + const double pct_cost, /*!< in: current progress percent */ fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ ulint space, /*!< in: space id */ @@ -3298,7 +3261,7 @@ row_merge_sort( dberr_t error = DB_SUCCESS; ulint merge_count = 0; ulint total_merge_sort_count; - float curr_progress = 0; + double curr_progress = 0; DBUG_ENTER("row_merge_sort"); @@ -3311,7 +3274,7 @@ row_merge_sort( /* Find the number N which 2^N is greater or equal than num_runs */ /* N is merge sort running count */ - total_merge_sort_count = (ulint) ceil(my_log2f(num_runs)); + total_merge_sort_count = (ulint) ceil(my_log2f((float)num_runs)); if(total_merge_sort_count <= 0) { total_merge_sort_count=1; } @@ -3497,9 +3460,9 @@ row_merge_insert_index_tuples( const row_merge_buf_t* row_buf, BtrBulk* btr_bulk, const ib_uint64_t table_total_rows, /*!< in: total rows of old table */ - const float pct_progress, /*!< in: total progress + const double pct_progress, /*!< in: total progress percent until now */ - const float pct_cost, /*!< in: current progress percent + const double pct_cost, /*!< in: current progress percent */ fil_space_crypt_t* crypt_data,/*!< in: table crypt data */ row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ @@ -3516,7 +3479,7 @@ row_merge_insert_index_tuples( ulint n_rows = 0; dtuple_t* dtuple; ib_uint64_t inserted_rows = 0; - float curr_progress = 0; + double curr_progress = 0; dict_index_t* old_index = NULL; const mrec_t* mrec = NULL; ulint n_ext = 0; @@ -4514,9 +4477,7 @@ row_merge_create_index( if (col_names && col_names[i]) { name = col_names[i]; } else { - name = ifield->col_name ? - dict_table_get_col_name_for_mysql(table, ifield->col_name) : - dict_table_get_col_name(table, ifield->col_no); + name = dict_table_get_col_name(table, ifield->col_no); } } @@ -4678,11 +4639,11 @@ row_merge_build_indexes( bool fts_psort_initiated = false; fil_space_crypt_t * crypt_data = NULL; - float total_static_cost = 0; - float total_dynamic_cost = 0; + double total_static_cost = 0; + double total_dynamic_cost = 0; uint total_index_blocks = 0; - float pct_cost=0; - float pct_progress=0; + double pct_cost=0; + double pct_progress=0; DBUG_ENTER("row_merge_build_indexes"); @@ -4713,10 +4674,7 @@ row_merge_build_indexes( /* If tablespace is encrypted, allocate additional buffer for encryption/decryption. */ - if ((crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_ON) || - (srv_encrypt_tables && - crypt_data && crypt_data->encryption == FIL_SPACE_ENCRYPTION_DEFAULT)) { - + if (crypt_data && crypt_data->should_encrypt()) { crypt_block = static_cast<row_merge_block_t*>( alloc.allocate_large(3 * srv_sort_buf_size, &crypt_pfx)); diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 4a063c9af83..fbf49d8d3a0 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -102,8 +103,6 @@ static ib_mutex_t row_drop_list_mutex; /** Flag: has row_mysql_drop_list been initialized? */ static ibool row_mysql_drop_list_inited = FALSE; -extern ib_mutex_t master_key_id_mutex; - /** Magic table names for invoking various monitor threads */ /* @{ */ static const char S_innodb_monitor[] = "innodb_monitor"; @@ -1015,9 +1014,6 @@ row_create_prebuilt( prebuilt->mysql_row_len = mysql_row_len; - prebuilt->ins_sel_stmt = false; - prebuilt->session = NULL; - prebuilt->fts_doc_id_in_read_set = 0; prebuilt->blob_heap = NULL; @@ -1407,282 +1403,12 @@ run_again: return(err); } -/** Perform explicit rollback in absence of UNDO logs. -@param[in] index apply rollback action on this index -@param[in] entry entry to remove/rollback. -@param[in,out] thr thread handler. -@param[in,out] mtr mini transaction. -@return error code or DB_SUCCESS */ -static -dberr_t -row_explicit_rollback( - dict_index_t* index, - const dtuple_t* entry, - que_thr_t* thr, - mtr_t* mtr) -{ - btr_cur_t cursor; - ulint flags; - ulint offsets_[REC_OFFS_NORMAL_SIZE]; - ulint* offsets; - mem_heap_t* heap = NULL; - dberr_t err = DB_SUCCESS; - - rec_offs_init(offsets_); - flags = BTR_NO_LOCKING_FLAG | BTR_NO_UNDO_LOG_FLAG; - - err = btr_cur_search_to_nth_level_with_no_latch( - index, 0, entry, PAGE_CUR_LE, - &cursor, __FILE__, __LINE__, mtr); - - offsets = rec_get_offsets( - btr_cur_get_rec(&cursor), index, offsets_, - ULINT_UNDEFINED, &heap); - - if (dict_index_is_clust(index)) { - err = btr_cur_del_mark_set_clust_rec( - flags, btr_cur_get_block(&cursor), - btr_cur_get_rec(&cursor), index, - offsets, thr, entry, mtr); - } else { - err = btr_cur_del_mark_set_sec_rec( - flags, &cursor, TRUE, thr, mtr); - } - ut_ad(err == DB_SUCCESS); - - /* Void call just to set mtr modification flag - to true failing which block is not scheduled for flush*/ - byte* log_ptr = mlog_open(mtr, 0); - ut_ad(log_ptr == NULL); - if (log_ptr != NULL) { - /* To keep complier happy. */ - mlog_close(mtr, log_ptr); - } - - if (heap != NULL) { - mem_heap_free(heap); - } - - return(err); -} - -/** Convert a row in the MySQL format to a row in the Innobase format. -This is specialized function used for intrinsic table with reduce branching. -@param[in,out] row row where field values are copied. -@param[in] prebuilt prebuilt handler -@param[in] mysql_rec row in mysql format. */ -static -void -row_mysql_to_innobase( - dtuple_t* row, - row_prebuilt_t* prebuilt, - const byte* mysql_rec) -{ - ut_ad(dict_table_is_intrinsic(prebuilt->table)); - - const byte* ptr = mysql_rec; - - for (ulint i = 0; i < prebuilt->n_template; i++) { - const mysql_row_templ_t* templ; - dfield_t* dfield; - - templ = prebuilt->mysql_template + i; - dfield = dtuple_get_nth_field(row, i); - - /* Check if column has null value. */ - if (templ->mysql_null_bit_mask != 0) { - if (mysql_rec[templ->mysql_null_byte_offset] - & (byte) (templ->mysql_null_bit_mask)) { - dfield_set_null(dfield); - continue; - } - } - - /* Extract the column value. */ - ptr = mysql_rec + templ->mysql_col_offset; - const dtype_t* dtype = dfield_get_type(dfield); - ulint col_len = templ->mysql_col_len; - - ut_ad(dtype->mtype == DATA_INT - || dtype->mtype == DATA_CHAR - || dtype->mtype == DATA_MYSQL - || dtype->mtype == DATA_VARCHAR - || dtype->mtype == DATA_VARMYSQL - || dtype->mtype == DATA_BINARY - || dtype->mtype == DATA_FIXBINARY - || dtype->mtype == DATA_FLOAT - || dtype->mtype == DATA_DOUBLE - || dtype->mtype == DATA_DECIMAL - || dtype->mtype == DATA_BLOB - || dtype->mtype == DATA_GEOMETRY - || dtype->mtype == DATA_POINT - || dtype->mtype == DATA_VAR_POINT); - -#ifdef UNIV_DEBUG - if (dtype_get_mysql_type(dtype) == DATA_MYSQL_TRUE_VARCHAR) { - ut_ad(templ->mysql_length_bytes > 0); - } -#endif /* UNIV_DEBUG */ - - /* For now varchar field this has to be always 0 so - memcpy of 0 bytes shouldn't affect the original col_len. */ - if (dtype->mtype == DATA_INT) { - /* Convert and Store in big-endian. */ - byte* buf = prebuilt->ins_upd_rec_buff - + templ->mysql_col_offset; - byte* copy_to = buf + col_len; - for (;;) { - copy_to--; - *copy_to = *ptr; - if (copy_to == buf) { - break; - } - ptr++; - } - - if (!(dtype->prtype & DATA_UNSIGNED)) { - *buf ^= 128; - } - - ptr = buf; - buf += col_len; - } else if (dtype_get_mysql_type(dtype) == - DATA_MYSQL_TRUE_VARCHAR) { - - ut_ad(dtype->mtype == DATA_VARCHAR - || dtype->mtype == DATA_VARMYSQL - || dtype->mtype == DATA_BINARY); - - col_len = 0; - row_mysql_read_true_varchar( - &col_len, ptr, templ->mysql_length_bytes); - ptr += templ->mysql_length_bytes; - } else if (dtype->mtype == DATA_BLOB) { - ptr = row_mysql_read_blob_ref(&col_len, ptr, col_len); - } else if (DATA_GEOMETRY_MTYPE(dtype->mtype)) { - /* Point, Var-Point, Geometry */ - ptr = row_mysql_read_geometry(&col_len, ptr, col_len); - } - - dfield_set_data(dfield, ptr, col_len); - } -} - -/** Does an insert for MySQL using cursor interface. -Cursor interface is low level interface that directly interacts at -Storage Level by-passing all the locking and transaction semantics. -For InnoDB case, this will also by-pass hidden column generation. -@param[in] mysql_rec row in the MySQL format -@param[in,out] prebuilt prebuilt struct in MySQL handle -@return error code or DB_SUCCESS */ -static -dberr_t -row_insert_for_mysql_using_cursor( - const byte* mysql_rec, - row_prebuilt_t* prebuilt) -{ - dberr_t err = DB_SUCCESS; - ins_node_t* node = NULL; - que_thr_t* thr = NULL; - mtr_t mtr; - - /* Step-1: Get the reference of row to insert. */ - row_get_prebuilt_insert_row(prebuilt); - node = prebuilt->ins_node; - thr = que_fork_get_first_thr(prebuilt->ins_graph); - - /* Step-2: Convert row from MySQL row format to InnoDB row format. */ - row_mysql_to_innobase(node->row, prebuilt, mysql_rec); - - /* Step-3: Append row-id index is not unique. */ - dict_index_t* clust_index = dict_table_get_first_index(node->table); - - if (!dict_index_is_unique(clust_index)) { - dict_sys_write_row_id( - node->row_id_buf, - dict_table_get_next_table_sess_row_id(node->table)); - } - - trx_write_trx_id(node->trx_id_buf, - dict_table_get_next_table_sess_trx_id(node->table)); - - /* Step-4: Iterate over all the indexes and insert entries. */ - dict_index_t* inserted_upto = NULL; - node->entry = UT_LIST_GET_FIRST(node->entry_list); - for (dict_index_t* index = UT_LIST_GET_FIRST(node->table->indexes); - index != NULL; - index = UT_LIST_GET_NEXT(indexes, index), - node->entry = UT_LIST_GET_NEXT(tuple_list, node->entry)) { - - node->index = index; - err = row_ins_index_entry_set_vals( - node->index, node->entry, node->row); - if (err != DB_SUCCESS) { - break; - } - - if (dict_index_is_clust(index)) { - err = row_ins_clust_index_entry( - node->index, node->entry, thr, 0, false); - } else { - err = row_ins_sec_index_entry( - node->index, node->entry, thr, false); - } - - if (err == DB_SUCCESS) { - inserted_upto = index; - } else { - break; - } - } - - /* Step-5: If error is encountered while inserting entries to any - of the index then entries inserted to previous indexes are removed - explicity. Automatic rollback is not in action as UNDO logs are - turned-off. */ - if (err != DB_SUCCESS) { - - node->entry = UT_LIST_GET_FIRST(node->entry_list); - - mtr_start(&mtr); - dict_disable_redo_if_temporary(node->table, &mtr); - - for (dict_index_t* index = - UT_LIST_GET_FIRST(node->table->indexes); - inserted_upto != NULL; - index = UT_LIST_GET_NEXT(indexes, index), - node->entry = UT_LIST_GET_NEXT(tuple_list, node->entry)) { - - row_explicit_rollback(index, node->entry, thr, &mtr); - - if (index == inserted_upto) { - break; - } - } - - mtr_commit(&mtr); - } else { - /* Not protected by dict_table_stats_lock() for performance - reasons, we would rather get garbage in stat_n_rows (which is - just an estimate anyway) than protecting the following code - , with a latch. */ - dict_table_n_rows_inc(node->table); - - srv_stats.n_rows_inserted.inc(); - } - - thr_get_trx(thr)->error_state = DB_SUCCESS; - return(err); -} - -/** Does an insert for MySQL using INSERT graph. This function will run/execute -INSERT graph. +/** Does an insert for MySQL. @param[in] mysql_rec row in the MySQL format @param[in,out] prebuilt prebuilt struct in MySQL handle @return error code or DB_SUCCESS */ -static dberr_t -row_insert_for_mysql_using_ins_graph( +row_insert_for_mysql( const byte* mysql_rec, row_prebuilt_t* prebuilt) { @@ -1896,26 +1622,6 @@ error_exit: return(err); } -/** Does an insert for MySQL. -@param[in] mysql_rec row in the MySQL format -@param[in,out] prebuilt prebuilt struct in MySQL handle -@return error code or DB_SUCCESS*/ -dberr_t -row_insert_for_mysql( - const byte* mysql_rec, - row_prebuilt_t* prebuilt) -{ - /* For intrinsic tables there a lot of restrictions that can be - relaxed including locking of table, transaction handling, etc. - Use direct cursor interface for inserting to intrinsic tables. */ - if (dict_table_is_intrinsic(prebuilt->table)) { - return(row_insert_for_mysql_using_cursor(mysql_rec, prebuilt)); - } else { - return(row_insert_for_mysql_using_ins_graph( - mysql_rec, prebuilt)); - } -} - /*********************************************************************//** Builds a dummy query graph used in selects. */ void @@ -1998,24 +1704,18 @@ row_get_prebuilt_update_vector( row_prebuilt_t* prebuilt) /*!< in: prebuilt struct in MySQL handle */ { - dict_table_t* table = prebuilt->table; - upd_node_t* node; - - ut_ad(prebuilt && table && prebuilt->trx); - if (prebuilt->upd_node == NULL) { /* Not called before for this handle: create an update node and query graph to the prebuilt struct */ - node = row_create_update_node_for_mysql(table, prebuilt->heap); - - prebuilt->upd_node = node; + prebuilt->upd_node = row_create_update_node_for_mysql( + prebuilt->table, prebuilt->heap); prebuilt->upd_graph = static_cast<que_fork_t*>( que_node_get_parent( pars_complete_graph_for_exec( - static_cast<upd_node_t*>(node), + prebuilt->upd_node, prebuilt->trx, prebuilt->heap, prebuilt))); @@ -2129,314 +1829,6 @@ public: }; -typedef std::vector<btr_pcur_t, ut_allocator<btr_pcur_t> > cursors_t; - -/** Delete row from table (corresponding entries from all the indexes). -Function will maintain cursor to the entries to invoke explicity rollback -just incase update action following delete fails. - -@param[in] node update node carrying information to delete. -@param[out] delete_entries vector of cursor to deleted entries. -@param[in] restore_delete if true, then restore DELETE records by - unmarking delete. -@return error code or DB_SUCCESS */ -static -dberr_t -row_delete_for_mysql_using_cursor( - const upd_node_t* node, - cursors_t& delete_entries, - bool restore_delete) -{ - mtr_t mtr; - dict_table_t* table = node->table; - mem_heap_t* heap = mem_heap_create(1000); - dberr_t err = DB_SUCCESS; - dtuple_t* entry; - - mtr_start(&mtr); - dict_disable_redo_if_temporary(table, &mtr); - - for (dict_index_t* index = UT_LIST_GET_FIRST(table->indexes); - index != NULL && err == DB_SUCCESS && !restore_delete; - index = UT_LIST_GET_NEXT(indexes, index)) { - - entry = row_build_index_entry( - node->row, node->ext, index, heap); - - btr_pcur_t pcur; - - btr_pcur_open(index, entry, PAGE_CUR_LE, - BTR_MODIFY_LEAF, &pcur, &mtr); - -#ifdef UNIV_DEBUG - ulint offsets_[REC_OFFS_NORMAL_SIZE]; - ulint* offsets = offsets_; - rec_offs_init(offsets_); - - offsets = rec_get_offsets( - btr_cur_get_rec(btr_pcur_get_btr_cur(&pcur)), - index, offsets, ULINT_UNDEFINED, &heap); - - ut_ad(!cmp_dtuple_rec( - entry, btr_cur_get_rec(btr_pcur_get_btr_cur(&pcur)), - offsets)); -#endif /* UNIV_DEBUG */ - - ut_ad(!rec_get_deleted_flag( - btr_cur_get_rec(btr_pcur_get_btr_cur(&pcur)), - dict_table_is_comp(index->table))); - - ut_ad(btr_pcur_get_block(&pcur)->made_dirty_with_no_latch); - - if (page_rec_is_infimum(btr_pcur_get_rec(&pcur)) - || page_rec_is_supremum(btr_pcur_get_rec(&pcur))) { - err = DB_ERROR; - } else { - btr_cur_t* btr_cur = btr_pcur_get_btr_cur(&pcur); - - btr_rec_set_deleted_flag( - btr_cur_get_rec(btr_cur), - buf_block_get_page_zip( - btr_cur_get_block(btr_cur)), - TRUE); - - /* Void call just to set mtr modification flag - to true failing which block is not scheduled for flush*/ - byte* log_ptr = mlog_open(&mtr, 0); - ut_ad(log_ptr == NULL); - if (log_ptr != NULL) { - /* To keep complier happy. */ - mlog_close(&mtr, log_ptr); - } - - btr_pcur_store_position(&pcur, &mtr); - - delete_entries.push_back(pcur); - } - } - - if (err != DB_SUCCESS || restore_delete) { - - /* Rollback half-way delete action that might have been - applied to few of the indexes. */ - cursors_t::iterator end = delete_entries.end(); - for (cursors_t::iterator it = delete_entries.begin(); - it != end; - ++it) { - - ibool success = btr_pcur_restore_position( - BTR_MODIFY_LEAF, &(*it), &mtr); - - if (!success) { - ut_a(success); - } else { - btr_cur_t* btr_cur = btr_pcur_get_btr_cur( - &(*it)); - - ut_ad(btr_cur_get_block( - btr_cur)->made_dirty_with_no_latch); - - btr_rec_set_deleted_flag( - btr_cur_get_rec(btr_cur), - buf_block_get_page_zip( - btr_cur_get_block(btr_cur)), - FALSE); - - /* Void call just to set mtr modification flag - to true failing which block is not scheduled for - flush. */ - byte* log_ptr = mlog_open(&mtr, 0); - ut_ad(log_ptr == NULL); - if (log_ptr != NULL) { - /* To keep complier happy. */ - mlog_close(&mtr, log_ptr); - } - } - } - } - - mtr_commit(&mtr); - - mem_heap_free(heap); - - return(err); -} - -/** Does an update of a row for MySQL by inserting new entry with update values. -@param[in] node update node carrying information to delete. -@param[out] delete_entries vector of cursor to deleted entries. -@param[in] thr thread handler -@return error code or DB_SUCCESS */ -static -dberr_t -row_update_for_mysql_using_cursor( - const upd_node_t* node, - cursors_t& delete_entries, - que_thr_t* thr) -{ - dberr_t err = DB_SUCCESS; - dict_table_t* table = node->table; - mem_heap_t* heap = mem_heap_create(1000); - dtuple_t* entry; - dfield_t* trx_id_field; - - /* Step-1: Update row-id column if table doesn't have unique index. */ - if (!dict_index_is_unique(dict_table_get_first_index(table))) { - /* Update the row_id column. */ - dfield_t* row_id_field; - - row_id_field = dtuple_get_nth_field( - node->upd_row, dict_table_get_n_cols(table) - 2); - - dict_sys_write_row_id( - static_cast<byte*>(row_id_field->data), - dict_table_get_next_table_sess_row_id(node->table)); - } - - /* Step-2: Update the trx_id column. */ - trx_id_field = dtuple_get_nth_field( - node->upd_row, dict_table_get_n_cols(table) - 1); - trx_write_trx_id(static_cast<byte*>(trx_id_field->data), - dict_table_get_next_table_sess_trx_id(node->table)); - - - /* Step-3: Check if UPDATE can lead to DUPLICATE key violation. - If yes, then avoid executing it and return error. Only after ensuring - that UPDATE is safe execute it as we can't rollback. */ - for (dict_index_t* index = UT_LIST_GET_FIRST(table->indexes); - index != NULL && err == DB_SUCCESS; - index = UT_LIST_GET_NEXT(indexes, index)) { - - entry = row_build_index_entry( - node->upd_row, node->upd_ext, index, heap); - - if (dict_index_is_clust(index)) { - if (!dict_index_is_auto_gen_clust(index)) { - err = row_ins_clust_index_entry( - index, entry, thr, - node->upd_ext - ? node->upd_ext->n_ext : 0, - true); - } - } else { - err = row_ins_sec_index_entry(index, entry, thr, true); - } - } - - if (err != DB_SUCCESS) { - /* This suggest update can't be executed safely. - Avoid executing update. Rollback DELETE action. */ - row_delete_for_mysql_using_cursor(node, delete_entries, true); - } - - /* Step-4: It is now safe to execute update if there is no error */ - for (dict_index_t* index = UT_LIST_GET_FIRST(table->indexes); - index != NULL && err == DB_SUCCESS; - index = UT_LIST_GET_NEXT(indexes, index)) { - - entry = row_build_index_entry( - node->upd_row, node->upd_ext, index, heap); - - if (dict_index_is_clust(index)) { - err = row_ins_clust_index_entry( - index, entry, thr, - node->upd_ext ? node->upd_ext->n_ext : 0, - false); - /* Commit the open mtr as we are processing UPDATE. */ - index->last_ins_cur->release(); - } else { - err = row_ins_sec_index_entry(index, entry, thr, false); - } - - /* Too big record is valid error and suggestion is to use - bigger page-size or different format. */ - ut_ad(err == DB_SUCCESS - || err == DB_TOO_BIG_RECORD - || err == DB_OUT_OF_FILE_SPACE); - - if (err == DB_TOO_BIG_RECORD) { - row_delete_for_mysql_using_cursor( - node, delete_entries, true); - } - } - - if (heap != NULL) { - mem_heap_free(heap); - } - return(err); -} - -/** Does an update or delete of a row for MySQL. -@param[in] mysql_rec row in the MySQL format -@param[in,out] prebuilt prebuilt struct in MySQL handle -@return error code or DB_SUCCESS */ -static -dberr_t -row_del_upd_for_mysql_using_cursor( - const byte* mysql_rec, - row_prebuilt_t* prebuilt) -{ - dberr_t err = DB_SUCCESS; - upd_node_t* node; - cursors_t delete_entries; - dict_index_t* clust_index; - que_thr_t* thr = NULL; - - /* Step-0: If there is cached insert position commit it before - starting delete/update action as this can result in btree structure - to change. */ - thr = que_fork_get_first_thr(prebuilt->upd_graph); - clust_index = dict_table_get_first_index(prebuilt->table); - clust_index->last_ins_cur->release(); - - /* Step-1: Select the appropriate cursor that will help build - the original row and updated row. */ - node = prebuilt->upd_node; - if (prebuilt->pcur->btr_cur.index == clust_index) { - btr_pcur_copy_stored_position(node->pcur, prebuilt->pcur); - } else { - btr_pcur_copy_stored_position(node->pcur, - prebuilt->clust_pcur); - } - - ut_ad(dict_table_is_intrinsic(prebuilt->table)); - ut_ad(!prebuilt->table->n_v_cols); - - /* Internal table is created by optimiser. So there - should not be any virtual columns. */ - row_upd_store_row(node, NULL, NULL); - - /* Step-2: Execute DELETE operation. */ - err = row_delete_for_mysql_using_cursor(node, delete_entries, false); - - /* Step-3: If only DELETE operation then exit immediately. */ - if (node->is_delete) { - if (err == DB_SUCCESS) { - dict_table_n_rows_dec(prebuilt->table); - srv_stats.n_rows_deleted.inc(); - } - } - - if (err == DB_SUCCESS && !node->is_delete) { - /* Step-4: Complete UPDATE operation by inserting new row with - updated data. */ - err = row_update_for_mysql_using_cursor( - node, delete_entries, thr); - - if (err == DB_SUCCESS) { - srv_stats.n_rows_updated.inc(); - } - } - - thr_get_trx(thr)->error_state = DB_SUCCESS; - cursors_t::iterator end = delete_entries.end(); - for (cursors_t::iterator it = delete_entries.begin(); it != end; ++it) { - btr_pcur_close(&(*it)); - } - - return(err); -} - /** Does an update or delete of a row for MySQL. @param[in] mysql_rec row in the MySQL format @param[in,out] prebuilt prebuilt struct in MySQL handle @@ -2794,41 +2186,8 @@ row_update_for_mysql( const byte* mysql_rec, row_prebuilt_t* prebuilt) { - if (dict_table_is_intrinsic(prebuilt->table)) { - return(row_del_upd_for_mysql_using_cursor(mysql_rec, prebuilt)); - } else { - ut_a(prebuilt->template_type == ROW_MYSQL_WHOLE_ROW); - return(row_update_for_mysql_using_upd_graph( - mysql_rec, prebuilt)); - } -} - -/** Delete all rows for the given table by freeing/truncating indexes. -@param[in,out] table table handler -@return error code or DB_SUCCESS */ -dberr_t -row_delete_all_rows( - dict_table_t* table) -{ - dberr_t err = DB_SUCCESS; - - /* Step-0: If there is cached insert position along with mtr - commit it before starting delete/update action. */ - dict_table_get_first_index(table)->last_ins_cur->release(); - - /* Step-1: Now truncate all the indexes and re-create them. - Note: This is ddl action even though delete all rows is - DML action. Any error during this action is ir-reversible. */ - for (dict_index_t* index = UT_LIST_GET_FIRST(table->indexes); - index != NULL && err == DB_SUCCESS; - index = UT_LIST_GET_NEXT(indexes, index)) { - - err = dict_truncate_index_tree_in_mem(index); - // TODO: what happen if get an error - ut_ad(err == DB_SUCCESS); - } - - return(err); + ut_a(prebuilt->template_type == ROW_MYSQL_WHOLE_ROW); + return(row_update_for_mysql_using_upd_graph(mysql_rec, prebuilt)); } /** This can only be used when srv_locks_unsafe_for_binlog is TRUE or this @@ -3062,9 +2421,6 @@ row_create_table_for_mysql( dict_table_t* table, /*!< in, own: table definition (will be freed, or on DB_SUCCESS added to the data dictionary cache) */ - const char* compression, - /*!< in: compression algorithm to use, - can be NULL */ trx_t* trx, /*!< in/out: transaction */ bool commit, /*!< in: if true, commit the transaction */ fil_encryption_t mode, /*!< in: encryption mode */ @@ -3152,46 +2508,11 @@ err_exit: /* We must delete the link file. */ RemoteDatafile::delete_link_file(table->name.m_name); - - } else if (compression != NULL && compression[0] != '\0') { -#ifdef MYSQL_COMPRESSION - ut_ad(!dict_table_in_shared_tablespace(table)); - - ut_ad(Compression::validate(compression) == DB_SUCCESS); - - err = fil_set_compression(table, compression); - - switch (err) { - case DB_SUCCESS: - break; - case DB_NOT_FOUND: - case DB_UNSUPPORTED: - case DB_IO_NO_PUNCH_HOLE_FS: - /* Return these errors */ - break; - case DB_IO_NO_PUNCH_HOLE_TABLESPACE: - /* Page Compression will not be used. */ - err = DB_SUCCESS; - break; - default: - ut_error; - } - - /* We can check for file system punch hole support - only after creating the tablespace. On Windows - we can query that information but not on Linux. */ - ut_ad(err == DB_SUCCESS - || err == DB_IO_NO_PUNCH_HOLE_FS); -#endif /* MYSQL_COMPRESSION */ - - /* In non-strict mode we ignore dodgy compression - settings. */ } } switch (err) { case DB_SUCCESS: - case DB_IO_NO_PUNCH_HOLE_FS: break; case DB_OUT_OF_FILE_SPACE: trx->error_state = DB_SUCCESS; @@ -3258,13 +2579,12 @@ row_create_index_for_mysql( dict_index_t* index, /*!< in, own: index definition (will be freed) */ trx_t* trx, /*!< in: transaction handle */ - const ulint* field_lengths, /*!< in: if not NULL, must contain + const ulint* field_lengths) /*!< in: if not NULL, must contain dict_index_get_n_fields(index) actual field lengths for the index columns, which are then checked for not being too large. */ - dict_table_t* handler) /*!< in/out: table handler. */ { ind_node_t* node; mem_heap_t* heap; @@ -3287,22 +2607,11 @@ row_create_index_for_mysql( is_fts = (index->type == DICT_FTS); - if (handler != NULL && dict_table_is_intrinsic(handler)) { - table = handler; - } - - if (table == NULL) { - - ut_ad(rw_lock_own(dict_operation_lock, RW_LOCK_X)); - ut_ad(mutex_own(&dict_sys->mutex)); - - table = dict_table_open_on_name(table_name, TRUE, TRUE, - DICT_ERR_IGNORE_NONE); + ut_ad(rw_lock_own(dict_operation_lock, RW_LOCK_X)); + ut_ad(mutex_own(&dict_sys->mutex)); - } else { - table->acquire(); - ut_ad(dict_table_is_intrinsic(table)); - } + table = dict_table_open_on_name(table_name, TRUE, TRUE, + DICT_ERR_IGNORE_NONE); if (!dict_table_is_temporary(table)) { trx_start_if_not_started_xa(trx, true); @@ -3361,13 +2670,9 @@ row_create_index_for_mysql( index_id_t index_id = index->id; - /* add index to dictionary cache and also free index object. - We allow instrinsic table to violate the size limits because - they are used by optimizer for all record formats. */ + /* add index to dictionary cache and also free index object. */ err = dict_index_add_to_cache( - table, index, FIL_NULL, - !dict_table_is_intrinsic(table) - && trx_is_strict(trx)); + table, index, FIL_NULL, trx_is_strict(trx)); if (err != DB_SUCCESS) { goto error_handling; @@ -3375,23 +2680,13 @@ row_create_index_for_mysql( /* as above function has freed index object re-load it now from dictionary cache using index_id */ - if (!dict_table_is_intrinsic(table)) { - index = dict_index_get_if_in_cache_low(index_id); - } else { - index = dict_table_find_index_on_id(table, index_id); - - /* trx_id field is used for tracking which transaction - created the index. For intrinsic table this is - ir-relevant and so re-use it for tracking consistent - view while processing SELECT as part of UPDATE. */ - index->trx_id = ULINT_UNDEFINED; - } + index = dict_index_get_if_in_cache_low(index_id); ut_a(index != NULL); index->table = table; err = dict_create_index_tree_in_mem(index, trx); - if (err != DB_SUCCESS && !dict_table_is_intrinsic(table)) { + if (err != DB_SUCCESS) { dict_index_remove_from_cache(table, index); } } @@ -3420,7 +2715,7 @@ error_handling: trx_rollback_to_savepoint(trx, NULL); } - row_drop_table_for_mysql(table_name, trx, FALSE, true, handler); + row_drop_table_for_mysql(table_name, trx, FALSE, true); if (trx_is_started(trx)) { @@ -3489,9 +2784,6 @@ row_table_add_foreign_constraints( DEBUG_SYNC_C("table_add_foreign_constraints"); - /* Check like this shouldn't be done for table that doesn't - have foreign keys but code still continues to run with void action. - Disable it for intrinsic table at-least */ if (err == DB_SUCCESS) { /* Check that also referencing constraints are ok */ dict_names_t fk_tables; @@ -3944,33 +3236,6 @@ row_discard_tablespace( return(err); } - /* For encrypted table, before we discard the tablespace, - we need save the encryption information into table, otherwise, - this information will be lost in fil_discard_tablespace along - with fil_space_free(). */ - if (dict_table_is_encrypted(table)) { - ut_ad(table->encryption_key == NULL - && table->encryption_iv == NULL); - - table->encryption_key = - static_cast<byte*>(mem_heap_alloc(table->heap, - ENCRYPTION_KEY_LEN)); - - table->encryption_iv = - static_cast<byte*>(mem_heap_alloc(table->heap, - ENCRYPTION_KEY_LEN)); - - fil_space_t* space = fil_space_get(table->space); - ut_ad(FSP_FLAGS_GET_ENCRYPTION(space->flags)); - - memcpy(table->encryption_key, - space->encryption_key, - ENCRYPTION_KEY_LEN); - memcpy(table->encryption_iv, - space->encryption_iv, - ENCRYPTION_KEY_LEN); - } - /* Discard the physical file that is used for the tablespace. */ err = fil_discard_tablespace(table->space); @@ -4171,17 +3436,18 @@ void fil_wait_crypt_bg_threads( dict_table_t* table) { - uint start = time(0); - uint last = start; + time_t start = time(0); + time_t last = start; if (table->space != 0) { - fil_space_crypt_mark_space_closing(table->space); + fil_space_crypt_mark_space_closing(table->space, table->crypt_data); } while (table->get_ref_count()> 0) { dict_mutex_exit_for_mysql(); os_thread_sleep(20000); dict_mutex_enter_for_mysql(); - uint now = time(0); + time_t now = time(0); + if (now >= last + 30) { ib::warn() << "Waited " << now - start @@ -4263,23 +3529,7 @@ row_drop_table_from_cache( is going to be destroyed below. */ trx->mod_tables.erase(table); - if (!dict_table_is_intrinsic(table)) { - dict_table_remove_from_cache(table); - } else { - for (dict_index_t* index = UT_LIST_GET_FIRST(table->indexes); - index != NULL; - index = UT_LIST_GET_FIRST(table->indexes)) { - - rw_lock_free(&index->lock); - - UT_LIST_REMOVE(table->indexes, index); - - dict_mem_index_free(index); - } - - dict_mem_table_free(table); - table = NULL; - } + dict_table_remove_from_cache(table); if (!is_temp && dict_load_table(tablename, true, @@ -4298,9 +3548,7 @@ This deletes the fil_space_t if found and the file on disk. @param[in] space_id Tablespace ID @param[in] tablename Table name, same as the tablespace name @param[in] filepath File path of tablespace to delete -@param[in] is_temp Is this a temporary table/tablespace -@param[in] is_encrypted Is this an encrypted table/tablespace -@param[in] trx Transaction handle +@param[in] table_flags table flags @return error code or DB_SUCCESS */ UNIV_INLINE dberr_t @@ -4308,28 +3556,20 @@ row_drop_single_table_tablespace( ulint space_id, const char* tablename, const char* filepath, - bool is_temp, - bool is_encrypted, - trx_t* trx) + ulint table_flags) { dberr_t err = DB_SUCCESS; - /* This might be a temporary single-table tablespace if the table - is compressed and temporary. If so, don't spam the log when we - delete one of these or if we can't find the tablespace. */ - bool print_msg = !is_temp && !is_encrypted; - /* If the tablespace is not in the cache, just delete the file. */ if (!fil_space_for_table_exists_in_mem( - space_id, tablename, print_msg, false, NULL, 0, NULL)) { + space_id, tablename, true, false, NULL, 0, NULL, + table_flags)) { /* Force a delete of any discarded or temporary files. */ fil_delete_file(filepath); - if (print_msg) { - ib::info() << "Removed datafile " << filepath - << " for table " << tablename; - } + ib::info() << "Removed datafile " << filepath + << " for table " << tablename; } else if (fil_delete_tablespace(space_id, BUF_REMOVE_FLUSH_NO_WRITE) != DB_SUCCESS) { @@ -4355,7 +3595,6 @@ will remain locked. because e.g. foreign key column @param[in] nonatomic Whether it is permitted to release and reacquire dict_operation_lock -@param[in,out] handler Table handler @return error code or DB_SUCCESS */ dberr_t row_drop_table_for_mysql( @@ -4363,18 +3602,16 @@ row_drop_table_for_mysql( trx_t* trx, bool drop_db, ibool create_failed, - bool nonatomic, - dict_table_t* handler) + bool nonatomic) { dberr_t err; dict_foreign_t* foreign; - dict_table_t* table = NULL; + dict_table_t* table; char* filepath = NULL; char* tablename = NULL; bool locked_dictionary = false; pars_info_t* info = NULL; mem_heap_t* heap = NULL; - bool is_intrinsic_temp_table = false; DBUG_ENTER("row_drop_table_for_mysql"); DBUG_PRINT("row_drop_table_for_mysql", ("table: '%s'", name)); @@ -4386,35 +3623,24 @@ row_drop_table_for_mysql( trx->op_info = "dropping table"; - if (handler != NULL && dict_table_is_intrinsic(handler)) { - table = handler; - is_intrinsic_temp_table = true; - } - - if (table == NULL) { + if (trx->dict_operation_lock_mode != RW_X_LATCH) { + /* Prevent foreign key checks etc. while we are + dropping the table */ - if (trx->dict_operation_lock_mode != RW_X_LATCH) { - /* Prevent foreign key checks etc. while we are - dropping the table */ - - row_mysql_lock_data_dictionary(trx); + row_mysql_lock_data_dictionary(trx); - locked_dictionary = true; - nonatomic = true; - } + locked_dictionary = true; + nonatomic = true; + } - ut_ad(mutex_own(&dict_sys->mutex)); - ut_ad(rw_lock_own(dict_operation_lock, RW_LOCK_X)); + ut_ad(mutex_own(&dict_sys->mutex)); + ut_ad(rw_lock_own(dict_operation_lock, RW_LOCK_X)); - table = dict_table_open_on_name( - name, TRUE, FALSE, - static_cast<dict_err_ignore_t>( - DICT_ERR_IGNORE_INDEX_ROOT - | DICT_ERR_IGNORE_CORRUPT)); - } else { - table->acquire(); - ut_ad(dict_table_is_intrinsic(table)); - } + table = dict_table_open_on_name( + name, TRUE, FALSE, + static_cast<dict_err_ignore_t>( + DICT_ERR_IGNORE_INDEX_ROOT + | DICT_ERR_IGNORE_CORRUPT)); if (!table) { err = DB_TABLE_NOT_FOUND; @@ -4495,10 +3721,7 @@ row_drop_table_for_mysql( } } - if (!dict_table_is_intrinsic(table)) { - dict_table_prevent_eviction(table); - } - + dict_table_prevent_eviction(table); dict_table_close(table, TRUE, FALSE); /* Check if the table is referenced by foreign key constraints from @@ -4600,16 +3823,11 @@ row_drop_table_for_mysql( fil_wait_crypt_bg_threads(table); if (table->get_ref_count() == 0) { - /* We don't take lock on intrinsic table so nothing to remove.*/ - if (!dict_table_is_intrinsic(table)) { - lock_remove_all_on_table(table, TRUE); - } + lock_remove_all_on_table(table, TRUE); ut_a(table->n_rec_locks == 0); } else if (table->get_ref_count() > 0 || table->n_rec_locks > 0) { ibool added; - ut_ad(!dict_table_is_intrinsic(table)); - added = row_add_table_to_background_drop_list( table->name.m_name); @@ -4641,7 +3859,7 @@ row_drop_table_for_mysql( /* If we get this far then the table to be dropped must not have any table or record locks on it. */ - ut_a(dict_table_is_intrinsic(table) || !lock_table_has_locks(table)); + ut_a(!lock_table_has_locks(table)); switch (trx_get_dict_operation(trx)) { case TRX_DICT_OP_NONE: @@ -4687,6 +3905,13 @@ row_drop_table_for_mysql( /* As we don't insert entries to SYSTEM TABLES for temp-tables we need to avoid running removal of these entries. */ if (!dict_table_is_temporary(table)) { + + /* If table has not yet have crypt_data, try to read it to + make freeing the table easier. */ + if (!table->crypt_data) { + table->crypt_data = fil_space_get_crypt_data(table->space); + } + /* We use the private SQL parser of Innobase to generate the query graphs needed in deleting the dictionary data from system tables in Innobase. Deleting a row from SYS_INDEXES table also @@ -4811,35 +4036,22 @@ row_drop_table_for_mysql( /* remove the index object associated. */ dict_drop_index_tree_in_mem(index, *page_no++); } - err = DB_SUCCESS; + err = row_drop_table_from_cache(tablename, table, trx); + goto funct_exit; } switch (err) { ulint space_id; - bool is_temp; - bool is_encrypted; bool ibd_file_missing; bool is_discarded; - bool shared_tablespace; + ulint table_flags; case DB_SUCCESS: space_id = table->space; ibd_file_missing = table->ibd_file_missing; is_discarded = dict_table_is_discarded(table); - is_temp = dict_table_is_temporary(table); - is_encrypted = dict_table_is_encrypted(table); - shared_tablespace = DICT_TF_HAS_SHARED_SPACE(table->flags); - - /* If there is a temp path then the temp flag is set. - However, during recovery, we might have a temp flag but - not know the temp path */ - ut_a(table->dir_path_of_temp_table == NULL || is_temp); - - /* We do not allow temporary tables with a remote path. */ - ut_a(!(is_temp && DICT_TF_HAS_DATA_DIR(table->flags))); - - /* Make sure the data_dir_path is set if needed. */ - dict_get_and_save_data_dir_path(table, true); + table_flags = table->flags; + ut_ad(!dict_table_is_temporary(table)); err = row_drop_ancillary_fts_tables(table, trx); if (err != DB_SUCCESS) { @@ -4849,15 +4061,12 @@ row_drop_table_for_mysql( /* Determine the tablespace filename before we drop dict_table_t. Free this memory before returning. */ if (DICT_TF_HAS_DATA_DIR(table->flags)) { + dict_get_and_save_data_dir_path(table, true); ut_a(table->data_dir_path); filepath = fil_make_filepath( table->data_dir_path, table->name.m_name, IBD, true); - } else if (table->dir_path_of_temp_table) { - filepath = fil_make_filepath( - table->dir_path_of_temp_table, - NULL, IBD, false); - } else if (!shared_tablespace) { + } else { filepath = fil_make_filepath( NULL, table->name.m_name, IBD, false); } @@ -4869,35 +4078,15 @@ row_drop_table_for_mysql( } /* Do not attempt to drop known-to-be-missing tablespaces, - nor system or shared general tablespaces. */ - if (is_discarded || ibd_file_missing || shared_tablespace + nor the system tablespace. */ + if (is_discarded || ibd_file_missing || is_system_tablespace(space_id)) { - /* For encrypted table, if ibd file can not be decrypt, - we also set ibd_file_missing. We still need to try to - remove the ibd file for this. */ - if (is_discarded || !is_encrypted - || !ibd_file_missing) { - break; - } - } - -#ifdef MYSQL_ENCRYPTION - if (is_encrypted) { - /* Require the mutex to block key rotation. */ - mutex_enter(&master_key_id_mutex); + break; } -#endif /* MYSQL_ENCRYPTION */ /* We can now drop the single-table tablespace. */ err = row_drop_single_table_tablespace( - space_id, tablename, filepath, - is_temp, is_encrypted, trx); - -#ifdef MYSQL_ENCRYPTION - if (is_encrypted) { - mutex_exit(&master_key_id_mutex); - } -#endif /* MYSQL_ENCRYPTION */ + space_id, tablename, filepath, table_flags); break; case DB_OUT_OF_FILE_SPACE: @@ -4974,109 +4163,11 @@ funct_exit: trx->op_info = ""; - /* No need to immediately invoke master thread as there is no work - generated by intrinsic table operation that needs master thread - attention. */ - if (!is_intrinsic_temp_table) { - srv_wake_master_thread(); - } + srv_wake_master_thread(); DBUG_RETURN(err); } -/*********************************************************************//** -Drop all temporary tables during crash recovery. */ -void -row_mysql_drop_temp_tables(void) -/*============================*/ -{ - trx_t* trx; - btr_pcur_t pcur; - mtr_t mtr; - mem_heap_t* heap; - - trx = trx_allocate_for_background(); - trx->op_info = "dropping temporary tables"; - row_mysql_lock_data_dictionary(trx); - - heap = mem_heap_create(200); - - mtr_start(&mtr); - - btr_pcur_open_at_index_side( - true, - dict_table_get_first_index(dict_sys->sys_tables), - BTR_SEARCH_LEAF, &pcur, true, 0, &mtr); - - for (;;) { - const rec_t* rec; - const byte* field; - ulint len; - const char* table_name; - dict_table_t* table; - - btr_pcur_move_to_next_user_rec(&pcur, &mtr); - - if (!btr_pcur_is_on_user_rec(&pcur)) { - break; - } - - /* The high order bit of N_COLS is set unless - ROW_FORMAT=REDUNDANT. */ - rec = btr_pcur_get_rec(&pcur); - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_TABLES__NAME, &len); - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_TABLES__N_COLS, &len); - if (len != 4 - || !(mach_read_from_4(field) & DICT_N_COLS_COMPACT)) { - continue; - } - - /* Older versions of InnoDB, which only supported tables - in ROW_FORMAT=REDUNDANT could write garbage to - SYS_TABLES.MIX_LEN, where we now store the is_temp flag. - Above, we assumed is_temp=0 if ROW_FORMAT=REDUNDANT. */ - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_TABLES__MIX_LEN, &len); - if (len != 4 - || !(mach_read_from_4(field) & DICT_TF2_TEMPORARY)) { - continue; - } - - /* This is a temporary table. */ - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_TABLES__NAME, &len); - if (len == UNIV_SQL_NULL || len == 0) { - /* Corrupted SYS_TABLES.NAME */ - continue; - } - - table_name = mem_heap_strdupl(heap, (const char*) field, len); - - btr_pcur_store_position(&pcur, &mtr); - btr_pcur_commit_specify_mtr(&pcur, &mtr); - - table = dict_load_table(table_name, true, - DICT_ERR_IGNORE_NONE); - - if (table) { - row_drop_table_for_mysql(table_name, trx, FALSE, FALSE); - trx_commit_for_mysql(trx); - } - - mtr_start(&mtr); - btr_pcur_restore_position(BTR_SEARCH_LEAF, - &pcur, &mtr); - } - - btr_pcur_close(&pcur); - mtr_commit(&mtr); - mem_heap_free(heap); - row_mysql_unlock_data_dictionary(trx); - trx_free_for_background(trx); -} - /*******************************************************************//** Drop all foreign keys in a database, see Bug#18942. Called at the end of row_drop_database_for_mysql(). @@ -5740,14 +4831,12 @@ end: goto funct_exit; } -#ifdef MYSQL_VIRTUAL_COLUMNS /* In case of copy alter, template db_name and table_name should be renamed only for newly created table. */ if (table->vc_templ != NULL && !new_is_tmp) { innobase_rename_vc_templ(table); } -#endif /* We only want to switch off some of the type checking in an ALTER TABLE...ALGORITHM=COPY, not in a RENAME. */ @@ -6145,7 +5234,8 @@ row_mysql_close(void) { ut_a(UT_LIST_GET_LEN(row_mysql_drop_list) == 0); - mutex_free(&row_drop_list_mutex); - - row_mysql_drop_list_inited = FALSE; + if (row_mysql_drop_list_inited) { + mutex_free(&row_drop_list_mutex); + row_mysql_drop_list_inited = FALSE; + } } diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc index bac55694056..b41e3642cf0 100644 --- a/storage/innobase/row/row0purge.cc +++ b/storage/innobase/row/row0purge.cc @@ -515,22 +515,22 @@ row_purge_remove_sec_if_poss_leaf( page = btr_cur_get_page(btr_cur); if (!lock_test_prdt_page_lock( - trx, - page_get_space_id(page), - page_get_page_no(page)) - && page_get_n_recs(page) < 2 - && page_get_page_no(page) != - dict_index_get_page(index)) { + trx, + page_get_space_id(page), + page_get_page_no(page)) + && page_get_n_recs(page) < 2 + && btr_cur_get_block(btr_cur) + ->page.id.page_no() != + dict_index_get_page(index)) { /* this is the last record on page, and it has a "page" lock on it, which mean search is still depending on it, so do not delete */ -#ifdef UNIV_DEBUG - ib::info() << "skip purging last" - " record on page " - << page_get_page_no(page) - << "."; -#endif /* UNIV_DEBUG */ + DBUG_LOG("purge", + "skip purging last" + " record on page " + << btr_cur_get_block(btr_cur) + ->page.id); btr_pcur_close(&pcur); mtr_commit(&mtr); @@ -868,7 +868,6 @@ try_again: goto err_exit; } -#ifdef MYSQL_VIRTUAL_COLUMNS if (node->table->n_v_cols && !node->table->vc_templ && dict_table_has_indexed_v_cols(node->table)) { /* Need server fully up for virtual column computation */ @@ -886,7 +885,6 @@ try_again: /* Initialize the template for the table */ innobase_init_vc_templ(node->table); } -#endif /* MYSQL_VIRTUAL_COLUMNS */ /* Disable purging for temp-tables as they are short-lived and no point in re-organzing such short lived tables */ @@ -1120,6 +1118,8 @@ row_purge_step( row_purge_end(thr); } + innobase_reset_background_thd(thr_get_trx(thr)->mysql_thd); + return(thr); } diff --git a/storage/innobase/row/row0quiesce.cc b/storage/innobase/row/row0quiesce.cc index 08d119768a9..d26d81c9239 100644 --- a/storage/innobase/row/row0quiesce.cc +++ b/storage/innobase/row/row0quiesce.cc @@ -480,210 +480,6 @@ row_quiesce_write_cfg( return(err); } -#ifdef MYSQL_ENCRYPTION - -/** Write the transfer key to CFP file. -@param[in] table write the data for this table -@param[in] file file to write to -@param[in] thd session -@return DB_SUCCESS or error code. */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -row_quiesce_write_transfer_key( - const dict_table_t* table, - FILE* file, - THD* thd) -{ - byte key_size[sizeof(ib_uint32_t)]; - byte row[ENCRYPTION_KEY_LEN * 3]; - byte* ptr = row; - byte* transfer_key = ptr; - lint elen; - - ut_ad(table->encryption_key != NULL - && table->encryption_iv != NULL); - - /* Write the encryption key size. */ - mach_write_to_4(key_size, ENCRYPTION_KEY_LEN); - - if (fwrite(&key_size, 1, sizeof(key_size), file) - != sizeof(key_size)) { - ib_senderrf( - thd, IB_LOG_LEVEL_WARN, ER_IO_WRITE_ERROR, - errno, strerror(errno), - "while writing key size."); - - return(DB_IO_ERROR); - } - - /* Generate and write the transfer key. */ - Encryption::random_value(transfer_key); - if (fwrite(transfer_key, 1, ENCRYPTION_KEY_LEN, file) - != ENCRYPTION_KEY_LEN) { - ib_senderrf( - thd, IB_LOG_LEVEL_WARN, ER_IO_WRITE_ERROR, - errno, strerror(errno), - "while writing transfer key."); - - return(DB_IO_ERROR); - } - - ptr += ENCRYPTION_KEY_LEN; - - /* Encrypt tablespace key. */ - elen = my_aes_encrypt( - reinterpret_cast<unsigned char*>(table->encryption_key), - ENCRYPTION_KEY_LEN, - ptr, - reinterpret_cast<unsigned char*>(transfer_key), - ENCRYPTION_KEY_LEN, - my_aes_256_ecb, - NULL, false); - - if (elen == MY_AES_BAD_DATA) { - ib_senderrf( - thd, IB_LOG_LEVEL_WARN, ER_IO_WRITE_ERROR, - errno, strerror(errno), - "while encrypt tablespace key."); - return(DB_ERROR); - } - - /* Write encrypted tablespace key */ - if (fwrite(ptr, 1, ENCRYPTION_KEY_LEN, file) - != ENCRYPTION_KEY_LEN) { - ib_senderrf( - thd, IB_LOG_LEVEL_WARN, ER_IO_WRITE_ERROR, - errno, strerror(errno), - "while writing encrypted tablespace key."); - - return(DB_IO_ERROR); - } - ptr += ENCRYPTION_KEY_LEN; - - /* Encrypt tablespace iv. */ - elen = my_aes_encrypt( - reinterpret_cast<unsigned char*>(table->encryption_iv), - ENCRYPTION_KEY_LEN, - ptr, - reinterpret_cast<unsigned char*>(transfer_key), - ENCRYPTION_KEY_LEN, - my_aes_256_ecb, - NULL, false); - - if (elen == MY_AES_BAD_DATA) { - ib_senderrf( - thd, IB_LOG_LEVEL_WARN, ER_IO_WRITE_ERROR, - errno, strerror(errno), - "while encrypt tablespace iv."); - return(DB_ERROR); - } - - /* Write encrypted tablespace iv */ - if (fwrite(ptr, 1, ENCRYPTION_KEY_LEN, file) - != ENCRYPTION_KEY_LEN) { - ib_senderrf( - thd, IB_LOG_LEVEL_WARN, ER_IO_WRITE_ERROR, - errno, strerror(errno), - "while writing encrypted tablespace iv."); - - return(DB_IO_ERROR); - } - - return(DB_SUCCESS); -} - -/** Write the encryption data after quiesce. -@param[in] table write the data for this table -@param[in] thd session -@return DB_SUCCESS or error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -row_quiesce_write_cfp( - dict_table_t* table, - THD* thd) -{ - dberr_t err; - char name[OS_FILE_MAX_PATH]; - - /* If table is not encrypted, return. */ - if (!dict_table_is_encrypted(table)) { - return(DB_SUCCESS); - } - - /* Get the encryption key and iv from space */ - /* For encrypted table, before we discard the tablespace, - we need save the encryption information into table, otherwise, - this information will be lost in fil_discard_tablespace along - with fil_space_free(). */ - if (table->encryption_key == NULL) { - table->encryption_key = - static_cast<byte*>(mem_heap_alloc(table->heap, - ENCRYPTION_KEY_LEN)); - - table->encryption_iv = - static_cast<byte*>(mem_heap_alloc(table->heap, - ENCRYPTION_KEY_LEN)); - - fil_space_t* space = fil_space_get(table->space); - ut_ad(space != NULL && FSP_FLAGS_GET_ENCRYPTION(space->flags)); - - memcpy(table->encryption_key, - space->encryption_key, - ENCRYPTION_KEY_LEN); - memcpy(table->encryption_iv, - space->encryption_iv, - ENCRYPTION_KEY_LEN); - } - - srv_get_encryption_data_filename(table, name, sizeof(name)); - - ib::info() << "Writing table encryption data to '" << name << "'"; - - FILE* file = fopen(name, "w+b"); - - if (file == NULL) { - ib_errf(thd, IB_LOG_LEVEL_WARN, ER_CANT_CREATE_FILE, - name, errno, strerror(errno)); - - err = DB_IO_ERROR; - } else { - err = row_quiesce_write_transfer_key(table, file, thd); - - if (fflush(file) != 0) { - - char msg[BUFSIZ]; - - ut_snprintf(msg, sizeof(msg), "%s flush() failed", - name); - - ib_senderrf( - thd, IB_LOG_LEVEL_WARN, ER_IO_WRITE_ERROR, - errno, strerror(errno), msg); - - err = DB_IO_ERROR; - } - - if (fclose(file) != 0) { - char msg[BUFSIZ]; - - ut_snprintf(msg, sizeof(msg), "%s flose() failed", - name); - - ib_senderrf( - thd, IB_LOG_LEVEL_WARN, ER_IO_WRITE_ERROR, - errno, strerror(errno), msg); - err = DB_IO_ERROR; - } - } - - /* Clean the encryption information */ - table->encryption_key = NULL; - table->encryption_iv = NULL; - - return(err); -} -#endif /* MYSQL_ENCRYPTION */ - /*********************************************************************//** Check whether a table has an FTS index defined on it. @return true if an FTS index exists on the table */ @@ -744,24 +540,9 @@ row_quiesce_table_start( } if (!trx_is_interrupted(trx)) { - extern ib_mutex_t master_key_id_mutex; - -#ifdef MYSQL_ENCRYPTION - if (dict_table_is_encrypted(table)) { - /* Require the mutex to block key rotation. */ - mutex_enter(&master_key_id_mutex); - } -#endif /* MYSQL_ENCRYPTION */ - buf_LRU_flush_or_remove_pages( table->space, BUF_REMOVE_FLUSH_WRITE, trx); -#ifdef MYSQL_ENCRYPTION - if (dict_table_is_encrypted(table)) { - mutex_exit(&master_key_id_mutex); - } -#endif /* MYSQL_ENCRYPTION */ - if (trx_is_interrupted(trx)) { ib::warn() << "Quiesce aborted!"; @@ -771,12 +552,6 @@ row_quiesce_table_start( ib::warn() << "There was an error writing to the" " meta data file"; -#ifdef MYSQL_ENCRYPTION - } else if (row_quiesce_write_cfp(table, trx->mysql_thd) - != DB_SUCCESS) { - ib::warn() << "There was an error writing to the" - " encryption info file"; -#endif /* MYSQL_ENCRYPTION */ } else { ib::info() << "Table " << table->name << " flushed to disk"; @@ -829,19 +604,6 @@ row_quiesce_table_complete( ib::info() << "Deleting the meta-data file '" << cfg_name << "'"; - if (dict_table_is_encrypted(table)) { - char cfp_name[OS_FILE_MAX_PATH]; - - srv_get_encryption_data_filename(table, - cfp_name, - sizeof(cfp_name)); - - os_file_delete_if_exists(innodb_data_file_key, cfp_name, NULL); - - ib::info() << "Deleting the meta-data file '" - << cfp_name << "'"; - } - if (trx_purge_state() != PURGE_STATE_DISABLED) { trx_purge_run(); } @@ -887,15 +649,6 @@ row_quiesce_set_state( ER_TABLE_IN_SYSTEM_TABLESPACE, table_name); return(DB_UNSUPPORTED); - - } else if (DICT_TF_HAS_SHARED_SPACE(table->flags)) { - std::ostringstream err_msg; - err_msg << "FLUSH TABLES FOR EXPORT on table " << table->name - << " in a general tablespace."; - ib_senderrf(trx->mysql_thd, IB_LOG_LEVEL_WARN, - ER_NOT_SUPPORTED_YET, err_msg.str().c_str()); - - return(DB_UNSUPPORTED); } else if (row_quiesce_table_has_fts_index(table)) { ib_senderrf(trx->mysql_thd, IB_LOG_LEVEL_WARN, diff --git a/storage/innobase/row/row0row.cc b/storage/innobase/row/row0row.cc index 5c71e6e98bc..0f15299e71a 100644 --- a/storage/innobase/row/row0row.cc +++ b/storage/innobase/row/row0row.cc @@ -129,7 +129,8 @@ row_build_index_entry_low( ut_ad(v_col->v_pos < dtuple_get_n_v_fields(row)); dfield2 = dtuple_get_nth_v_field(row, v_col->v_pos); - ut_ad(dfield_is_null(dfield2) || dfield2->data); + ut_ad(dfield_is_null(dfield2) || + dfield_get_len(dfield2) == 0 || dfield2->data); } else { dfield2 = dtuple_get_nth_field(row, col_no); ut_ad(dfield_get_type(dfield2)->mtype == DATA_MISSING diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index d23a0d8c432..4cd7a443d79 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -221,7 +221,6 @@ row_sel_sec_rec_is_for_clust_rec( /* For virtual column, its value will need to be reconstructed from base column in cluster index */ if (is_virtual) { -#ifdef MYSQL_VIRTUAL_COLUMNS const dict_v_col_t* v_col; const dtuple_t* row; dfield_t* vfield; @@ -243,7 +242,6 @@ row_sel_sec_rec_is_for_clust_rec( clust_len = vfield->len; clust_field = static_cast<byte*>(vfield->data); -#endif /* MYSQL_VIRTUAL_COLUMNS */ } else { clust_pos = dict_col_get_clust_pos(col, clust_index); @@ -3617,10 +3615,7 @@ row_sel_get_clust_rec_for_mysql( func_exit: *out_rec = clust_rec; - /* Store the current position if select_lock_type is not - LOCK_NONE or if we are scanning using InnoDB APIs */ - if (prebuilt->select_lock_type != LOCK_NONE - || prebuilt->innodb_api) { + if (prebuilt->select_lock_type != LOCK_NONE) { /* We may use the cursor in update or in unlock_row(): store its position */ @@ -4091,320 +4086,6 @@ row_search_idx_cond_check( return(result); } -/** Traverse to next/previous record. -@param[in] moves_up if true, move to next record else previous -@param[in] match_mode 0 or ROW_SEL_EXACT or ROW_SEL_EXACT_PREFIX -@param[in,out] pcur cursor to record -@param[in] mtr mini transaction - -@return DB_SUCCESS or error code */ -static -dberr_t -row_search_traverse( - bool moves_up, - ulint match_mode, - btr_pcur_t* pcur, - mtr_t* mtr) -{ - dberr_t err = DB_SUCCESS; - - if (moves_up) { - if (!btr_pcur_move_to_next(pcur, mtr)) { - err = (match_mode != 0) - ? DB_RECORD_NOT_FOUND : DB_END_OF_INDEX; - return(err); - } - } else { - if (!btr_pcur_move_to_prev(pcur, mtr)) { - err = (match_mode != 0) - ? DB_RECORD_NOT_FOUND : DB_END_OF_INDEX; - return(err); - } - } - - return(err); -} - -/** Searches for rows in the database using cursor. -Function is for temporary tables that are not shared accross connections -and so lot of complexity is reduced especially locking and transaction related. -The cursor is an iterator over the table/index. - -@param[out] buf buffer for the fetched row in MySQL format -@param[in] mode search mode PAGE_CUR_L -@param[in,out] prebuilt prebuilt struct for the table handler; - this contains the info to search_tuple, - index; if search tuple contains 0 field then - we position the cursor at start or the end of - index, depending on 'mode' -@param[in] match_mode 0 or ROW_SEL_EXACT or ROW_SEL_EXACT_PREFIX -@param[in] direction 0 or ROW_SEL_NEXT or ROW_SEL_PREV; - Note: if this is != 0, then prebuilt must has a - pcur with stored position! In opening of a - cursor 'direction' should be 0. -@return DB_SUCCESS or error code */ -dberr_t -row_search_no_mvcc( - byte* buf, - page_cur_mode_t mode, - row_prebuilt_t* prebuilt, - ulint match_mode, - ulint direction) -{ - dict_index_t* index = prebuilt->index; - const dtuple_t* search_tuple = prebuilt->search_tuple; - btr_pcur_t* pcur = prebuilt->pcur; - - const rec_t* result_rec = NULL; - const rec_t* clust_rec = NULL; - - dberr_t err = DB_SUCCESS; - - mem_heap_t* heap = NULL; - ulint offsets_[REC_OFFS_NORMAL_SIZE]; - ulint* offsets = offsets_; - rec_offs_init(offsets_); - ut_ad(index && pcur && search_tuple); - - /* Step-0: Re-use the cached mtr. */ - mtr_t* mtr = &index->last_sel_cur->mtr; - dict_index_t* clust_index = dict_table_get_first_index(index->table); - - /* Step-1: Build the select graph. */ - if (direction == 0 && prebuilt->sel_graph == NULL) { - row_prebuild_sel_graph(prebuilt); - } - - que_thr_t* thr = que_fork_get_first_thr(prebuilt->sel_graph); - - bool moves_up; - - if (direction == 0) { - - if (mode == PAGE_CUR_GE || mode == PAGE_CUR_G) { - moves_up = true; - } else { - moves_up = false; - } - - } else if (direction == ROW_SEL_NEXT) { - moves_up = true; - } else { - moves_up = false; - } - - /* Step-2: Open or Restore the cursor. - If search key is specified, cursor is open using the key else - cursor is open to return all the records. */ - if (direction != 0) { - if (index->last_sel_cur->invalid) { - - /* Index tree has changed and so active cached cursor - is no more valid. Re-set it based on the last selected - position. */ - index->last_sel_cur->release(); - - mtr_start(mtr); - dict_disable_redo_if_temporary(index->table, mtr); - - mem_heap_t* heap = mem_heap_create(256); - dtuple_t* tuple; - - tuple = dict_index_build_data_tuple( - index, pcur->old_rec, - pcur->old_n_fields, heap); - - btr_pcur_open_with_no_init( - index, tuple, pcur->search_mode, - BTR_SEARCH_LEAF, pcur, 0, mtr); - - mem_heap_free(heap); - } else { - /* Restore the cursor for reading next record from cache - information. */ - ut_ad(index->last_sel_cur->rec != NULL); - - pcur->btr_cur.page_cur.rec = index->last_sel_cur->rec; - pcur->btr_cur.page_cur.block = - index->last_sel_cur->block; - - err = row_search_traverse( - moves_up, match_mode, pcur, mtr); - if (err != DB_SUCCESS) { - return(err); - } - } - } else { - /* There could be previous uncommitted transaction if SELECT - is operation as part of SELECT (IF NOT FOUND) INSERT - (IF DUPLICATE) UPDATE plan. */ - index->last_sel_cur->release(); - - /* Capture table snapshot in form of trx-id. */ - index->trx_id = dict_table_get_curr_table_sess_trx_id( - index->table); - - /* Fresh search commences. */ - mtr_start(mtr); - dict_disable_redo_if_temporary(index->table, mtr); - - if (dtuple_get_n_fields(search_tuple) > 0) { - - btr_pcur_open_with_no_init( - index, search_tuple, mode, BTR_SEARCH_LEAF, - pcur, 0, mtr); - - } else if (mode == PAGE_CUR_G || mode == PAGE_CUR_L) { - - btr_pcur_open_at_index_side( - mode == PAGE_CUR_G, index, BTR_SEARCH_LEAF, - pcur, false, 0, mtr); - - } - } - - /* Step-3: Traverse the records filtering non-qualifiying records. */ - for (/* No op */; - err == DB_SUCCESS; - err = row_search_traverse(moves_up, match_mode, pcur, mtr)) { - - const rec_t* rec = btr_pcur_get_rec(pcur); - - if (page_rec_is_infimum(rec) - || page_rec_is_supremum(rec) - || rec_get_deleted_flag( - rec, dict_table_is_comp(index->table))) { - - /* The infimum record on a page cannot be in the - result set, and neither can a record lock be placed on - it: we skip such a record. */ - continue; - } - - offsets = rec_get_offsets( - rec, index, offsets, ULINT_UNDEFINED, &heap); - - /* Note that we cannot trust the up_match value in the cursor - at this place because we can arrive here after moving the - cursor! Thus we have to recompare rec and search_tuple to - determine if they match enough. */ - if (match_mode == ROW_SEL_EXACT) { - /* Test if the index record matches completely to - search_tuple in prebuilt: if not, then we return with - DB_RECORD_NOT_FOUND */ - if (0 != cmp_dtuple_rec(search_tuple, rec, offsets)) { - err = DB_RECORD_NOT_FOUND; - break; - } - } else if (match_mode == ROW_SEL_EXACT_PREFIX) { - if (!cmp_dtuple_is_prefix_of_rec( - search_tuple, rec, offsets)) { - err = DB_RECORD_NOT_FOUND; - break; - } - } - - /* Get the clustered index. We always need clustered index - record for snapshort verification. */ - if (index != clust_index) { - - err = row_sel_get_clust_rec_for_mysql( - prebuilt, index, rec, thr, &clust_rec, - &offsets, &heap, NULL, mtr); - - if (err != DB_SUCCESS) { - break; - } - - if (rec_get_deleted_flag( - clust_rec, dict_table_is_comp(index->table))) { - - /* The record is delete marked in clustered - index. We can skip this record. */ - continue; - } - - result_rec = clust_rec; - } else { - result_rec = rec; - } - - /* Step-4: Check if row is part of the consistent view that was - captured while SELECT statement started execution. */ - { - trx_id_t trx_id; - - ulint len; - ulint trx_id_off = rec_get_nth_field_offs( - offsets, clust_index->n_uniq, &len); - - ut_ad(len == DATA_TRX_ID_LEN); - - trx_id = trx_read_trx_id(result_rec + trx_id_off); - - if (trx_id > index->trx_id) { - /* This row was recently added skip it from - SELECT view. */ - continue; - } - } - - /* Step-5: Cache the row-id of selected row to prebuilt cache.*/ - if (prebuilt->clust_index_was_generated) { - row_sel_store_row_id_to_prebuilt( - prebuilt, result_rec, clust_index, offsets); - } - - /* Step-6: Convert selected record to MySQL format and - store it. */ - if (prebuilt->template_type == ROW_MYSQL_DUMMY_TEMPLATE) { - - const rec_t* ret_rec = - (index != clust_index - && prebuilt->need_to_access_clustered) - ? result_rec : rec; - - offsets = rec_get_offsets(ret_rec, index, offsets, - ULINT_UNDEFINED, &heap); - - memcpy(buf + 4, ret_rec - rec_offs_extra_size(offsets), - rec_offs_size(offsets)); - - mach_write_to_4(buf, rec_offs_extra_size(offsets) + 4); - - } else if (!row_sel_store_mysql_rec( - buf, prebuilt, result_rec, NULL, TRUE, - clust_index, offsets)) { - err = DB_ERROR; - break; - } - - /* Step-7: Store cursor position to fetch next record. - MySQL calls this function iteratively get_next(), get_next() - fashion. */ - ut_ad(err == DB_SUCCESS); - index->last_sel_cur->rec = btr_pcur_get_rec(pcur); - index->last_sel_cur->block = btr_pcur_get_block(pcur); - - /* This is needed in order to restore the cursor if index - structure changes while SELECT is still active. */ - pcur->old_rec = dict_index_copy_rec_order_prefix( - index, rec, &pcur->old_n_fields, - &pcur->old_rec_buf, &pcur->buf_size); - - break; - } - - if (err != DB_SUCCESS) { - index->last_sel_cur->release(); - } - - if (heap != NULL) { - mem_heap_free(heap); - } - return(err); -} - /** Extract virtual column data from a virtual index record and fill a dtuple @param[in] rec the virtual (secondary) index record @param[in] index the virtual index @@ -4459,7 +4140,7 @@ row_sel_fill_vrow( } /** Searches for rows in the database using cursor. -Function is mainly used for tables that are shared accorss connection and +Function is mainly used for tables that are shared across connections and so it employs technique that can help re-construct the rows that transaction is suppose to see. It also has optimization such as pre-caching the rows, using AHI, etc. @@ -4713,13 +4394,11 @@ row_search_mvcc( && dict_index_is_clust(index) && !prebuilt->templ_contains_blob && !prebuilt->used_in_HANDLER - && (prebuilt->mysql_row_len < UNIV_PAGE_SIZE / 8) - && !prebuilt->innodb_api) { + && (prebuilt->mysql_row_len < UNIV_PAGE_SIZE / 8)) { mode = PAGE_CUR_GE; if (trx->mysql_n_tables_locked == 0 - && !prebuilt->ins_sel_stmt && prebuilt->select_lock_type == LOCK_NONE && trx->isolation_level > TRX_ISO_READ_UNCOMMITTED && MVCC::is_view_active(trx->read_view)) { @@ -5590,7 +5269,7 @@ locks_ok: This optimization can avoid many IOs for certain schemas. */ ibool row_contains_all_values = TRUE; - int i; + uint i; for (i = 0; i < prebuilt->n_template; i++) { /* Condition (1) from above: is the field in the index (prefix or not)? */ @@ -5758,7 +5437,6 @@ requires_clust_rec: && !prebuilt->templ_contains_fixed_point && !prebuilt->clust_index_was_generated && !prebuilt->used_in_HANDLER - && !prebuilt->innodb_api && prebuilt->template_type != ROW_MYSQL_DUMMY_TEMPLATE && !prebuilt->in_fts_query) { @@ -5848,7 +5526,7 @@ requires_clust_rec: rec_offs_size(offsets)); mach_write_to_4(buf, rec_offs_extra_size(offsets) + 4); - } else if (!prebuilt->idx_cond && !prebuilt->innodb_api) { + } else if (!prebuilt->idx_cond) { /* The record was not yet converted to MySQL format. */ if (!row_sel_store_mysql_rec( buf, prebuilt, result_rec, vrow, @@ -5891,18 +5569,13 @@ idx_cond_failed: || !dict_index_is_clust(index) || direction != 0 || prebuilt->select_lock_type != LOCK_NONE - || prebuilt->used_in_HANDLER - || prebuilt->innodb_api) { + || prebuilt->used_in_HANDLER) { /* Inside an update always store the cursor position */ if (!spatial_search) { btr_pcur_store_position(pcur, &mtr); } - - if (prebuilt->innodb_api) { - prebuilt->innodb_api_rec = result_rec; - } } goto normal_return; @@ -6362,29 +6035,7 @@ row_search_autoinc_read_column( data = rec_get_nth_field(rec, offsets, col_no, &len); - switch (mtype) { - case DATA_INT: - ut_a(len <= sizeof value); - value = mach_read_int_type(data, len, unsigned_type); - break; - - case DATA_FLOAT: - ut_a(len == sizeof(float)); - value = (ib_uint64_t) mach_float_read(data); - break; - - case DATA_DOUBLE: - ut_a(len == sizeof(double)); - value = (ib_uint64_t) mach_double_read(data); - break; - - default: - ut_error; - } - - if (!unsigned_type && static_cast<int64_t>(value) < 0) { - value = 0; - } + value = row_parse_int(data, len, mtype, unsigned_type); func_exit: if (UNIV_LIKELY_NULL(heap)) { @@ -6430,42 +6081,27 @@ row_search_get_max_rec( return(rec); } -/*******************************************************************//** -Read the max AUTOINC value from an index. -@return DB_SUCCESS if all OK else error code, DB_RECORD_NOT_FOUND if -column name can't be found in index */ -dberr_t -row_search_max_autoinc( -/*===================*/ - dict_index_t* index, /*!< in: index to search */ - const char* col_name, /*!< in: name of autoinc column */ - ib_uint64_t* value) /*!< out: AUTOINC value read */ +/** Read the max AUTOINC value from an index. +@param[in] index index starting with an AUTO_INCREMENT column +@return the largest AUTO_INCREMENT value +@retval 0 if no records were found */ +ib_uint64_t +row_search_max_autoinc(dict_index_t* index) { - dict_field_t* dfield = dict_index_get_nth_field(index, 0); - dberr_t error = DB_SUCCESS; - *value = 0; - - if (strcmp(col_name, dfield->name) != 0) { - error = DB_RECORD_NOT_FOUND; - } else { - mtr_t mtr; - const rec_t* rec; - - mtr_start(&mtr); + const dict_field_t* dfield = dict_index_get_nth_field(index, 0); - rec = row_search_get_max_rec(index, &mtr); + ib_uint64_t value = 0; - if (rec != NULL) { - ibool unsigned_type = ( - dfield->col->prtype & DATA_UNSIGNED); - - *value = row_search_autoinc_read_column( - index, rec, 0, - dfield->col->mtype, unsigned_type); - } + mtr_t mtr; + mtr.start(); - mtr_commit(&mtr); + if (const rec_t* rec = row_search_get_max_rec(index, &mtr)) { + value = row_search_autoinc_read_column( + index, rec, 0, + dfield->col->mtype, + dfield->col->prtype & DATA_UNSIGNED); } - return(error); + mtr.commit(); + return(value); } diff --git a/storage/innobase/row/row0trunc.cc b/storage/innobase/row/row0trunc.cc index 25dc274b0c5..b8002b439da 100644 --- a/storage/innobase/row/row0trunc.cc +++ b/storage/innobase/row/row0trunc.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2013, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -395,8 +396,6 @@ public: IORequest request(IORequest::WRITE); - request.disable_compression(); - io_err = os_file_write( request, m_log_file_name, handle, log_buf, 0, sz); @@ -488,8 +487,6 @@ public: IORequest request(IORequest::WRITE); - request.disable_compression(); - err = os_file_write( request, m_log_file_name, handle, buffer, 0, sizeof(buffer)); @@ -671,8 +668,6 @@ TruncateLogParser::parse( IORequest request(IORequest::READ); - request.disable_compression(); - /* Align the memory for file i/o if we might have O_DIRECT set*/ byte* log_buf = static_cast<byte*>(ut_align(buf, UNIV_PAGE_SIZE)); @@ -909,7 +904,7 @@ TruncateLogger::operator()(mtr_t* mtr, btr_pcur_t* pcur) /* For compressed tables we need to store extra meta-data required during btr_create(). */ - if (fsp_flags_is_compressed(m_flags)) { + if (FSP_FLAGS_GET_ZIP_SSIZE(m_flags)) { const dict_index_t* dict_index = find(index.m_id); @@ -1229,12 +1224,6 @@ row_truncate_complete( { bool is_file_per_table = dict_table_is_file_per_table(table); - if (table->memcached_sync_count == DICT_TABLE_IN_DDL) { - /* We need to set the memcached sync back to 0, unblock - memcached operations. */ - table->memcached_sync_count = 0; - } - row_mysql_unlock_data_dictionary(trx); DEBUG_SYNC_C("ib_trunc_table_trunc_completing"); @@ -1315,7 +1304,6 @@ row_truncate_fts( fts_table.name = table->name; fts_table.flags2 = table->flags2; fts_table.flags = table->flags; - fts_table.tablespace = table->tablespace; fts_table.space = table->space; /* table->data_dir_path is used for FTS AUX table @@ -1326,20 +1314,6 @@ row_truncate_fts( ut_ad(table->data_dir_path != NULL); } - /* table->tablespace() may not be always populated or - if table->tablespace() uses "innodb_general" name, - fetch the real name. */ - if (DICT_TF_HAS_SHARED_SPACE(table->flags) - && (table->tablespace() == NULL - || dict_table_has_temp_general_tablespace_name( - table->tablespace()))) { - dict_get_and_save_space_name(table, true); - ut_ad(table->tablespace() != NULL); - ut_ad(!dict_table_has_temp_general_tablespace_name( - table->tablespace())); - } - - fts_table.tablespace = table->tablespace(); fts_table.data_dir_path = table->data_dir_path; dberr_t err; @@ -1610,8 +1584,6 @@ row_truncate_prepare(dict_table_t* table, ulint* flags) dict_get_and_save_data_dir_path(table, true); - dict_get_and_save_space_name(table, true); - if (*flags != ULINT_UNDEFINED) { dberr_t err = fil_prepare_for_truncate(table->space); @@ -1850,23 +1822,6 @@ row_truncate_table_for_mysql( table, trx, fsp_flags, logger, err)); } - /* Check if memcached DML is running on this table. if is, we don't - allow truncate this table. */ - if (table->memcached_sync_count != 0) { - ib::error() << "Cannot truncate table " - << table->name - << " by DROP+CREATE because there are memcached" - " operations running on it."; - err = DB_ERROR; - trx_rollback_to_savepoint(trx, NULL); - return(row_truncate_complete( - table, trx, fsp_flags, logger, err)); - } else { - /* We need to set this counter to -1 for blocking - memcached operations. */ - table->memcached_sync_count = DICT_TABLE_IN_DDL; - } - /* Remove all locks except the table-level X lock. */ lock_remove_all_on_table(table, FALSE); trx->table_id = table->id; @@ -2015,9 +1970,9 @@ row_truncate_table_for_mysql( return(row_truncate_complete( table, trx, fsp_flags, logger, err)); } - } else { /* For temporary tables we don't have entries in SYSTEM TABLES*/ + ut_ad(fsp_is_system_temporary(table->space)); for (dict_index_t* index = UT_LIST_GET_FIRST(table->indexes); index != NULL; index = UT_LIST_GET_NEXT(indexes, index)) { @@ -2040,10 +1995,7 @@ row_truncate_table_for_mysql( } } - if (is_file_per_table - && !dict_table_is_temporary(table) - && fsp_flags != ULINT_UNDEFINED) { - + if (is_file_per_table && fsp_flags != ULINT_UNDEFINED) { fil_reinit_space_header( table->space, table->indexes.count + FIL_IBD_FILE_INITIAL_SIZE + 1); @@ -2218,75 +2170,51 @@ truncate_t::fixup_tables_in_non_system_tablespace() done and erased from this list. */ ut_a((*it)->m_space_id != TRX_SYS_SPACE); - /* Step-1: Drop tablespace (only for single-tablespace), - drop indexes and re-create indexes. */ - - if (fsp_is_file_per_table((*it)->m_space_id, - (*it)->m_tablespace_flags)) { - /* The table is file_per_table */ - - ib::info() << "Completing truncate for table with " - "id (" << (*it)->m_old_table_id << ") " - "residing in file-per-table tablespace with " - "id (" << (*it)->m_space_id << ")"; + /* Drop tablespace, drop indexes and re-create indexes. */ - if (!fil_space_get((*it)->m_space_id)) { + ib::info() << "Completing truncate for table with " + "id (" << (*it)->m_old_table_id << ") " + "residing in file-per-table tablespace with " + "id (" << (*it)->m_space_id << ")"; - /* Create the database directory for name, - if it does not exist yet */ - fil_create_directory_for_tablename( - (*it)->m_tablename); + if (!fil_space_get((*it)->m_space_id)) { - err = fil_ibd_create( - (*it)->m_space_id, - (*it)->m_tablename, - (*it)->m_dir_path, - (*it)->m_tablespace_flags, - FIL_IBD_FILE_INITIAL_SIZE, - (*it)->m_encryption, - (*it)->m_key_id); + /* Create the database directory for name, + if it does not exist yet */ + fil_create_directory_for_tablename( + (*it)->m_tablename); - if (err != DB_SUCCESS) { - /* If checkpoint is not yet done - and table is dropped and then we might - still have REDO entries for this table - which are INVALID. Ignore them. */ - ib::warn() << "Failed to create" - " tablespace for " - << (*it)->m_space_id - << " space-id"; - err = DB_ERROR; - break; - } - } - - ut_ad(fil_space_get((*it)->m_space_id)); - - err = fil_recreate_tablespace( + err = fil_ibd_create( (*it)->m_space_id, - (*it)->m_format_flags, - (*it)->m_tablespace_flags, (*it)->m_tablename, - **it, log_get_lsn()); - - } else { - /* Table is in a shared tablespace */ + (*it)->m_dir_path, + (*it)->m_tablespace_flags, + FIL_IBD_FILE_INITIAL_SIZE, + (*it)->m_encryption, + (*it)->m_key_id); - ib::info() << "Completing truncate for table with " - "id (" << (*it)->m_old_table_id << ") " - "residing in shared tablespace with " - "id (" << (*it)->m_space_id << ")"; + if (err != DB_SUCCESS) { + /* If checkpoint is not yet done + and table is dropped and then we might + still have REDO entries for this table + which are INVALID. Ignore them. */ + ib::warn() << "Failed to create" + " tablespace for " + << (*it)->m_space_id + << " space-id"; + err = DB_ERROR; + break; + } + } - /* Temp-tables in temp-tablespace are never restored.*/ - ut_ad((*it)->m_space_id != srv_tmp_space.space_id()); + ut_ad(fil_space_get((*it)->m_space_id)); - err = fil_recreate_table( - (*it)->m_space_id, - (*it)->m_format_flags, - (*it)->m_tablespace_flags, - (*it)->m_tablename, - **it); - } + err = fil_recreate_tablespace( + (*it)->m_space_id, + (*it)->m_format_flags, + (*it)->m_tablespace_flags, + (*it)->m_tablename, + **it, log_get_lsn()); /* Step-2: Update the SYS_XXXX tables to reflect new table-id and root_page_no. */ @@ -2630,7 +2558,7 @@ truncate_t::parse( ut_ad(!m_indexes.empty()); - if (fsp_flags_is_compressed(m_tablespace_flags)) { + if (FSP_FLAGS_GET_ZIP_SSIZE(m_tablespace_flags)) { /* Parse the number of index fields from TRUNCATE log record */ for (ulint i = 0; i < m_indexes.size(); ++i) { @@ -2929,12 +2857,12 @@ truncate_t::create_indexes( ++it) { btr_create_t btr_redo_create_info( - fsp_flags_is_compressed(flags) + FSP_FLAGS_GET_ZIP_SSIZE(flags) ? &it->m_fields[0] : NULL); btr_redo_create_info.format_flags = format_flags; - if (fsp_flags_is_compressed(flags)) { + if (FSP_FLAGS_GET_ZIP_SSIZE(flags)) { btr_redo_create_info.n_fields = it->m_n_fields; /* Skip the NUL appended field */ @@ -3069,7 +2997,7 @@ truncate_t::write( } /* If tablespace compressed then field info of each index. */ - if (fsp_flags_is_compressed(flags)) { + if (FSP_FLAGS_GET_ZIP_SSIZE(flags)) { for (ulint i = 0; i < m_indexes.size(); ++i) { diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index 4e515bab30d..048f75e413f 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -36,7 +36,6 @@ Created 12/27/1996 Heikki Tuuri #include "dict0mem.h" #include "trx0undo.h" #include "rem0rec.h" -#ifndef UNIV_HOTBACKUP #include "dict0boot.h" #include "dict0crea.h" #include "mach0data.h" @@ -58,7 +57,6 @@ Created 12/27/1996 Heikki Tuuri #include "fts0fts.h" #include "fts0types.h" #include <algorithm> - #include <mysql/plugin.h> #include <mysql/service_wsrep.h> @@ -127,7 +125,6 @@ row_upd_changes_first_fields_binary( const upd_t* update, /*!< in: update vector for the row */ ulint n); /*!< in: how many first fields to check */ - /*********************************************************************//** Checks if index currently is mentioned as a referenced index in a foreign key constraint. @@ -454,7 +451,6 @@ upd_node_create( return(node); } -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Updates the trx id and roll ptr field in a clustered index record in database @@ -488,7 +484,6 @@ row_upd_rec_sys_fields_in_recovery( } } -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Sets the trx id or roll ptr field of a clustered index entry. */ void @@ -633,7 +628,6 @@ row_upd_changes_disowned_external( return(false); } -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Replaces the new column values stored in the update vector to the @@ -689,7 +683,6 @@ row_upd_rec_in_place( } } -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Writes into the redo log the values of trx id and roll ptr and enough info to determine their positions within a clustered index record. @@ -718,7 +711,6 @@ row_upd_write_sys_vals_to_log( return(log_ptr); } -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Parses the log data of system field values. @@ -752,7 +744,6 @@ row_upd_parse_sys_vals( return(const_cast<byte*>(ptr)); } -#ifndef UNIV_HOTBACKUP /***********************************************************//** Writes to the redo log the new values of the fields occurring in the index. */ void @@ -830,7 +821,6 @@ row_upd_index_write_log( mlog_close(mtr, log_ptr); } -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Parses the log data written by row_upd_index_write_log. @@ -917,7 +907,6 @@ row_upd_index_parse( return(const_cast<byte*>(ptr)); } -#ifndef UNIV_HOTBACKUP /***************************************************************//** Builds an update vector from those fields which in a secondary index entry differ from a record that has the equal ordering fields. NOTE: we compare @@ -1032,9 +1021,8 @@ row_upd_build_difference_binary( n_diff = 0; trx_id_pos = dict_index_get_sys_col_pos(index, DATA_TRX_ID); - ut_ad(dict_table_is_intrinsic(index->table) - || (dict_index_get_sys_col_pos(index, DATA_ROLL_PTR) - == trx_id_pos + 1)); + ut_ad(dict_index_get_sys_col_pos(index, DATA_ROLL_PTR) + == trx_id_pos + 1); if (!offsets) { offsets = rec_get_offsets(rec, index, offsets_, @@ -1058,8 +1046,7 @@ row_upd_build_difference_binary( } /* DB_ROLL_PTR */ - if (i == trx_id_pos + 1 - && !dict_table_is_intrinsic(index->table)) { + if (i == trx_id_pos + 1) { continue; } } @@ -1078,7 +1065,6 @@ row_upd_build_difference_binary( } } -#ifdef MYSQL_VIRTUAL_COLUMNS /* Check the virtual columns updates. Even if there is no non-virtual column (base columns) change, we will still need to build the indexed virtual column value so that undo log would log them ( @@ -1143,7 +1129,6 @@ row_upd_build_difference_binary( mem_heap_free(v_heap); } } -#endif /* MYSQL_VIRTUAL_COLUMNS */ update->n_fields = n_diff; ut_ad(update->validate()); @@ -2063,7 +2048,6 @@ row_upd_eval_new_vals( } } -#ifdef MYSQL_VIRTUAL_COLUMNS /** Stores to the heap the virtual columns that need for any indexes @param[in,out] node row update node @param[in] update an update vector if it is update @@ -2142,7 +2126,6 @@ row_upd_store_v_row( mem_heap_free(heap); } } -#endif /* MYSQL_VIRTUAL_COLUMNS */ /** Stores to the heap the row on which the node->pcur is positioned. @param[in] node row update node @@ -2192,12 +2175,10 @@ row_upd_store_row( node->row = row_build(ROW_COPY_DATA, clust_index, rec, offsets, NULL, NULL, NULL, ext, node->heap); -#ifdef MYSQL_VIRTUAL_COLUMNS if (node->table->n_v_cols) { row_upd_store_v_row(node, node->is_delete ? NULL : node->update, thd, mysql_table); } -#endif /* MYSQL_VIRTUAL_COLUMNS */ if (node->is_delete) { node->upd_row = NULL; @@ -2232,7 +2213,6 @@ srv_mbr_print(const byte* data) << ", " << d << "\n"; } - /***********************************************************//** Updates a secondary index entry of a row. @return DB_SUCCESS if operation successfully completed, else error @@ -2255,7 +2235,7 @@ row_upd_sec_index_entry( dberr_t err = DB_SUCCESS; trx_t* trx = thr_get_trx(thr); ulint mode; - ulint flags = 0; + ulint flags; enum row_search_result search_result; ut_ad(trx->id != 0); @@ -2273,9 +2253,7 @@ row_upd_sec_index_entry( entry = row_build_index_entry(node->row, node->ext, index, heap); ut_a(entry); - if (!dict_table_is_intrinsic(index->table)) { - log_free_check(); - } + log_free_check(); DEBUG_SYNC_C_IF_THD(trx->mysql_thd, "before_row_upd_sec_index_entry"); @@ -2288,12 +2266,10 @@ row_upd_sec_index_entry( on restart for recovery. Disable locking as temp-tables are not shared across connection. */ if (dict_table_is_temporary(index->table)) { - flags |= BTR_NO_LOCKING_FLAG; + flags = BTR_NO_LOCKING_FLAG; mtr.set_log_mode(MTR_LOG_NO_REDO); - - if (dict_table_is_intrinsic(index->table)) { - flags |= BTR_NO_UNDO_LOG_FLAG; - } + } else { + flags = 0; } if (!index->is_committed()) { @@ -2400,7 +2376,8 @@ row_upd_sec_index_entry( << " of table " << index->table->name << " was not found on update: " << *entry << " at: " << rec_index_print(rec, index); - srv_mbr_print((unsigned char*)entry->fields[0].data); + if (entry->fields[0].data) + srv_mbr_print((unsigned char*)entry->fields[0].data); #ifdef UNIV_DEBUG mtr_commit(&mtr); mtr_start(&mtr); @@ -2411,7 +2388,6 @@ row_upd_sec_index_entry( case ROW_FOUND: ut_ad(err == DB_SUCCESS); - /* Delete mark the old index record; it can already be delete marked if we return after a lock wait in row_ins_sec_index_entry() below */ @@ -2877,10 +2853,6 @@ row_upd_clust_rec( if (dict_table_is_temporary(index->table)) { flags |= BTR_NO_LOCKING_FLAG; mtr->set_log_mode(MTR_LOG_NO_REDO); - - if (dict_table_is_intrinsic(index->table)) { - flags |= BTR_NO_UNDO_LOG_FLAG; - } } /* NOTE: this transaction has an s-lock or x-lock on the record and @@ -3058,7 +3030,7 @@ row_upd_clust_step( ulint offsets_[REC_OFFS_NORMAL_SIZE]; ulint* offsets; ibool referenced; - ulint flags = 0; + ulint flags; ibool foreign = FALSE; trx_t* trx = thr_get_trx(thr); @@ -3085,12 +3057,10 @@ row_upd_clust_step( on restart for recovery. Disable locking as temp-tables are not shared across connection. */ if (dict_table_is_temporary(index->table)) { - flags |= BTR_NO_LOCKING_FLAG; + flags = BTR_NO_LOCKING_FLAG; mtr.set_log_mode(MTR_LOG_NO_REDO); - - if (dict_table_is_intrinsic(index->table)) { - flags |= BTR_NO_UNDO_LOG_FLAG; - } + } else { + flags = 0; } /* If the restoration does not succeed, then the same @@ -3290,9 +3260,8 @@ row_upd( switch (node->state) { case UPD_NODE_UPDATE_CLUSTERED: case UPD_NODE_INSERT_CLUSTERED: - if (!dict_table_is_intrinsic(node->table)) { - log_free_check(); - } + log_free_check(); + err = row_upd_clust_step(node, thr); if (err != DB_SUCCESS) { @@ -3504,29 +3473,16 @@ void upd_node_t::dbug_trace() for (upd_cascade_t::const_iterator i = cascade_upd_nodes->begin(); i != cascade_upd_nodes->end(); ++i) { - - const upd_node_t* update_node = *i; - ib::info() << "cascade_upd_nodes: Cascade to table: " << - update_node->table->name; - /* JAN: TODO: MySQL 5.7 DBUG_LOG("upd_node_t", "cascade_upd_nodes: Cascade to table: " - << update_node->table->name); - */ + << (*i)->table->name); } for (upd_cascade_t::const_iterator j = new_upd_nodes->begin(); j != new_upd_nodes->end(); ++j) { - - const upd_node_t* update_node = *j; - ib::info() << "cascade_upd_nodes: Cascade to table: " << - update_node->table->name; - /* JAN: TODO: MySQL 5.7 DBUG_LOG("upd_node_t", "new_upd_nodes: Cascade to table: " - << update_node->table->name); - */ + << (*j)->table->name); } DBUG_VOID_RETURN; } #endif /* !DBUG_OFF */ -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/row/row0vers.cc b/storage/innobase/row/row0vers.cc index fd17683fb48..69e7a889de9 100644 --- a/storage/innobase/row/row0vers.cc +++ b/storage/innobase/row/row0vers.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -467,7 +468,6 @@ row_vers_non_vc_match( return(ret); } -#ifdef MYSQL_VIRTUAL_COLUMNS /** build virtual column value from current cluster index record data @param[in,out] row the cluster index row in dtuple form @param[in] clust_index clustered index @@ -841,7 +841,6 @@ row_vers_build_cur_vrow( ULINT_UNDEFINED, &heap); return(cur_vrow); } -#endif /* MYSQL_VIRTUAL_COLUMNS */ /*****************************************************************//** Finds out if a version of the record, where the version >= the current @@ -878,8 +877,8 @@ row_vers_old_has_index_entry( mem_heap_t* v_heap = NULL; const dtuple_t* cur_vrow = NULL; - ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX) - || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX)); + ut_ad(mtr_memo_contains_page_flagged(mtr, rec, MTR_MEMO_PAGE_X_FIX + | MTR_MEMO_PAGE_S_FIX)); ut_ad(!rw_lock_own(&(purge_sys->latch), RW_LOCK_S)); clust_index = dict_table_get_first_index(index->table); @@ -913,7 +912,6 @@ row_vers_old_has_index_entry( if (dict_index_has_virtual(index)) { -#ifdef MYSQL_VIRTUAL_COLUMNS #ifdef DBUG_OFF # define dbug_v_purge false @@ -963,7 +961,6 @@ row_vers_old_has_index_entry( } clust_offsets = rec_get_offsets(rec, clust_index, NULL, ULINT_UNDEFINED, &heap); -#endif /* MYSQL_VIRTUAL_COLUMNS */ } else { entry = row_build_index_entry( @@ -1001,7 +998,6 @@ row_vers_old_has_index_entry( } } } else if (dict_index_has_virtual(index)) { -#ifdef MYSQL_VIRTUAL_COLUMNS /* The current cluster index record could be deleted, but the previous version of it might not. We will need to get the virtual column data from undo record @@ -1009,7 +1005,6 @@ row_vers_old_has_index_entry( cur_vrow = row_vers_build_cur_vrow( also_curr, rec, clust_index, &clust_offsets, index, ientry, roll_ptr, trx_id, heap, v_heap, mtr); -#endif /* MYSQL_VIRTUAL_COLUMNS */ } version = rec; @@ -1140,8 +1135,8 @@ row_vers_build_for_consistent_read( dberr_t err; ut_ad(dict_index_is_clust(index)); - ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX) - || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX)); + ut_ad(mtr_memo_contains_page_flagged(mtr, rec, MTR_MEMO_PAGE_X_FIX + | MTR_MEMO_PAGE_S_FIX)); ut_ad(!rw_lock_own(&(purge_sys->latch), RW_LOCK_S)); ut_ad(rec_offs_validate(rec, index, *offsets)); @@ -1252,8 +1247,8 @@ row_vers_build_for_semi_consistent_read( trx_id_t rec_trx_id = 0; ut_ad(dict_index_is_clust(index)); - ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX) - || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX)); + ut_ad(mtr_memo_contains_page_flagged(mtr, rec, MTR_MEMO_PAGE_X_FIX + | MTR_MEMO_PAGE_S_FIX)); ut_ad(!rw_lock_own(&(purge_sys->latch), RW_LOCK_S)); ut_ad(rec_offs_validate(rec, index, *offsets)); diff --git a/storage/innobase/srv/srv0mon.cc b/storage/innobase/srv/srv0mon.cc index 6d213a2c761..b2722c2a9bf 100644 --- a/storage/innobase/srv/srv0mon.cc +++ b/storage/innobase/srv/srv0mon.cc @@ -25,7 +25,6 @@ Database monitor counter interfaces Created 12/9/2009 Jimmy Yang *******************************************************/ -#ifndef UNIV_HOTBACKUP #include "buf0buf.h" #include "dict0mem.h" #include "ibuf0ibuf.h" @@ -54,7 +53,6 @@ Created 12/9/2009 Jimmy Yang #define MONITOR_BUF_PAGE_WRITTEN(name, description, code) \ MONITOR_BUF_PAGE(name, description, code, "written", PAGE_WRITTEN) - /** This array defines basic static information of monitor counters, including each monitor's name, module it belongs to, a short description and its property/type and corresponding monitor_id. @@ -303,6 +301,12 @@ static monitor_info_t innodb_counter_info[] = MONITOR_EXISTING | MONITOR_DEFAULT_ON), MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES_READ}, + {"buffer_pages0_read", "buffer", + "Number of page 0 read (innodb_pages0_read)", + static_cast<monitor_type_t>( + MONITOR_EXISTING | MONITOR_DEFAULT_ON), + MONITOR_DEFAULT_START, MONITOR_OVLD_PAGES0_READ}, + {"buffer_index_sec_rec_cluster_reads", "buffer", "Number of secondary record reads triggered cluster read", static_cast<monitor_type_t>( @@ -985,41 +989,6 @@ static monitor_info_t innodb_counter_info[] = MONITOR_NONE, MONITOR_DEFAULT_START, MONITOR_OVLD_PAGE_COMPRESS_SAVED}, - {"compress_trim_sect512", "compression", - "Number of sect-512 TRIMed by page compression", - MONITOR_NONE, - MONITOR_DEFAULT_START, MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT512}, - - {"compress_trim_sect1024", "compression", - "Number of sect-1024 TRIMed by page compression", - MONITOR_NONE, - MONITOR_DEFAULT_START, MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT1024}, - - {"compress_trim_sect2048", "compression", - "Number of sect-2048 TRIMed by page compression", - MONITOR_NONE, - MONITOR_DEFAULT_START, MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT2048}, - - {"compress_trim_sect4096", "compression", - "Number of sect-4K TRIMed by page compression", - MONITOR_NONE, - MONITOR_DEFAULT_START, MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT4096}, - - {"compress_trim_sect8192", "compression", - "Number of sect-8K TRIMed by page compression", - MONITOR_NONE, - MONITOR_DEFAULT_START, MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT8192}, - - {"compress_trim_sect16384", "compression", - "Number of sect-16K TRIMed by page compression", - MONITOR_NONE, - MONITOR_DEFAULT_START, MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT16384}, - - {"compress_trim_sect32768", "compression", - "Number of sect-32K TRIMed by page compression", - MONITOR_NONE, - MONITOR_DEFAULT_START, MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT32768}, - {"compress_pages_page_compressed", "compression", "Number of pages compressed by page compression", MONITOR_NONE, @@ -1030,11 +999,6 @@ static monitor_info_t innodb_counter_info[] = MONITOR_NONE, MONITOR_DEFAULT_START, MONITOR_OVLD_PAGE_COMPRESSED_TRIM_OP}, - {"compress_page_compressed_trim_op_saved", "compression", - "Number of TRIM operation saved by page compression", - MONITOR_NONE, - MONITOR_DEFAULT_START, MONITOR_OVLD_PAGE_COMPRESSED_TRIM_OP_SAVED}, - {"compress_pages_page_decompressed", "compression", "Number of pages decompressed by page compression", MONITOR_NONE, @@ -1780,6 +1744,11 @@ srv_mon_process_existing_counter( value = stat.n_pages_read; break; + /* innodb_pages0_read */ + case MONITOR_OVLD_PAGES0_READ: + value = srv_stats.page0_read; + break; + /* Number of times secondary index lookup triggered cluster lookup */ case MONITOR_OVLD_INDEX_SEC_REC_CLUSTER_READS: value = srv_stats.n_sec_rec_cluster_reads; @@ -2064,36 +2033,12 @@ srv_mon_process_existing_counter( case MONITOR_OVLD_PAGE_COMPRESS_SAVED: value = srv_stats.page_compression_saved; break; - case MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT512: - value = srv_stats.page_compression_trim_sect512; - break; - case MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT1024: - value = srv_stats.page_compression_trim_sect1024; - break; - case MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT2048: - value = srv_stats.page_compression_trim_sect2048; - break; - case MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT4096: - value = srv_stats.page_compression_trim_sect4096; - break; - case MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT8192: - value = srv_stats.page_compression_trim_sect8192; - break; - case MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT16384: - value = srv_stats.page_compression_trim_sect16384; - break; - case MONITOR_OVLD_PAGE_COMPRESS_TRIM_SECT32768: - value = srv_stats.page_compression_trim_sect32768; - break; case MONITOR_OVLD_PAGES_PAGE_COMPRESSED: value = srv_stats.pages_page_compressed; break; case MONITOR_OVLD_PAGE_COMPRESSED_TRIM_OP: value = srv_stats.page_compressed_trim_op; break; - case MONITOR_OVLD_PAGE_COMPRESSED_TRIM_OP_SAVED: - value = srv_stats.page_compressed_trim_op_saved; - break; case MONITOR_OVLD_PAGES_PAGE_DECOMPRESSED: value = srv_stats.pages_page_decompressed; break; @@ -2239,4 +2184,3 @@ srv_mon_default_on(void) } } } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index e7ca007e73e..a884b5c1bad 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -3,7 +3,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -45,8 +45,6 @@ Created 10/8/1995 Heikki Tuuri // // #include "mysql/psi/mysql_stage.h" // #include "mysql/psi/psi.h" -// JAN: TODO: MySQL 5.7 missing header -//#include "sql_thd_internal_api.h" #include "ha_prototypes.h" @@ -55,7 +53,6 @@ Created 10/8/1995 Heikki Tuuri #include "buf0lru.h" #include "dict0boot.h" #include "dict0load.h" -#include "dict0stats_bg.h" #include "fsp0sysspace.h" #include "ibuf0ibuf.h" #include "lock0lock.h" @@ -92,17 +89,13 @@ UNIV_INTERN ulong srv_fatal_semaphore_wait_threshold = DEFAULT_SRV_FATAL_SEMAPH in microseconds, in order to reduce the lagging of the purge thread. */ ulint srv_dml_needed_delay = 0; -ibool srv_monitor_active = FALSE; -ibool srv_error_monitor_active = FALSE; - -ibool srv_buf_dump_thread_active = FALSE; - -bool srv_buf_resize_thread_active = false; +bool srv_monitor_active; +bool srv_error_monitor_active; +bool srv_buf_dump_thread_active; +bool srv_dict_stats_thread_active; +bool srv_buf_resize_thread_active; -UNIV_INTERN ibool srv_log_scrub_active = FALSE; -UNIV_INTERN my_bool srv_scrub_log = FALSE; - -ibool srv_dict_stats_thread_active = FALSE; +my_bool srv_scrub_log; const char* srv_main_thread_op_info = ""; @@ -191,9 +184,7 @@ my_bool srv_use_native_aio = TRUE; my_bool srv_numa_interleave = FALSE; /* If this flag is TRUE, then we will use fallocate(PUCH_HOLE) to the pages */ -UNIV_INTERN my_bool srv_use_trim = FALSE; -/* If this flag is TRUE, then we will use posix fallocate for file extentsion */ -UNIV_INTERN my_bool srv_use_posix_fallocate = FALSE; +UNIV_INTERN my_bool srv_use_trim; /* If this flag is TRUE, then we disable doublewrite buffer */ UNIV_INTERN my_bool srv_use_atomic_writes = FALSE; /* If this flag IS TRUE, then we use this algorithm for page compressing the pages */ @@ -204,8 +195,6 @@ UNIV_INTERN long srv_mtflush_threads = MTFLUSH_DEFAULT_WORKER; UNIV_INTERN my_bool srv_use_mtflush = FALSE; #ifdef UNIV_DEBUG -/** Force all user tables to use page compression. */ -ulong srv_debug_compress; /** Used by SET GLOBAL innodb_master_thread_disabled_debug = X. */ my_bool srv_master_thread_disabled_debug; /** Event used to inform that master thread is disabled. */ @@ -216,9 +205,9 @@ extern bool trx_commit_disallowed; #endif /* UNIV_DEBUG */ /*------------------------- LOG FILES ------------------------ */ -char* srv_log_group_home_dir = NULL; +char* srv_log_group_home_dir; -ulong srv_n_log_files = SRV_N_LOG_FILES_MAX; +ulong srv_n_log_files; /** At startup, this is the current redo log file size. During startup, if this is different from srv_log_file_size_requested (innodb_log_file_size), the redo log will be rebuilt and this size @@ -370,11 +359,6 @@ starting from SRV_FORCE_IGNORE_CORRUPT, so that data can be recovered by SELECT or mysqldump. When this is nonzero, we do not allow any user modifications to the data. */ ulong srv_force_recovery; -#ifndef DBUG_OFF -/** Inject a crash at different steps of the recovery process. -This is for testing and debugging only. */ -ulong srv_force_recovery_crash; -#endif /* !DBUG_OFF */ /** Print all user-level transactions deadlocks to mysqld stderr */ @@ -387,7 +371,7 @@ my_bool srv_cmp_per_index_enabled = FALSE; merge to completion before shutdown. If it is set to 2, do not even flush the buffer pool to data files at the shutdown: we effectively 'crash' InnoDB (but lose no committed transactions). */ -ulint srv_fast_shutdown = 0; +uint srv_fast_shutdown; /* Generate a innodb_status.<pid> file */ ibool srv_innodb_status = FALSE; @@ -427,7 +411,7 @@ ulong srv_replication_delay = 0; /*-------------------------------------------*/ UNIV_INTERN ulong srv_n_spin_wait_rounds = 15; -ulong srv_spin_wait_delay = 6; +uint srv_spin_wait_delay; ibool srv_priority_boost = TRUE; static ulint srv_n_rows_inserted_old = 0; @@ -647,7 +631,11 @@ struct srv_sys_t{ ulint n_sys_threads; /*!< size of the sys_threads array */ - srv_slot_t* sys_threads; /*!< server thread table */ + srv_slot_t* sys_threads; /*!< server thread table; + os_event_set() and + os_event_reset() on + sys_threads[]->event are + covered by srv_sys_t::mutex */ ulint n_threads_active[SRV_MASTER + 1]; /*!< number of threads active @@ -660,13 +648,16 @@ struct srv_sys_t{ static srv_sys_t* srv_sys = NULL; -/** Event to signal the monitor thread. */ +/** Event to signal srv_monitor_thread. Not protected by a mutex. +Set after setting srv_print_innodb_monitor. */ os_event_t srv_monitor_event; -/** Event to signal the error thread */ +/** Event to signal the shutdown of srv_error_monitor_thread. +Not protected by a mutex. */ os_event_t srv_error_event; -/** Event to signal the buffer pool dump/load thread */ +/** Event for waking up buf_dump_thread. Not protected by a mutex. +Set on shutdown or by buf_dump_start() or buf_load_start(). */ os_event_t srv_buf_dump_event; /** Event to signal the buffer pool resize thread */ @@ -870,7 +861,6 @@ srv_suspend_thread_low( /*===================*/ srv_slot_t* slot) /*!< in/out: thread slot */ { - ut_ad(!srv_read_only_mode); ut_ad(srv_sys_mutex_own()); @@ -928,33 +918,73 @@ srv_suspend_thread( return(sig_count); } -/*********************************************************************//** -Releases threads of the type given from suspension in the thread table. -NOTE! The server mutex has to be reserved by the caller! -@return number of threads released: this may be less than n if not - enough threads were suspended at the moment. */ -ulint -srv_release_threads( -/*================*/ - srv_thread_type type, /*!< in: thread type */ - ulint n) /*!< in: number of threads to release */ +/** Resume the calling thread. +@param[in,out] slot thread slot +@param[in] sig_count signal count (if wait) +@param[in] wait whether to wait for the event +@param[in] timeout_usec timeout in microseconds (0=infinite) +@return whether the wait timed out */ +static +bool +srv_resume_thread(srv_slot_t* slot, int64_t sig_count = 0, bool wait = true, + ulint timeout_usec = 0) { - ulint i; - ulint count = 0; + bool timeout; + + ut_ad(!srv_read_only_mode); + ut_ad(slot->in_use); + ut_ad(slot->suspended); + + if (!wait) { + timeout = false; + } else if (timeout_usec) { + timeout = OS_SYNC_TIME_EXCEEDED == os_event_wait_time_low( + slot->event, timeout_usec, sig_count); + } else { + timeout = false; + os_event_wait_low(slot->event, sig_count); + } + + srv_sys_mutex_enter(); + ut_ad(slot->in_use); + ut_ad(slot->suspended); + + slot->suspended = FALSE; + ++srv_sys->n_threads_active[slot->type]; + srv_sys_mutex_exit(); + return(timeout); +} + +/** Ensure that a given number of threads of the type given are running +(or are already terminated). +@param[in] type thread type +@param[in] n number of threads that have to run */ +void +srv_release_threads(enum srv_thread_type type, ulint n) +{ + ulint running; ut_ad(srv_thread_type_validate(type)); ut_ad(n > 0); - srv_sys_mutex_enter(); + do { + srv_sys_mutex_enter(); - for (i = 0; i < srv_sys->n_sys_threads; i++) { - srv_slot_t* slot; + running = 0; - slot = &srv_sys->sys_threads[i]; + for (ulint i = 0; i < srv_sys->n_sys_threads; i++) { + srv_slot_t* slot; - if (slot->in_use - && srv_slot_get_type(slot) == type - && slot->suspended) { + slot = &srv_sys->sys_threads[i]; + + if (!slot->in_use || srv_slot_get_type(slot) != type) { + continue; + } else if (!slot->suspended) { + if (++running >= n) { + break; + } + continue; + } switch (type) { case SRV_NONE: @@ -984,21 +1014,11 @@ srv_release_threads( break; } - slot->suspended = FALSE; - - ++srv_sys->n_threads_active[type]; - os_event_set(slot->event); - - if (++count == n) { - break; - } } - } - - srv_sys_mutex_exit(); - return(count); + srv_sys_mutex_exit(); + } while (running && running < n); } /*********************************************************************//** @@ -1011,11 +1031,8 @@ srv_free_slot( { srv_sys_mutex_enter(); - if (!slot->suspended) { - /* Mark the thread as inactive. */ - srv_suspend_thread_low(slot); - } - + /* Mark the thread as inactive. */ + srv_suspend_thread_low(slot); /* Free the slot for reuse. */ ut_ad(slot->in_use); slot->in_use = FALSE; @@ -1023,11 +1040,10 @@ srv_free_slot( srv_sys_mutex_exit(); } -/*********************************************************************//** -Initializes the server. */ +/** Initialize the server. */ +static void -srv_init(void) -/*==========*/ +srv_init() { ulint n_sys_threads = 0; ulint srv_sys_sz = sizeof(*srv_sys); @@ -1046,10 +1062,7 @@ srv_init(void) srv_sys->n_sys_threads = n_sys_threads; - /* Even in read-only mode we flush pages related to intrinsic table - and so mutex creation is needed. */ - { - + if (!srv_read_only_mode) { mutex_create(LATCH_ID_SRV_SYS, &srv_sys->mutex); mutex_create(LATCH_ID_SRV_SYS_TASKS, &srv_sys->tasks_mutex); @@ -1116,10 +1129,14 @@ void srv_free(void) /*==========*/ { + if (!srv_sys) { + return; + } + mutex_free(&srv_innodb_monitor_mutex); mutex_free(&page_zip_stat_per_index_mutex); - { + if (!srv_read_only_mode) { mutex_free(&srv_sys->mutex); mutex_free(&srv_sys->tasks_mutex); @@ -1142,6 +1159,8 @@ srv_free(void) srv_master_thread_disabled_event = NULL; #endif /* UNIV_DEBUG */ + dict_ind_free(); + trx_i_s_cache_free(trx_i_s_cache); ut_free(srv_sys); @@ -1150,22 +1169,6 @@ srv_free(void) } /*********************************************************************//** -Initializes the synchronization primitives, memory system, and the thread -local storage. */ -void -srv_general_init(void) -/*==================*/ -{ - sync_check_init(); - /* Reset the system variables in the recovery module. */ - recv_sys_var_init(); - os_thread_init(); - trx_pool_init(); - que_init(); - row_mysql_init(); -} - -/*********************************************************************//** Normalizes init parameter values to use units we use inside InnoDB. */ static void @@ -1194,10 +1197,12 @@ srv_boot(void) srv_normalize_init_values(); - /* Initialize synchronization primitives, memory management, and thread - local storage */ - - srv_general_init(); + sync_check_init(); + os_thread_init(); + /* Reset the system variables in the recovery module. */ + recv_sys_var_init(); + trx_pool_init(); + row_mysql_init(); /* Initialize this module */ @@ -1581,6 +1586,7 @@ srv_export_innodb_status(void) export_vars.innodb_pages_created = stat.n_pages_created; export_vars.innodb_pages_read = stat.n_pages_read; + export_vars.innodb_page0_read = srv_stats.page0_read; export_vars.innodb_pages_written = stat.n_pages_written; @@ -1630,13 +1636,10 @@ srv_export_innodb_status(void) export_vars.innodb_available_undo_logs = srv_available_undo_logs; export_vars.innodb_page_compression_saved = srv_stats.page_compression_saved; - export_vars.innodb_page_compression_trim_sect512 = srv_stats.page_compression_trim_sect512; - export_vars.innodb_page_compression_trim_sect4096 = srv_stats.page_compression_trim_sect4096; export_vars.innodb_index_pages_written = srv_stats.index_pages_written; export_vars.innodb_non_index_pages_written = srv_stats.non_index_pages_written; export_vars.innodb_pages_page_compressed = srv_stats.pages_page_compressed; export_vars.innodb_page_compressed_trim_op = srv_stats.page_compressed_trim_op; - export_vars.innodb_page_compressed_trim_op_saved = srv_stats.page_compressed_trim_op_saved; export_vars.innodb_pages_page_decompressed = srv_stats.pages_page_decompressed; export_vars.innodb_pages_page_compression_error = srv_stats.pages_page_compression_error; export_vars.innodb_pages_decrypted = srv_stats.pages_decrypted; @@ -1696,6 +1699,8 @@ srv_export_innodb_status(void) crypt_stat.pages_flushed; export_vars.innodb_encryption_rotation_estimated_iops = crypt_stat.estimated_iops; + export_vars.innodb_encryption_key_requests = + srv_stats.n_key_requests; export_vars.innodb_scrub_page_reorganizations = scrub_stat.page_reorganizations; @@ -1709,6 +1714,7 @@ srv_export_innodb_status(void) scrub_stat.page_split_failures_missing_index; export_vars.innodb_scrub_page_split_failures_unknown = scrub_stat.page_split_failures_unknown; + export_vars.innodb_scrub_log = srv_stats.n_log_scrubs; mutex_exit(&srv_innodb_monitor_mutex); } @@ -1718,11 +1724,7 @@ A thread which prints the info output by various InnoDB monitors. @return a dummy parameter */ extern "C" os_thread_ret_t -DECLARE_THREAD(srv_monitor_thread)( -/*===============================*/ - void* arg MY_ATTRIBUTE((unused))) - /*!< in: a dummy parameter required by - os_thread_create */ +DECLARE_THREAD(srv_monitor_thread)(void*) { int64_t sig_count; double time_elapsed; @@ -1743,9 +1745,7 @@ DECLARE_THREAD(srv_monitor_thread)( #ifdef UNIV_PFS_THREAD pfs_register_thread(srv_monitor_thread_key); #endif /* UNIV_PFS_THREAD */ - srv_monitor_active = TRUE; - UT_NOT_USED(arg); srv_last_monitor_time = ut_time(); last_table_monitor_time = ut_time(); last_tablespace_monitor_time = ut_time(); @@ -1878,7 +1878,7 @@ loop: goto loop; exit_func: - srv_monitor_active = FALSE; + srv_monitor_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ @@ -1894,11 +1894,7 @@ too long. These can be used to track bugs which cause hangs. @return a dummy parameter */ extern "C" os_thread_ret_t -DECLARE_THREAD(srv_error_monitor_thread)( -/*=====================================*/ - void* arg MY_ATTRIBUTE((unused))) - /*!< in: a dummy parameter required by - os_thread_create */ +DECLARE_THREAD(srv_error_monitor_thread)(void*) { /* number of successive fatal timeouts observed */ ulint fatal_cnt = 0; @@ -1924,7 +1920,6 @@ DECLARE_THREAD(srv_error_monitor_thread)( #ifdef UNIV_PFS_THREAD pfs_register_thread(srv_error_monitor_thread_key); #endif /* UNIV_PFS_THREAD */ - srv_error_monitor_active = TRUE; loop: /* Try to track a strange bug reported by Harald Fuchs and others, @@ -1956,9 +1951,7 @@ loop: if (sync_array_print_long_waits(&waiter, &sema) && sema == old_sema && os_thread_eq(waiter, old_waiter)) { #if defined(WITH_WSREP) && defined(WITH_INNODB_DISALLOW_WRITES) - if (true) { - // JAN: TODO: MySQL 5.7 - //if (srv_allow_writes_event->is_set) { + if (os_event_is_set(srv_allow_writes_event)) { #endif /* WITH_WSREP */ fatal_cnt++; #if defined(WITH_WSREP) && defined(WITH_INNODB_DISALLOW_WRITES) @@ -1997,7 +1990,7 @@ loop: goto loop; } - srv_error_monitor_active = FALSE; + srv_error_monitor_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ @@ -2042,62 +2035,22 @@ srv_get_active_thread_type(void) srv_sys_mutex_exit(); - /* Check only on shutdown. */ - if (ret == SRV_NONE - && srv_shutdown_state != SRV_SHUTDOWN_NONE - && trx_purge_state() != PURGE_STATE_DISABLED - && trx_purge_state() != PURGE_STATE_EXIT) { - - ret = SRV_PURGE; - } - - return(ret); -} - -/**********************************************************************//** -Check whether any background thread are active. If so print which thread -is active. Send the threads wakeup signal. -@return name of thread that is active or NULL */ -const char* -srv_any_background_threads_are_active(void) -/*=======================================*/ -{ - const char* thread_active = NULL; - - if (srv_read_only_mode) { - if (srv_buf_resize_thread_active) { - thread_active = "buf_resize_thread"; + if (ret == SRV_NONE && srv_shutdown_state != SRV_SHUTDOWN_NONE + && purge_sys != NULL) { + /* Check only on shutdown. */ + switch (trx_purge_state()) { + case PURGE_STATE_RUN: + case PURGE_STATE_STOP: + ret = SRV_PURGE; + break; + case PURGE_STATE_INIT: + case PURGE_STATE_DISABLED: + case PURGE_STATE_EXIT: + break; } - os_event_set(srv_buf_resize_event); - return(thread_active); - } else if (srv_error_monitor_active) { - thread_active = "srv_error_monitor_thread"; - } else if (lock_sys->timeout_thread_active) { - thread_active = "srv_lock_timeout thread"; - } else if (srv_monitor_active) { - thread_active = "srv_monitor_thread"; - } else if (srv_buf_dump_thread_active) { - thread_active = "buf_dump_thread"; - } else if (srv_buf_resize_thread_active) { - thread_active = "buf_resize_thread"; - } else if (srv_dict_stats_thread_active) { - thread_active = "dict_stats_thread"; - } else if (srv_scrub_log && srv_log_scrub_thread_active) { - thread_active = "log_scrub_thread"; } - os_event_set(srv_error_event); - os_event_set(srv_monitor_event); - os_event_set(srv_buf_dump_event); - os_event_set(lock_sys->timeout_event); - os_event_set(dict_stats_event); - os_event_set(srv_buf_resize_event); - - if (srv_scrub_log) { - os_event_set(log_scrub_event); - } - - return(thread_active); + return(ret); } /*******************************************************************//** @@ -2126,15 +2079,7 @@ srv_active_wake_master_thread_low() if (slot->in_use) { ut_a(srv_slot_get_type(slot) == SRV_MASTER); - - if (slot->suspended) { - - slot->suspended = FALSE; - - ++srv_sys->n_threads_active[SRV_MASTER]; - - os_event_set(slot->event); - } + os_event_set(slot->event); } srv_sys_mutex_exit(); @@ -2652,7 +2597,7 @@ suspend_thread: manual also mentions this string in several places. */ srv_main_thread_op_info = "waiting for server activity"; - os_event_wait(slot->event); + srv_resume_thread(slot); if (srv_shutdown_state != SRV_SHUTDOWN_EXIT_THREADS) { goto loop; @@ -2669,13 +2614,17 @@ Check if purge should stop. static bool srv_purge_should_exit( + MYSQL_THD thd, ulint n_purged) /*!< in: pages purged in last batch */ { switch (srv_shutdown_state) { case SRV_SHUTDOWN_NONE: - /* Normal operation. */ - break; - + if ((!srv_was_started || srv_running) + && !thd_kill_level(thd)) { + /* Normal operation. */ + break; + } + /* close_connections() was called; fall through */ case SRV_SHUTDOWN_CLEANUP: case SRV_SHUTDOWN_EXIT_THREADS: /* Exit unless slow shutdown requested or all done. */ @@ -2741,8 +2690,7 @@ DECLARE_THREAD(srv_worker_thread)( ut_ad(!srv_read_only_mode); ut_a(srv_force_recovery < SRV_FORCE_NO_BACKGROUND); my_thread_init(); - // JAN: TODO: MySQL 5.7 - // THD *thd= create_thd(false, true, true); + THD* thd = innobase_create_background_thd(); #ifdef UNIV_DEBUG_THREAD_CREATION ib::info() << "Worker thread starting, id " @@ -2760,13 +2708,12 @@ DECLARE_THREAD(srv_worker_thread)( srv_sys_mutex_exit(); /* We need to ensure that the worker threads exit after the - purge coordinator thread. Otherwise the purge coordinaor can + purge coordinator thread. Otherwise the purge coordinator can end up waiting forever in trx_purge_wait_for_workers_to_complete() */ do { srv_suspend_thread(slot); - - os_event_wait(slot->event); + srv_resume_thread(slot); if (srv_task_execute()) { @@ -2786,7 +2733,6 @@ DECLARE_THREAD(srv_worker_thread)( ut_a(!purge_sys->running); ut_a(purge_sys->state == PURGE_STATE_EXIT); - ut_a(srv_shutdown_state > SRV_SHUTDOWN_NONE); rw_lock_x_unlock(&purge_sys->latch); @@ -2795,9 +2741,8 @@ DECLARE_THREAD(srv_worker_thread)( << os_thread_pf(os_thread_get_curr_id()); #endif /* UNIV_DEBUG_THREAD_CREATION */ - // JAN: TODO: MySQL 5.7 - // destroy_thd(thd); - my_thread_end(); + innobase_destroy_background_thd(thd); + my_thread_end(); /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ os_thread_exit(); @@ -2812,6 +2757,7 @@ static ulint srv_do_purge( /*=========*/ + MYSQL_THD thd, ulint n_threads, /*!< in: number of threads to use */ ulint* n_total_purged) /*!< in/out: total pages purged */ { @@ -2882,7 +2828,7 @@ srv_do_purge( *n_total_purged += n_pages_purged; - } while (!srv_purge_should_exit(n_pages_purged) + } while (!srv_purge_should_exit(thd, n_pages_purged) && n_pages_purged > 0 && purge_sys->state == PURGE_STATE_RUN); @@ -2895,6 +2841,7 @@ static void srv_purge_coordinator_suspend( /*==========================*/ + MYSQL_THD thd, srv_slot_t* slot, /*!< in/out: Purge coordinator thread slot */ ulint rseg_history_len) /*!< in: history list length @@ -2911,8 +2858,6 @@ srv_purge_coordinator_suspend( int64_t sig_count = srv_suspend_thread(slot); do { - ulint ret; - rw_lock_x_lock(&purge_sys->latch); purge_sys->running = false; @@ -2921,32 +2866,11 @@ srv_purge_coordinator_suspend( /* We don't wait right away on the the non-timed wait because we want to signal the thread that wants to suspend purge. */ - - if (stop) { - os_event_wait_low(slot->event, sig_count); - ret = 0; - } else if (rseg_history_len <= trx_sys->rseg_history_len) { - ret = os_event_wait_time_low( - slot->event, SRV_PURGE_MAX_TIMEOUT, sig_count); - } else { - /* We don't want to waste time waiting, if the - history list increased by the time we got here, - unless purge has been stopped. */ - ret = 0; - } - - srv_sys_mutex_enter(); - - /* The thread can be in state !suspended after the timeout - but before this check if another thread sent a wakeup signal. */ - - if (slot->suspended) { - slot->suspended = FALSE; - ++srv_sys->n_threads_active[slot->type]; - ut_a(srv_sys->n_threads_active[slot->type] == 1); - } - - srv_sys_mutex_exit(); + const bool wait = stop + || rseg_history_len <= trx_sys->rseg_history_len; + const bool timeout = srv_resume_thread( + slot, sig_count, wait, + stop ? 0 : SRV_PURGE_MAX_TIMEOUT); sig_count = srv_suspend_thread(slot); @@ -2958,6 +2882,19 @@ srv_purge_coordinator_suspend( if (!stop) { ut_a(purge_sys->n_stop == 0); purge_sys->running = true; + + if (timeout + && rseg_history_len == trx_sys->rseg_history_len + && trx_sys->rseg_history_len < 5000) { + /* No new records were added since the + wait started. Simply wait for new + records. The magic number 5000 is an + approximation for the case where we + have cached UNDO log records which + prevent truncate of the UNDO + segments. */ + stop = true; + } } else { ut_a(purge_sys->n_stop > 0); @@ -2966,33 +2903,9 @@ srv_purge_coordinator_suspend( } rw_lock_x_unlock(&purge_sys->latch); + } while (stop && !thd_kill_level(thd)); - if (ret == OS_SYNC_TIME_EXCEEDED) { - - /* No new records added since wait started then simply - wait for new records. The magic number 5000 is an - approximation for the case where we have cached UNDO - log records which prevent truncate of the UNDO - segments. */ - - if (rseg_history_len == trx_sys->rseg_history_len - && trx_sys->rseg_history_len < 5000) { - - stop = true; - } - } - - } while (stop); - - srv_sys_mutex_enter(); - - if (slot->suspended) { - slot->suspended = FALSE; - ++srv_sys->n_threads_active[slot->type]; - ut_a(srv_sys->n_threads_active[slot->type] == 1); - } - - srv_sys_mutex_exit(); + srv_resume_thread(slot, 0, false); } /*********************************************************************//** @@ -3006,8 +2919,7 @@ DECLARE_THREAD(srv_purge_coordinator_thread)( required by os_thread_create */ { my_thread_init(); - // JAN: TODO: MySQL 5.7 - // THD *thd= create_thd(false, true, true); + THD* thd = innobase_create_background_thd(); srv_slot_t* slot; ulint n_total_purged = ULINT_UNDEFINED; @@ -3041,28 +2953,30 @@ DECLARE_THREAD(srv_purge_coordinator_thread)( purge didn't purge any records then wait for activity. */ if (srv_shutdown_state == SRV_SHUTDOWN_NONE + && !thd_kill_level(thd) && (purge_sys->state == PURGE_STATE_STOP || n_total_purged == 0)) { - srv_purge_coordinator_suspend(slot, rseg_history_len); + srv_purge_coordinator_suspend(thd, slot, rseg_history_len); } - if (srv_purge_should_exit(n_total_purged)) { - ut_a(!slot->suspended); + ut_ad(!slot->suspended); + + if (srv_purge_should_exit(thd, n_total_purged)) { break; } n_total_purged = 0; rseg_history_len = srv_do_purge( - srv_n_purge_threads, &n_total_purged); + thd, srv_n_purge_threads, &n_total_purged); - } while (!srv_purge_should_exit(n_total_purged)); + } while (!srv_purge_should_exit(thd, n_total_purged)); /* Ensure that we don't jump out of the loop unless the exit condition is satisfied. */ - ut_a(srv_purge_should_exit(n_total_purged)); + ut_a(srv_purge_should_exit(thd, n_total_purged)); ulint n_pages_purged = ULINT_MAX; @@ -3121,8 +3035,7 @@ DECLARE_THREAD(srv_purge_coordinator_thread)( srv_release_threads(SRV_WORKER, srv_n_purge_threads - 1); } - // JAN: TODO: MYSQL 5.7 - // destroy_thd(thd); + innobase_destroy_background_thd(thd); my_thread_end(); /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ @@ -3169,11 +3082,9 @@ srv_get_task_queue_length(void) return(n_tasks); } -/**********************************************************************//** -Wakeup the purge threads. */ +/** Wake up the purge threads. */ void -srv_purge_wakeup(void) -/*==================*/ +srv_purge_wakeup() { ut_ad(!srv_read_only_mode); @@ -3220,27 +3131,6 @@ srv_was_tablespace_truncated(const fil_space_t* space) return(false); } - bool has_shared_space = FSP_FLAGS_GET_SHARED(space->flags); - - if (is_system_tablespace(space->id) || has_shared_space) { - return(false); - } - - return(truncate_t::was_tablespace_truncated(space->id)); -} - -/** Call exit(3) */ -void -srv_fatal_error() -{ - - ib::error() << "Cannot continue operation."; - - fflush(stderr); - - ut_d(innodb_calling_exit = true); - - srv_shutdown_all_bg_threads(); - - exit(3); + return (!is_system_tablespace(space->id) + && truncate_t::was_tablespace_truncated(space->id)); } diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 4be92906851..8afa90502d0 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -3,7 +3,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2016, MariaDB Corporation +Copyright (c) 2013, 2017, MariaDB Corporation Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -63,7 +63,7 @@ Created 2/16/1996 Heikki Tuuri #include "fsp0fsp.h" #include "rem0rec.h" #include "mtr0mtr.h" -#include "log0log.h" +#include "log0crypt.h" #include "log0recv.h" #include "page0page.h" #include "page0cur.h" @@ -76,42 +76,39 @@ Created 2/16/1996 Heikki Tuuri #include "srv0start.h" #include "srv0srv.h" #include "btr0defragment.h" - #include "fsp0sysspace.h" #include "row0trunc.h" #include <mysql/service_wsrep.h> - -#ifndef UNIV_HOTBACKUP -# include "trx0rseg.h" -# include "os0proc.h" -# include "buf0flu.h" -# include "buf0rea.h" -# include "buf0mtflu.h" -# include "dict0boot.h" -# include "dict0load.h" -# include "dict0stats_bg.h" -# include "que0que.h" -# include "usr0sess.h" -# include "lock0lock.h" -# include "trx0roll.h" -# include "trx0purge.h" -# include "lock0lock.h" -# include "pars0pars.h" -# include "btr0sea.h" -# include "rem0cmp.h" -# include "dict0crea.h" -# include "row0ins.h" -# include "row0sel.h" -# include "row0upd.h" -# include "row0row.h" -# include "row0mysql.h" -# include "row0trunc.h" -# include "btr0pcur.h" -# include "os0event.h" -# include "zlib.h" -# include "ut0crc32.h" -# include "btr0scrub.h" -# include "ut0new.h" +#include "trx0rseg.h" +#include "os0proc.h" +#include "buf0flu.h" +#include "buf0rea.h" +#include "buf0mtflu.h" +#include "dict0boot.h" +#include "dict0load.h" +#include "dict0stats_bg.h" +#include "que0que.h" +#include "usr0sess.h" +#include "lock0lock.h" +#include "trx0roll.h" +#include "trx0purge.h" +#include "lock0lock.h" +#include "pars0pars.h" +#include "btr0sea.h" +#include "rem0cmp.h" +#include "dict0crea.h" +#include "row0ins.h" +#include "row0sel.h" +#include "row0upd.h" +#include "row0row.h" +#include "row0mysql.h" +#include "row0trunc.h" +#include "btr0pcur.h" +#include "os0event.h" +#include "zlib.h" +#include "ut0crc32.h" +#include "btr0scrub.h" +#include "ut0new.h" #ifdef HAVE_LZO1X #include <lzo/lzo1x.h> @@ -140,6 +137,10 @@ bool srv_sys_tablespaces_open = false; ibool srv_was_started = FALSE; /** TRUE if innobase_start_or_create_for_mysql() has been called */ static ibool srv_start_has_been_called = FALSE; +#ifdef UNIV_DEBUG +/** InnoDB system tablespace to set during recovery */ +UNIV_INTERN uint srv_sys_space_size_debug; +#endif /* UNIV_DEBUG */ /** Bit flags for tracking background thread creation. They are used to determine which threads need to be stopped if we need to abort during @@ -186,7 +187,6 @@ static bool dict_stats_thread_started = false; static bool buf_flush_page_cleaner_thread_started = false; /** Name of srv_monitor_file */ static char* srv_monitor_file_name; -#endif /* !UNIV_HOTBACKUP */ /** Minimum expected tablespace size. (10M) */ static const ulint MIN_EXPECTED_TABLESPACE_SIZE = 5 * 1024 * 1024; @@ -194,9 +194,6 @@ static const ulint MIN_EXPECTED_TABLESPACE_SIZE = 5 * 1024 * 1024; /** */ #define SRV_MAX_N_PENDING_SYNC_IOS 100 -/** The round off to MB is similar as done in srv_parse_megabytes() */ -#define CALC_NUMBER_OF_PAGES(size) ((size) / (1024 * 1024)) * \ - ((1024 * 1024) / (UNIV_PAGE_SIZE)) #ifdef UNIV_PFS_THREAD /* Keys to register InnoDB threads with performance schema */ mysql_pfs_key_t buf_dump_thread_key; @@ -281,7 +278,6 @@ srv_file_check_mode( return(true); } -#ifndef UNIV_HOTBACKUP /********************************************************************//** I/o-handler thread function. @return OS_THREAD_DUMMY_RETURN */ @@ -340,9 +336,7 @@ DECLARE_THREAD(io_handler_thread)( OS_THREAD_DUMMY_RETURN; } -#endif /* !UNIV_HOTBACKUP */ -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Creates a log file. @return DB_SUCCESS or error code */ @@ -424,12 +418,14 @@ create_log_files( file should be recoverable. The buffer pool was clean, and we can simply create all log files from the scratch. */ - RECOVERY_CRASH(6); + DBUG_EXECUTE_IF("innodb_log_abort_6", return(DB_ERROR);); } + DBUG_PRINT("ib_log", ("After innodb_log_abort_6")); ut_ad(!buf_pool_check_no_pending_io()); - RECOVERY_CRASH(7); + DBUG_EXECUTE_IF("innodb_log_abort_7", return(DB_ERROR);); + DBUG_PRINT("ib_log", ("After innodb_log_abort_7")); for (unsigned i = 0; i < srv_n_log_files; i++) { sprintf(logfilename + dirnamelen, @@ -442,21 +438,18 @@ create_log_files( } } - RECOVERY_CRASH(8); + DBUG_EXECUTE_IF("innodb_log_abort_8", return(DB_ERROR);); + DBUG_PRINT("ib_log", ("After innodb_log_abort_8")); /* We did not create the first log file initially as ib_logfile0, so that crash recovery cannot find it until it has been completed and renamed. */ sprintf(logfilename + dirnamelen, "ib_logfile%u", INIT_LOG_FILE0); - /* Disable the doublewrite buffer for log files, not required */ - fil_space_t* log_space = fil_space_create( - "innodb_redo_log", SRV_LOG_SPACE_FIRST_ID, - fsp_flags_set_page_size(0, univ_page_size), - FIL_TYPE_LOG, - NULL /* No encryption yet */ - ); + "innodb_redo_log", SRV_LOG_SPACE_FIRST_ID, 0, FIL_TYPE_LOG, + NULL, /* innodb_encrypt_log works at a different level */ + true /* this is create */); ut_a(fil_validate()); ut_a(log_space != NULL); @@ -491,6 +484,9 @@ create_log_files( /* Create a log checkpoint. */ log_mutex_enter(); + if (log_sys->is_encrypted() && !log_crypt_init()) { + return(DB_ERROR); + } ut_d(recv_no_log_write = false); recv_reset_logs(lsn); log_mutex_exit(); @@ -498,10 +494,16 @@ create_log_files( return(DB_SUCCESS); } -/*********************************************************************//** -Renames the first log file. */ +/** Rename the first redo log file. +@param[in,out] logfilename buffer for the log file name +@param[in] dirnamelen length of the directory path +@param[in] lsn FIL_PAGE_FILE_FLUSH_LSN value +@param[in,out] logfile0 name of the first log file +@return error code +@retval DB_SUCCESS on successful operation */ +MY_ATTRIBUTE((warn_unused_result, nonnull)) static -void +dberr_t create_log_files_rename( /*====================*/ char* logfilename, /*!< in/out: buffer for log file name */ @@ -512,6 +514,10 @@ create_log_files_rename( /* If innodb_flush_method=O_DSYNC, we need to explicitly flush the log buffers. */ fil_flush(SRV_LOG_SPACE_FIRST_ID); + + DBUG_EXECUTE_IF("innodb_log_abort_9", return(DB_ERROR);); + DBUG_PRINT("ib_log", ("After innodb_log_abort_9")); + /* Close the log files, so that we can rename the first one. */ fil_close_log_files(false); @@ -520,26 +526,26 @@ create_log_files_rename( checkpoint has been created. */ sprintf(logfilename + dirnamelen, "ib_logfile%u", 0); - RECOVERY_CRASH(9); - ib::info() << "Renaming log file " << logfile0 << " to " << logfilename; log_mutex_enter(); ut_ad(strlen(logfile0) == 2 + strlen(logfilename)); - bool success = os_file_rename( - innodb_log_file_key, logfile0, logfilename); - ut_a(success); - - RECOVERY_CRASH(10); + dberr_t err = os_file_rename( + innodb_log_file_key, logfile0, logfilename) + ? DB_SUCCESS : DB_ERROR; /* Replace the first file with ib_logfile0. */ strcpy(logfile0, logfilename); log_mutex_exit(); + DBUG_EXECUTE_IF("innodb_log_abort_10", err = DB_ERROR;); + fil_open_log_and_system_tablespace_files(); - ib::warn() << "New log files created, LSN=" << lsn; + ib::info() << "New log files created, LSN=" << lsn; + + return(err); } /*********************************************************************//** @@ -651,7 +657,6 @@ srv_undo_tablespace_open( { os_file_t fh; bool ret; - ulint flags; dberr_t err = DB_ERROR; char undo_name[sizeof "innodb_undo000"]; @@ -681,18 +686,6 @@ srv_undo_tablespace_open( os_offset_t size; fil_space_t* space; - bool atomic_write; - -#if !defined(NO_FALLOCATE) && defined(UNIV_LINUX) - if (!srv_use_doublewrite_buf) { - atomic_write = fil_fusionio_enable_atomic_write(fh); - } else { - atomic_write = false; - } -#else - atomic_write = false; -#endif /* !NO_FALLOCATE && UNIV_LINUX */ - size = os_file_get_size(fh); ut_a(size != (os_offset_t) -1); @@ -708,11 +701,9 @@ srv_undo_tablespace_open( fil_set_max_space_id_if_bigger(space_id); - /* Set the compressed page size to 0 (non-compressed) */ - flags = fsp_flags_init( - univ_page_size, false, false, false, false, false, 0, ATOMIC_WRITES_DEFAULT); space = fil_space_create( - undo_name, space_id, flags, FIL_TYPE_TABLESPACE, NULL); + undo_name, space_id, FSP_FLAGS_PAGE_SSIZE(), + FIL_TYPE_TABLESPACE, NULL, true); ut_a(fil_validate()); ut_a(space); @@ -724,7 +715,7 @@ srv_undo_tablespace_open( the unit has been scaled to pages and page number is always 32 bits. */ if (fil_node_create( - name, (ulint) n_pages, space, false, atomic_write)) { + name, (ulint) n_pages, space, false, TRUE)) { err = DB_SUCCESS; } @@ -1123,76 +1114,65 @@ srv_start_wait_for_purge_to_start() /** Create the temporary file tablespace. @param[in] create_new_db whether we are creating a new database -@param[in,out] tmp_space Shared Temporary SysTablespace @return DB_SUCCESS or error code. */ static dberr_t -srv_open_tmp_tablespace( - bool create_new_db, - SysTablespace* tmp_space) +srv_open_tmp_tablespace(bool create_new_db) { ulint sum_of_new_sizes; /* Will try to remove if there is existing file left-over by last unclean shutdown */ - tmp_space->set_sanity_check_status(true); - tmp_space->delete_files(); - tmp_space->set_ignore_read_only(true); + srv_tmp_space.set_sanity_check_status(true); + srv_tmp_space.delete_files(); + srv_tmp_space.set_ignore_read_only(true); ib::info() << "Creating shared tablespace for temporary tables"; bool create_new_temp_space; - ulint temp_space_id = ULINT_UNDEFINED; - - dict_hdr_get_new_id(NULL, NULL, &temp_space_id, NULL, true); - - tmp_space->set_space_id(temp_space_id); - RECOVERY_CRASH(100); + srv_tmp_space.set_space_id(SRV_TMP_SPACE_ID); - dberr_t err = tmp_space->check_file_spec( - &create_new_temp_space, 12 * 1024 * 1024); + dberr_t err = srv_tmp_space.check_file_spec( + &create_new_temp_space, 12 * 1024 * 1024); if (err == DB_FAIL) { - ib::error() << "The " << tmp_space->name() + ib::error() << "The " << srv_tmp_space.name() << " data file must be writable!"; err = DB_ERROR; } else if (err != DB_SUCCESS) { ib::error() << "Could not create the shared " - << tmp_space->name() << "."; + << srv_tmp_space.name() << "."; - } else if ((err = tmp_space->open_or_create( + } else if ((err = srv_tmp_space.open_or_create( true, create_new_db, &sum_of_new_sizes, NULL)) != DB_SUCCESS) { ib::error() << "Unable to create the shared " - << tmp_space->name(); + << srv_tmp_space.name(); } else { mtr_t mtr; - ulint size = tmp_space->get_sum_of_sizes(); - - ut_a(temp_space_id != ULINT_UNDEFINED); - ut_a(tmp_space->space_id() == temp_space_id); + ulint size = srv_tmp_space.get_sum_of_sizes(); /* Open this shared temp tablespace in the fil_system so that it stays open until shutdown. */ - if (fil_space_open(tmp_space->name())) { + if (fil_space_open(srv_tmp_space.name())) { /* Initialize the header page */ mtr_start(&mtr); mtr_set_log_mode(&mtr, MTR_LOG_NO_REDO); - fsp_header_init(tmp_space->space_id(), size, &mtr); + fsp_header_init(SRV_TMP_SPACE_ID, size, &mtr); mtr_commit(&mtr); } else { /* This file was just opened in the code above! */ - ib::error() << "The " << tmp_space->name() + ib::error() << "The " << srv_tmp_space.name() << " data file cannot be re-opened" " after check_file_spec() succeeded!"; @@ -1229,11 +1209,10 @@ srv_start_state_is_set( /** Shutdown all background threads created by InnoDB. */ +static void srv_shutdown_all_bg_threads() { - ulint i; - srv_shutdown_state = SRV_SHUTDOWN_EXIT_THREADS; if (!srv_start_state) { @@ -1243,17 +1222,16 @@ srv_shutdown_all_bg_threads() /* All threads end up waiting for certain events. Put those events to the signaled state. Then the threads will exit themselves after os_event_wait(). */ - for (i = 0; i < 1000; i++) { + for (uint i = 0; i < 1000; ++i) { /* NOTE: IF YOU CREATE THREADS IN INNODB, YOU MUST EXIT THEM HERE OR EARLIER */ - if (!srv_read_only_mode) { - - if (srv_start_state_is_set(SRV_START_STATE_LOCK_SYS)) { - /* a. Let the lock timeout thread exit */ - os_event_set(lock_sys->timeout_event); - } + if (srv_start_state_is_set(SRV_START_STATE_LOCK_SYS)) { + /* a. Let the lock timeout thread exit */ + os_event_set(lock_sys->timeout_event); + } + if (!srv_read_only_mode) { /* b. srv error monitor thread exits automatically, no need to do anything here */ @@ -1267,56 +1245,57 @@ srv_shutdown_all_bg_threads() /* d. Wakeup purge threads. */ srv_purge_wakeup(); } + + if (srv_n_fil_crypt_threads_started) { + os_event_set(fil_crypt_threads_event); + } + + if (log_scrub_thread_active) { + os_event_set(log_scrub_event); + } } if (srv_start_state_is_set(SRV_START_STATE_IO)) { + ut_ad(!srv_read_only_mode); + /* e. Exit the i/o threads */ - if (!srv_read_only_mode) { - if (recv_sys->flush_start != NULL) { - os_event_set(recv_sys->flush_start); - } - if (recv_sys->flush_end != NULL) { - os_event_set(recv_sys->flush_end); - } + if (recv_sys->flush_start != NULL) { + os_event_set(recv_sys->flush_start); + } + if (recv_sys->flush_end != NULL) { + os_event_set(recv_sys->flush_end); } os_event_set(buf_flush_event); - if (!buf_page_cleaner_is_active - && os_aio_all_slots_free()) { - os_aio_wake_all_threads_at_shutdown(); + /* f. dict_stats_thread is signaled from + logs_empty_and_mark_files_at_shutdown() and + should have already quit or is quitting right + now. */ + + if (srv_use_mtflush) { + /* g. Exit the multi threaded flush threads */ + buf_mtflu_io_thread_exit(); } } - /* f. dict_stats_thread is signaled from - logs_empty_and_mark_files_at_shutdown() and should have - already quit or is quitting right now. */ - - if (srv_use_mtflush) { - /* g. Exit the multi threaded flush threads */ - buf_mtflu_io_thread_exit(); + if (!buf_page_cleaner_is_active && os_aio_all_slots_free()) { + os_aio_wake_all_threads_at_shutdown(); } - bool active = os_thread_active(); + const bool active = os_thread_active(); os_thread_sleep(100000); if (!active) { - break; + return; } } - if (i == 1000) { - ib::warn() << os_thread_count << " threads created by InnoDB" - " had not exited at shutdown!"; -#ifdef UNIV_DEBUG - os_aio_print_pending_io(stderr); - ut_ad(0); -#endif /* UNIV_DEBUG */ - } else { - /* Reset the start state. */ - srv_start_state = SRV_START_STATE_NONE; - } + ib::warn() << os_thread_count << " threads created by InnoDB" + " had not exited at shutdown!"; + ut_d(os_aio_print_pending_io(stderr)); + ut_ad(0); } #ifdef UNIV_DEBUG @@ -1333,6 +1312,7 @@ srv_shutdown_all_bg_threads() @param[in] line Line number @param[in] err Reason for aborting InnoDB startup @return DB_SUCCESS or error code. */ +MY_ATTRIBUTE((warn_unused_result, nonnull)) static dberr_t srv_init_abort_low( @@ -1372,6 +1352,8 @@ lsn_t srv_prepare_to_delete_redo_log_files( ulint n_files) { + DBUG_ENTER("srv_prepare_to_delete_redo_log_files"); + lsn_t flushed_lsn; ulint pending_io = 0; ulint count = 0; @@ -1380,7 +1362,8 @@ srv_prepare_to_delete_redo_log_files( /* Clean the buffer pool. */ buf_flush_sync_all_buf_pools(); - RECOVERY_CRASH(1); + DBUG_EXECUTE_IF("innodb_log_abort_1", DBUG_RETURN(0);); + DBUG_PRINT("ib_log", ("After innodb_log_abort_1")); log_mutex_enter(); @@ -1389,17 +1372,35 @@ srv_prepare_to_delete_redo_log_files( flushed_lsn = log_sys->lsn; { - ib::warn warning; + ib::info info; if (srv_log_file_size == 0) { - warning << "Upgrading redo log: "; + info << "Upgrading redo log: "; + } else if (n_files != srv_n_log_files + || srv_log_file_size + != srv_log_file_size_requested) { + if (srv_encrypt_log + == log_sys->is_encrypted()) { + info << (srv_encrypt_log + ? "Resizing encrypted" + : "Resizing"); + } else if (srv_encrypt_log) { + info << "Encrypting and resizing"; + } else { + info << "Removing encryption" + " and resizing"; + } + + info << " redo log from " << n_files + << "*" << srv_log_file_size << " to "; + } else if (srv_encrypt_log) { + info << "Encrypting redo log: "; } else { - warning << "Resizing redo log from " - << n_files << "*" - << srv_log_file_size << " to "; + info << "Removing redo log encryption: "; } - warning << srv_n_log_files << "*" - << srv_log_file_size_requested - << " pages, LSN=" << flushed_lsn; + + info << srv_n_log_files << "*" + << srv_log_file_size_requested + << " pages; LSN=" << flushed_lsn; } /* Flush the old log files. */ @@ -1432,7 +1433,7 @@ srv_prepare_to_delete_redo_log_files( } while (buf_pool_check_no_pending_io()); - return(flushed_lsn); + DBUG_RETURN(flushed_lsn); } /******************************************************************** @@ -1445,9 +1446,7 @@ innobase_start_or_create_for_mysql(void) { bool create_new_db = false; lsn_t flushed_lsn; - ulint sum_of_data_file_sizes; - ulint tablespace_size_in_header; - dberr_t err; + dberr_t err = DB_SUCCESS; ulint srv_n_log_files_found = srv_n_log_files; mtr_t mtr; purge_pq_t* purge_queue; @@ -1456,17 +1455,18 @@ innobase_start_or_create_for_mysql(void) size_t dirnamelen; unsigned i = 0; - /* Reset the start state. */ - srv_start_state = SRV_START_STATE_NONE; - high_level_read_only = srv_read_only_mode || srv_force_recovery > SRV_FORCE_NO_TRX_UNDO; + /* Reset the start state. */ + srv_start_state = SRV_START_STATE_NONE; + if (srv_read_only_mode) { ib::info() << "Started in read only mode"; - /* There is no write except to intrinsic table and so turn-off - doublewrite mechanism completely. */ + /* There is no write to InnoDB tablespaces (not even + temporary ones, because also CREATE TEMPORARY TABLE is + refused in read-only mode). */ srv_use_doublewrite_buf = FALSE; } @@ -1480,21 +1480,7 @@ innobase_start_or_create_for_mysql(void) } #endif /* HAVE_LZO1X */ -#ifdef UNIV_LINUX -# ifdef HAVE_FALLOC_PUNCH_HOLE_AND_KEEP_SIZE - ib::info() << "PUNCH HOLE support available"; -# else - ib::info() << "PUNCH HOLE support not available"; -# endif /* HAVE_FALLOC_PUNCH_HOLE_AND_KEEP_SIZE */ -#endif /* UNIV_LINUX */ - - if (sizeof(ulint) != sizeof(void*)) { - ib::error() << "Size of InnoDB's ulint is " << sizeof(ulint) - << ", but size of void* is " << sizeof(void*) - << ". The sizes should be the same so that on" - " a 64-bit platforms you can allocate more than 4 GB" - " of memory."; - } + compile_time_assert(sizeof(ulint) == sizeof(void*)); #ifdef UNIV_DEBUG ib::info() << "!!!!!!!! UNIV_DEBUG switched on !!!!!!!!!"; @@ -1607,7 +1593,7 @@ innobase_start_or_create_for_mysql(void) ib::error() << "Unrecognized value " << srv_file_flush_method_str << " for innodb_flush_method"; - return(srv_init_abort(DB_ERROR)); + err = DB_ERROR; } /* Note that the call srv_boot() also changes the values of @@ -1702,14 +1688,12 @@ innobase_start_or_create_for_mysql(void) srv_boot(); - if (ut_crc32_sse2_enabled) { - ib::info() << "Using SSE crc32 instructions"; - } else if (ut_crc32_power8_enabled) { - ib::info() << "Using POWER8 crc32 instructions"; - } else { - ib::info() << "Using generic crc32 instructions"; + if (err != DB_SUCCESS) { + return(srv_init_abort(err)); } + ib::info() << ut_crc32_implementation; + if (!srv_read_only_mode) { mutex_create(LATCH_ID_SRV_MONITOR_FILE, @@ -1856,26 +1840,26 @@ innobase_start_or_create_for_mysql(void) thread_started[t] = true; } - /* Even in read-only mode there could be flush job generated by - intrinsic table operations. */ - buf_flush_page_cleaner_init(); + if (!srv_read_only_mode) { + buf_flush_page_cleaner_init(); - os_thread_create(buf_flush_page_cleaner_coordinator, - NULL, NULL); + os_thread_create(buf_flush_page_cleaner_coordinator, + NULL, NULL); - buf_flush_page_cleaner_thread_started = true; + buf_flush_page_cleaner_thread_started = true; - for (i = 1; i < srv_n_page_cleaners; ++i) { - os_thread_create(buf_flush_page_cleaner_worker, - NULL, NULL); - } + for (i = 1; i < srv_n_page_cleaners; ++i) { + os_thread_create(buf_flush_page_cleaner_worker, + NULL, NULL); + } - /* Make sure page cleaner is active. */ - while (!buf_page_cleaner_is_active) { - os_thread_sleep(10000); - } + /* Make sure page cleaner is active. */ + while (!buf_page_cleaner_is_active) { + os_thread_sleep(10000); + } - srv_start_state_set(SRV_START_STATE_IO); + srv_start_state_set(SRV_START_STATE_IO); + } if (srv_n_log_files * srv_log_file_size * UNIV_PAGE_SIZE >= 512ULL * 1024ULL * 1024ULL * 1024ULL) { @@ -2006,23 +1990,21 @@ innobase_start_or_create_for_mysql(void) logfilename, dirnamelen, flushed_lsn, logfile0); + if (err == DB_SUCCESS) { + err = create_log_files_rename( + logfilename, + dirnamelen, + flushed_lsn, logfile0); + } + if (err != DB_SUCCESS) { return(srv_init_abort(err)); } - create_log_files_rename( - logfilename, dirnamelen, - flushed_lsn, logfile0); - /* Suppress the message about crash recovery. */ flushed_lsn = log_get_lsn(); goto files_checked; - } else if (i < 2) { - /* must have at least 2 log files */ - ib::error() << "Only one log file" - " found."; - return(srv_init_abort(err)); } /* opened all files */ @@ -2075,10 +2057,10 @@ innobase_start_or_create_for_mysql(void) /* Disable the doublewrite buffer for log files. */ fil_space_t* log_space = fil_space_create( "innodb_redo_log", - SRV_LOG_SPACE_FIRST_ID, - fsp_flags_set_page_size(0, univ_page_size), + SRV_LOG_SPACE_FIRST_ID, 0, FIL_TYPE_LOG, - NULL /* no encryption yet */); + NULL /* no encryption yet */, + true /* create */); ut_a(fil_validate()); ut_a(log_space); @@ -2109,6 +2091,7 @@ files_checked: shutdown */ fil_open_log_and_system_tablespace_files(); + ut_d(fil_space_get(0)->recv_size = srv_sys_space_size_debug); err = srv_undo_tablespaces_init( create_new_db, @@ -2130,18 +2113,11 @@ files_checked: dict_stats_thread_init(); } - if (!srv_read_only_mode && srv_scrub_log) { - /* TODO(minliz): have/use log_scrub_thread_init() instead? */ - log_scrub_event = os_event_create(0); - } - trx_sys_file_format_init(); trx_sys_create(); if (create_new_db) { - dberr_t err = DB_SUCCESS; - ut_a(!srv_read_only_mode); mtr_start(&mtr); @@ -2179,13 +2155,15 @@ files_checked: err = fil_write_flushed_lsn(flushed_lsn); + if (err == DB_SUCCESS) { + err = create_log_files_rename( + logfilename, dirnamelen, + flushed_lsn, logfile0); + } + if (err != DB_SUCCESS) { return(srv_init_abort(err)); } - - create_log_files_rename( - logfilename, dirnamelen, flushed_lsn, logfile0); - } else { /* Check if we support the max format that is stamped @@ -2230,26 +2208,24 @@ files_checked: recv_sys->dblwr.pages.clear(); + if (err == DB_SUCCESS && !srv_read_only_mode) { + log_mutex_enter(); + if (log_sys->is_encrypted() && !log_crypt_init()) { + err = DB_ERROR; + } + log_mutex_exit(); + } + if (err == DB_SUCCESS) { /* Initialize the change buffer. */ err = dict_boot(); } if (err != DB_SUCCESS) { - - /* A tablespace was not found during recovery. The - user must force recovery. */ - - if (err == DB_TABLESPACE_NOT_FOUND) { - - srv_fatal_error(); - - ut_error; - } - - return(srv_init_abort(DB_ERROR)); + return(srv_init_abort(err)); } + /* This must precede recv_apply_hashed_log_recs(TRUE). */ purge_queue = trx_sys_init_at_db_start(); if (srv_force_recovery < SRV_FORCE_NO_LOG_REDO) { @@ -2257,12 +2233,12 @@ files_checked: respective file pages, for the last batch of recv_group_scan_log_recs(). */ - dberr_t err = recv_apply_hashed_log_recs(TRUE); + err = recv_apply_hashed_log_recs(TRUE); DBUG_PRINT("ib_log", ("apply completed")); if (err != DB_SUCCESS) { - ib::warn() << "recv_apply_hashed_log_recs " - << " failed with error " << err; + UT_DELETE(purge_queue); + return(srv_init_abort(err)); } if (recv_needed_recovery) { @@ -2281,6 +2257,66 @@ files_checked: " InnoDB database from a backup!"; } + if (!srv_read_only_mode) { + const ulint flags = FSP_FLAGS_PAGE_SSIZE(); + for (ulint id = 0; id <= srv_undo_tablespaces; id++) { + if (fil_space_get(id)) { + fsp_flags_try_adjust(id, flags); + } + } + + if (sum_of_new_sizes > 0) { + /* New data file(s) were added */ + mtr.start(); + fsp_header_inc_size(0, sum_of_new_sizes, &mtr); + mtr.commit(); + /* Immediately write the log record about + increased tablespace size to disk, so that it + is durable even if mysqld would crash + quickly */ + log_buffer_flush_to_disk(); + } + } + + const ulint tablespace_size_in_header + = fsp_header_get_tablespace_size(); + const ulint sum_of_data_file_sizes + = srv_sys_space.get_sum_of_sizes(); + /* Compare the system tablespace file size to what is + stored in FSP_SIZE. In open_or_create_data_files() + we already checked that the file sizes match the + innodb_data_file_path specification. */ + if (srv_read_only_mode + || sum_of_data_file_sizes == tablespace_size_in_header) { + /* Do not complain about the size. */ + } else if (!srv_sys_space.can_auto_extend_last_file() + || sum_of_data_file_sizes + < tablespace_size_in_header) { + ib::error() << "Tablespace size stored in header is " + << tablespace_size_in_header + << " pages, but the sum of data file sizes is " + << sum_of_data_file_sizes << " pages"; + + if (srv_force_recovery == 0 + && sum_of_data_file_sizes + < tablespace_size_in_header) { + ib::error() << + "Cannot start InnoDB. The tail of" + " the system tablespace is" + " missing. Have you edited" + " innodb_data_file_path in my.cnf" + " in an inappropriate way, removing" + " data files from there?" + " You can set innodb_force_recovery=1" + " in my.cnf to force" + " a startup if you are trying to" + " recover a badly corrupt database."; + + UT_DELETE(purge_queue); + return(srv_init_abort(DB_ERROR)); + } + } + /* The purge system needs to create the purge view and therefore requires that the trx_sys is inited. */ @@ -2291,6 +2327,118 @@ files_checked: recv_recovery_from_checkpoint_finish(); + /* Upgrade or resize or rebuild the redo logs before + generating any dirty pages, so that the old redo log + files will not be written to. */ + + if (srv_force_recovery == SRV_FORCE_NO_LOG_REDO) { + /* Completely ignore the redo log. */ + } else if (srv_read_only_mode) { + /* Leave the redo log alone. */ + } else if (srv_log_file_size_requested == srv_log_file_size + && srv_n_log_files_found == srv_n_log_files + && log_sys->is_encrypted() == srv_encrypt_log) { + /* No need to upgrade or resize the redo log. */ + } else { + /* Prepare to delete the old redo log files */ + flushed_lsn = srv_prepare_to_delete_redo_log_files(i); + + DBUG_EXECUTE_IF("innodb_log_abort_1", + return(srv_init_abort(DB_ERROR));); + /* Prohibit redo log writes from any other + threads until creating a log checkpoint at the + end of create_log_files(). */ + ut_d(recv_no_log_write = true); + ut_ad(!buf_pool_check_no_pending_io()); + + DBUG_EXECUTE_IF("innodb_log_abort_3", + return(srv_init_abort(DB_ERROR));); + DBUG_PRINT("ib_log", ("After innodb_log_abort_3")); + + /* Stamp the LSN to the data files. */ + err = fil_write_flushed_lsn(flushed_lsn); + + DBUG_EXECUTE_IF("innodb_log_abort_4", err = DB_ERROR;); + DBUG_PRINT("ib_log", ("After innodb_log_abort_4")); + + if (err != DB_SUCCESS) { + return(srv_init_abort(err)); + } + + /* Close and free the redo log files, so that + we can replace them. */ + fil_close_log_files(true); + + DBUG_EXECUTE_IF("innodb_log_abort_5", + return(srv_init_abort(DB_ERROR));); + DBUG_PRINT("ib_log", ("After innodb_log_abort_5")); + + /* Free the old log file space. */ + log_group_close_all(); + + ib::info() << "Starting to delete and rewrite log" + " files."; + + srv_log_file_size = srv_log_file_size_requested; + + err = create_log_files( + logfilename, dirnamelen, flushed_lsn, + logfile0); + + if (err == DB_SUCCESS) { + err = create_log_files_rename( + logfilename, dirnamelen, flushed_lsn, + logfile0); + } + + if (err != DB_SUCCESS) { + return(srv_init_abort(err)); + } + } + + + /* Validate a few system page types that were left + uninitialized by older versions of MySQL. */ + if (!high_level_read_only) { + mtr_t mtr; + buf_block_t* block; + mtr.start(); + mtr.set_sys_modified(); + /* Bitmap page types will be reset in + buf_dblwr_check_block() without redo logging. */ + block = buf_page_get( + page_id_t(IBUF_SPACE_ID, + FSP_IBUF_HEADER_PAGE_NO), + univ_page_size, RW_X_LATCH, &mtr); + fil_block_check_type(block, FIL_PAGE_TYPE_SYS, &mtr); + /* Already MySQL 3.23.53 initialized + FSP_IBUF_TREE_ROOT_PAGE_NO to + FIL_PAGE_INDEX. No need to reset that one. */ + block = buf_page_get( + page_id_t(TRX_SYS_SPACE, TRX_SYS_PAGE_NO), + univ_page_size, RW_X_LATCH, &mtr); + fil_block_check_type(block, FIL_PAGE_TYPE_TRX_SYS, + &mtr); + block = buf_page_get( + page_id_t(TRX_SYS_SPACE, + FSP_FIRST_RSEG_PAGE_NO), + univ_page_size, RW_X_LATCH, &mtr); + fil_block_check_type(block, FIL_PAGE_TYPE_SYS, &mtr); + block = buf_page_get( + page_id_t(TRX_SYS_SPACE, FSP_DICT_HDR_PAGE_NO), + univ_page_size, RW_X_LATCH, &mtr); + fil_block_check_type(block, FIL_PAGE_TYPE_SYS, &mtr); + mtr.commit(); + } + + /* Roll back any recovered data dictionary transactions, so + that the data dictionary tables will be free of any locks. + The data dictionary latch should guarantee that there is at + most one data dictionary transaction active at a time. */ + if (srv_force_recovery < SRV_FORCE_NO_TRX_UNDO) { + trx_rollback_or_clean_recovered(FALSE); + } + /* Fix-up truncate of tables in the system tablespace if server crashed while truncate was active. The non- system tables are done after tablespace discovery. Do @@ -2303,7 +2451,6 @@ files_checked: /* Open or Create SYS_TABLESPACES and SYS_DATAFILES so that tablespace names and other metadata can be found. */ - srv_sys_tablespaces_open = true; err = dict_create_or_check_sys_tablespace(); if (err != DB_SUCCESS) { return(srv_init_abort(err)); @@ -2337,17 +2484,6 @@ files_checked: dict_check_tablespaces_and_store_max_id(validate); } -#ifdef MYSQL_ENCRYPTION - /* Rotate the encryption key for recovery. It's because - server could crash in middle of key rotation. Some tablespace - didn't complete key rotation. Here, we will resume the - rotation. */ - if (!srv_read_only_mode - && srv_force_recovery < SRV_FORCE_NO_LOG_REDO) { - fil_encryption_rotate(); - } -#endif /* MYSQL_ENCRYPTION */ - /* Fix-up truncate of table if server crashed while truncate was active. */ err = truncate_t::fixup_tables_in_non_system_tablespace(); @@ -2356,67 +2492,6 @@ files_checked: return(srv_init_abort(err)); } - if (!srv_force_recovery - && !recv_sys->found_corrupt_log - && (srv_log_file_size_requested != srv_log_file_size - || srv_n_log_files_found != srv_n_log_files)) { - dberr_t err = DB_SUCCESS; - - /* Prepare to replace the redo log files. */ - - if (srv_read_only_mode) { - ib::error() << "Cannot resize log files" - " in read-only mode."; - return(srv_init_abort(DB_READ_ONLY)); - } - - /* Prepare to delete the old redo log files */ - flushed_lsn = srv_prepare_to_delete_redo_log_files(i); - - /* Prohibit redo log writes from any other - threads until creating a log checkpoint at the - end of create_log_files(). */ - ut_d(recv_no_log_write = true); - ut_ad(!buf_pool_check_no_pending_io()); - - RECOVERY_CRASH(3); - - /* Stamp the LSN to the data files. */ - err = fil_write_flushed_lsn(flushed_lsn); - - if (err != DB_SUCCESS) { - return(srv_init_abort(err)); - } - - RECOVERY_CRASH(4); - - /* Close and free the redo log files, so that - we can replace them. */ - fil_close_log_files(true); - - RECOVERY_CRASH(5); - - /* Free the old log file space. */ - log_group_close_all(); - - ib::warn() << "Starting to delete and rewrite log" - " files."; - - srv_log_file_size = srv_log_file_size_requested; - - err = create_log_files( - logfilename, dirnamelen, flushed_lsn, - logfile0); - - if (err != DB_SUCCESS) { - return(srv_init_abort(err)); - } - - create_log_files_rename( - logfilename, dirnamelen, flushed_lsn, - logfile0); - } - recv_recovery_rollback_active(); /* It is possible that file_format tag has never @@ -2427,30 +2502,20 @@ files_checked: trx_sys_file_format_tag_init(); } - if (!create_new_db && sum_of_new_sizes > 0) { - /* New data file(s) were added */ - mtr_start(&mtr); - - fsp_header_inc_size(0, sum_of_new_sizes, &mtr); - - mtr_commit(&mtr); - - /* Immediately write the log record about increased tablespace - size to disk, so that it is durable even if mysqld would crash - quickly */ - - log_buffer_flush_to_disk(); - } + ut_ad(err == DB_SUCCESS); + ut_a(sum_of_new_sizes != ULINT_UNDEFINED); /* Open temp-tablespace and keep it open until shutdown. */ - err = srv_open_tmp_tablespace(create_new_db, &srv_tmp_space); + err = srv_open_tmp_tablespace(create_new_db); if (err != DB_SUCCESS) { return(srv_init_abort(err)); } + /* Create the doublewrite buffer to a new tablespace */ - if (buf_dblwr == NULL && !buf_dblwr_create()) { + if (!srv_read_only_mode && srv_force_recovery < SRV_FORCE_NO_TRX_UNDO + && !buf_dblwr_create()) { return(srv_init_abort(DB_ERROR)); } @@ -2498,14 +2563,17 @@ files_checked: lock_wait_timeout_thread, NULL, thread_ids + 2 + SRV_MAX_N_IO_THREADS); thread_started[2 + SRV_MAX_N_IO_THREADS] = true; + lock_sys->timeout_thread_active = true; /* Create the thread which warns of long semaphore waits */ + srv_error_monitor_active = true; thread_handles[3 + SRV_MAX_N_IO_THREADS] = os_thread_create( srv_error_monitor_thread, NULL, thread_ids + 3 + SRV_MAX_N_IO_THREADS); thread_started[3 + SRV_MAX_N_IO_THREADS] = true; /* Create the thread which prints InnoDB monitor info */ + srv_monitor_active = true; thread_handles[4 + SRV_MAX_N_IO_THREADS] = os_thread_create( srv_monitor_thread, NULL, thread_ids + 4 + SRV_MAX_N_IO_THREADS); @@ -2515,20 +2583,22 @@ files_checked: /* Create the SYS_FOREIGN and SYS_FOREIGN_COLS system tables */ err = dict_create_or_check_foreign_constraint_tables(); - if (err != DB_SUCCESS) { - return(srv_init_abort(err)); - } - - /* Create the SYS_TABLESPACES system table */ - err = dict_create_or_check_sys_tablespace(); - if (err != DB_SUCCESS) { - return(srv_init_abort(err)); + if (err == DB_SUCCESS) { + err = dict_create_or_check_sys_tablespace(); + if (err == DB_SUCCESS) { + err = dict_create_or_check_sys_virtual(); + } } - srv_sys_tablespaces_open = true; - - /* Create the SYS_VIRTUAL system table */ - err = dict_create_or_check_sys_virtual(); - if (err != DB_SUCCESS) { + switch (err) { + case DB_SUCCESS: + break; + case DB_READ_ONLY: + if (srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO) { + break; + } + ib::error() << "Cannot create system tables in read-only mode"; + /* fall through */ + default: return(srv_init_abort(err)); } @@ -2575,10 +2645,10 @@ files_checked: purge_sys->state = PURGE_STATE_DISABLED; } - /* wake main loop of page cleaner up */ - os_event_set(buf_flush_event); - if (!srv_read_only_mode) { + /* wake main loop of page cleaner up */ + os_event_set(buf_flush_event); + if (srv_use_mtflush) { /* Start multi-threaded flush threads */ mtflush_ctx = buf_mtflu_handler_init( @@ -2593,68 +2663,6 @@ files_checked: } } - sum_of_data_file_sizes = srv_sys_space.get_sum_of_sizes(); - ut_a(sum_of_new_sizes != ULINT_UNDEFINED); - - tablespace_size_in_header = fsp_header_get_tablespace_size(); - - if (!srv_read_only_mode - && !srv_sys_space.can_auto_extend_last_file() - && sum_of_data_file_sizes != tablespace_size_in_header) { - - ib::error() << "Tablespace size stored in header is " - << tablespace_size_in_header << " pages, but the sum" - " of data file sizes is " << sum_of_data_file_sizes - << " pages"; - - if (srv_force_recovery == 0 - && sum_of_data_file_sizes < tablespace_size_in_header) { - /* This is a fatal error, the tail of a tablespace is - missing */ - - ib::error() - << "Cannot start InnoDB." - " The tail of the system tablespace is" - " missing. Have you edited" - " innodb_data_file_path in my.cnf in an" - " inappropriate way, removing" - " ibdata files from there?" - " You can set innodb_force_recovery=1" - " in my.cnf to force" - " a startup if you are trying" - " to recover a badly corrupt database."; - - return(srv_init_abort(DB_ERROR)); - } - } - - if (!srv_read_only_mode - && srv_sys_space.can_auto_extend_last_file() - && sum_of_data_file_sizes < tablespace_size_in_header) { - - ib::error() << "Tablespace size stored in header is " - << tablespace_size_in_header << " pages, but the sum" - " of data file sizes is only " - << sum_of_data_file_sizes << " pages"; - - if (srv_force_recovery == 0) { - - ib::error() - << "Cannot start InnoDB. The tail of" - " the system tablespace is" - " missing. Have you edited" - " innodb_data_file_path in my.cnf in an" - " InnoDB: inappropriate way, removing" - " ibdata files from there?" - " You can set innodb_force_recovery=1" - " in my.cnf to force" - " InnoDB: a startup if you are trying to" - " recover a badly corrupt database."; - - return(srv_init_abort(DB_ERROR)); - } - } - if (srv_print_verbose_log) { ib::info() << INNODB_VERSION_STR << " started; log sequence number " @@ -2666,23 +2674,6 @@ files_checked: << srv_force_recovery << " !!!"; } - if (!srv_read_only_mode) { - /* Create thread(s) that handles key rotation. This is - needed already here as log_preflush_pool_modified_pages - will flush dirty pages and that might need e.g. - fil_crypt_threads_event. */ - fil_system_enter(); - fil_crypt_threads_init(); - fil_system_exit(); - - /* - Create a checkpoint before logging anything new, so that - the current encryption key in use is definitely logged - before any log blocks encrypted with that key. - */ - log_make_checkpoint_at(LSN_MAX, TRUE); - } - if (srv_force_recovery == 0) { /* In the insert buffer we may have even bigger tablespace id's, because we may have dropped those tablespaces, but @@ -2707,6 +2698,8 @@ files_checked: /* Create the buffer pool dump/load thread */ buf_dump_thread_handle= os_thread_create(buf_dump_thread, NULL, NULL); + + srv_buf_dump_thread_active = true; buf_dump_thread_started = true; #ifdef WITH_WSREP } else { @@ -2716,31 +2709,43 @@ files_checked: } #endif /* WITH_WSREP */ + /* Create thread(s) that handles key rotation. This is + needed already here as log_preflush_pool_modified_pages + will flush dirty pages and that might need e.g. + fil_crypt_threads_event. */ + fil_system_enter(); + fil_crypt_threads_init(); + fil_system_exit(); + + /* + Create a checkpoint before logging anything new, so that + the current encryption key in use is definitely logged + before any log blocks encrypted with that key. + */ + log_make_checkpoint_at(LSN_MAX, TRUE); + /* Create the dict stats gathering thread */ - dict_stats_thread_handle = os_thread_create(dict_stats_thread, NULL, NULL); + dict_stats_thread_handle = os_thread_create( + dict_stats_thread, NULL, NULL); + srv_dict_stats_thread_active = true; dict_stats_thread_started = true; /* Create the thread that will optimize the FTS sub-system. */ fts_optimize_init(); - srv_start_state_set(SRV_START_STATE_STAT); + /* Init data for datafile scrub threads */ + btr_scrub_init(); - /* Create the log scrub thread */ - if (srv_scrub_log) { - os_thread_create(log_scrub_thread, NULL, NULL); - } + /* Initialize online defragmentation. */ + btr_defragment_init(); + + srv_start_state_set(SRV_START_STATE_STAT); } /* Create the buffer pool resize thread */ + srv_buf_resize_thread_active = true; os_thread_create(buf_resize_thread, NULL, NULL); - /* Init data for datafile scrub threads */ - btr_scrub_init(); - - /* Initialize online defragmentation. */ - btr_defragment_init(); - - srv_was_started = TRUE; return(DB_SUCCESS); } @@ -2775,27 +2780,26 @@ srv_fts_close(void) #endif /****************************************************************//** -Shuts down the InnoDB database. -@return DB_SUCCESS or error code */ -dberr_t -innobase_shutdown_for_mysql(void) -/*=============================*/ +Shuts down background threads that can generate undo pages. */ +void +srv_shutdown_bg_undo_sources(void) +/*===========================*/ { - if (!srv_was_started) { - if (srv_is_being_started) { - ib::warn() << "Shutting down an improperly started," - " or created database!"; - } - - return(DB_SUCCESS); - } - - if (!srv_read_only_mode) { + if (srv_start_state_is_set(SRV_START_STATE_STAT)) { + ut_ad(!srv_read_only_mode); fts_optimize_shutdown(); dict_stats_shutdown(); + } +} + +/** Shut down InnoDB. */ +void +innodb_shutdown() +{ + ut_ad(!srv_running); - /* Shutdown key rotation threads */ - fil_crypt_threads_end(); + if (srv_fast_shutdown) { + srv_shutdown_bg_undo_sources(); } /* 1. Flush the buffer pool to disk, write the current lsn to @@ -2833,31 +2837,55 @@ innobase_shutdown_for_mysql(void) srv_misc_tmpfile = 0; } - if (!srv_read_only_mode) { + ut_ad(dict_stats_event || !srv_was_started || srv_read_only_mode); + ut_ad(dict_sys || !srv_was_started); + ut_ad(trx_sys || !srv_was_started); + ut_ad(buf_dblwr || !srv_was_started || srv_read_only_mode + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); + ut_ad(lock_sys || !srv_was_started); + ut_ad(btr_search_sys || !srv_was_started); + ut_ad(ibuf || !srv_was_started); + ut_ad(log_sys || !srv_was_started); + + if (dict_stats_event) { dict_stats_thread_deinit(); - if (srv_scrub_log) { - /* TODO(minliz): have/use log_scrub_thread_deinit() instead? */ - os_event_destroy(log_scrub_event); - log_scrub_event = NULL; - } } - if (!srv_read_only_mode) { + if (srv_start_state_is_set(SRV_START_STATE_STAT)) { + ut_ad(!srv_read_only_mode); + /* srv_shutdown_bg_undo_sources() already invoked + fts_optimize_shutdown(); dict_stats_shutdown(); */ + fil_crypt_threads_cleanup(); + btr_scrub_cleanup(); + /* FIXME: call btr_defragment_shutdown(); */ } - /* Cleanup data for datafile scrubbing */ - btr_scrub_cleanup(); - /* This must be disabled before closing the buffer pool and closing the data dictionary. */ - btr_search_disable(true); - ibuf_close(); - log_shutdown(); - trx_sys_file_format_close(); - trx_sys_close(); - lock_sys_close(); + if (dict_sys) { + btr_search_disable(true); + } + if (ibuf) { + ibuf_close(); + } + if (log_sys) { + log_shutdown(); + } + if (trx_sys) { + trx_sys_file_format_close(); + trx_sys_close(); + } + if (purge_sys) { + trx_purge_sys_close(); + } + if (buf_dblwr) { + buf_dblwr_free(); + } + if (lock_sys) { + lock_sys_close(); + } trx_pool_close(); @@ -2869,13 +2897,17 @@ innobase_shutdown_for_mysql(void) mutex_free(&srv_misc_tmpfile_mutex); } - dict_close(); - btr_search_sys_free(); + if (dict_sys) { + dict_close(); + } + + if (btr_search_sys) { + btr_search_sys_free(); + } /* 3. Free all InnoDB's own mutexes and the os_fast_mutexes inside them */ os_aio_free(); - que_close(); row_mysql_close(); srv_free(); fil_close(); @@ -2896,18 +2928,15 @@ innobase_shutdown_for_mysql(void) fclose(dict_foreign_err_file); } - if (srv_print_verbose_log) { + if (srv_was_started && srv_print_verbose_log) { ib::info() << "Shutdown completed; log sequence number " << srv_shutdown_lsn; } + srv_start_state = SRV_START_STATE_NONE; srv_was_started = FALSE; srv_start_has_been_called = FALSE; - - return(DB_SUCCESS); } -#endif /* !UNIV_HOTBACKUP */ - /******************************************************************** Signal all per-table background threads to shutdown, and wait for them to do @@ -3019,36 +3048,3 @@ srv_get_meta_data_filename( ut_free(path); } -/** Get the encryption-data filename from the table name for a -single-table tablespace. -@param[in] table table object -@param[out] filename filename -@param[in] max_len filename max length */ -void -srv_get_encryption_data_filename( - dict_table_t* table, - char* filename, - ulint max_len) -{ - ulint len; - char* path; - /* Make sure the data_dir_path is set. */ - dict_get_and_save_data_dir_path(table, false); - - if (DICT_TF_HAS_DATA_DIR(table->flags)) { - ut_a(table->data_dir_path); - - path = fil_make_filepath( - table->data_dir_path, table->name.m_name, CFP, true); - } else { - path = fil_make_filepath(NULL, table->name.m_name, CFP, false); - } - - ut_a(path); - len = ut_strlen(path); - ut_a(max_len >= len); - - strcpy(filename, path); - - ut_free(path); -} diff --git a/storage/innobase/sync/sync0arr.cc b/storage/innobase/sync/sync0arr.cc index f01149f8a8d..c9423a04fba 100644 --- a/storage/innobase/sync/sync0arr.cc +++ b/storage/innobase/sync/sync0arr.cc @@ -1340,7 +1340,7 @@ sync_arr_fill_sys_semphore_waits_table( */ OK(field_store_string(fields[SYS_SEMAPHORE_WAITS_FILE], innobase_basename(cell->file))); OK(field_store_ulint(fields[SYS_SEMAPHORE_WAITS_LINE], cell->line)); - OK(field_store_ulint(fields[SYS_SEMAPHORE_WAITS_WAIT_TIME], (longlong)difftime(time(NULL), cell->reservation_time))); + OK(field_store_ulint(fields[SYS_SEMAPHORE_WAITS_WAIT_TIME], (ulint)difftime(time(NULL), cell->reservation_time))); if (type == SYNC_MUTEX) { mutex = static_cast<WaitMutex*>(cell->latch.mutex); diff --git a/storage/innobase/sync/sync0debug.cc b/storage/innobase/sync/sync0debug.cc index 072555750dc..1d9e8933185 100644 --- a/storage/innobase/sync/sync0debug.cc +++ b/storage/innobase/sync/sync0debug.cc @@ -1544,8 +1544,6 @@ sync_latch_meta_init() LATCH_ADD(SYNC_DEBUG_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); - LATCH_ADD(MASTER_KEY_ID_MUTEX, SYNC_NO_ORDER_CHECK, master_key_id_mutex_key); - /* JAN: TODO: Add PFS instrumentation */ LATCH_ADD(SCRUB_STAT_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); LATCH_ADD(DEFRAGMENT_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); @@ -1582,7 +1580,6 @@ sync_latch_meta_init() } /** Destroy the latch meta data */ -#ifdef JAN_DISABLED_FOR_NOW_AS_THIS_CAUSES_CRASH static void sync_latch_meta_destroy() @@ -1596,7 +1593,6 @@ sync_latch_meta_destroy() latch_meta.clear(); } -#endif /** Track mutex file creation name and line number. This is to avoid storing { const char* name; uint16_t line; } in every instance. This results in the @@ -1810,8 +1806,6 @@ sync_check_close() create_tracker = NULL; -#ifdef JAN_DISABLED_FOR_NOW_AS_THIS_CAUSES_CRASH sync_latch_meta_destroy(); -#endif } diff --git a/storage/innobase/sync/sync0rw.cc b/storage/innobase/sync/sync0rw.cc index 5f617ae22f5..5cb90ce5b8d 100644 --- a/storage/innobase/sync/sync0rw.cc +++ b/storage/innobase/sync/sync0rw.cc @@ -498,7 +498,7 @@ rw_lock_x_lock_wait_func( rw_lock_stats.rw_x_spin_round_count.add(n_spins); if (count_os_wait > 0) { - lock->count_os_wait += count_os_wait; + lock->count_os_wait += static_cast<uint32_t>(count_os_wait); rw_lock_stats.rw_x_os_wait_count.add(count_os_wait); } diff --git a/storage/innobase/sync/sync0sync.cc b/storage/innobase/sync/sync0sync.cc index 9aba075caa6..099a56c5457 100644 --- a/storage/innobase/sync/sync0sync.cc +++ b/storage/innobase/sync/sync0sync.cc @@ -98,7 +98,6 @@ mysql_pfs_key_t sync_array_mutex_key; mysql_pfs_key_t thread_mutex_key; mysql_pfs_key_t zip_pad_mutex_key; mysql_pfs_key_t row_drop_list_mutex_key; -mysql_pfs_key_t master_key_id_mutex_key; #endif /* UNIV_PFS_MUTEX */ #ifdef UNIV_PFS_RWLOCK mysql_pfs_key_t btr_search_latch_key; diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 49c3889c2f1..a732b1ea150 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -282,23 +283,16 @@ trx_purge_sys_close(void) sess_close(purge_sys->sess); - purge_sys->sess = NULL; - purge_sys->view.close(); purge_sys->view.~ReadView(); rw_lock_free(&purge_sys->latch); mutex_free(&purge_sys->pq_mutex); - if (purge_sys->purge_queue != NULL) { - UT_DELETE(purge_sys->purge_queue); - purge_sys->purge_queue = NULL; - } + UT_DELETE(purge_sys->purge_queue); os_event_destroy(purge_sys->event); - purge_sys->event = NULL; - UT_DELETE(purge_sys->rseg_iter); ut_free(purge_sys); @@ -395,19 +389,33 @@ trx_purge_add_update_undo_to_history( } } -/**********************************************************************//** -Frees an undo log segment which is in the history list. Cuts the end of the -history list at the youngest undo log in this segment. */ +/** Remove undo log header from the history list. +@param[in,out] rseg_hdr rollback segment header +@param[in] log_hdr undo log segment header +@param[in,out] mtr mini transaction. */ +static +void +trx_purge_remove_log_hdr( + trx_rsegf_t* rseg_hdr, + trx_ulogf_t* log_hdr, + mtr_t* mtr) +{ + flst_remove(rseg_hdr + TRX_RSEG_HISTORY, + log_hdr + TRX_UNDO_HISTORY_NODE, mtr); + my_atomic_addlint(&trx_sys->rseg_history_len, -1); +} + +/** Frees an undo log segment which is in the history list. Removes the +undo log hdr from the history list. +@param[in,out] rseg rollback segment +@param[in] hdr_addr file address of log_hdr +@param[in] noredo skip redo logging. */ static void trx_purge_free_segment( -/*===================*/ - trx_rseg_t* rseg, /*!< in: rollback segment */ - fil_addr_t hdr_addr, /*!< in: the file address of log_hdr */ - ulint n_removed_logs, /*!< in: count of how many undo logs we - will cut off from the end of the - history list */ - bool noredo) /*!< in: skip redo logging */ + trx_rseg_t* rseg, + fil_addr_t hdr_addr, + bool noredo) { mtr_t mtr; trx_rsegf_t* rseg_hdr; @@ -473,10 +481,7 @@ trx_purge_free_segment( history list: otherwise, in case of a database crash, the segment could become inaccessible garbage in the file space. */ - flst_cut_end(rseg_hdr + TRX_RSEG_HISTORY, - log_hdr + TRX_UNDO_HISTORY_NODE, n_removed_logs, &mtr); - - my_atomic_addlint(&trx_sys->rseg_history_len, -n_removed_logs); + trx_purge_remove_log_hdr(rseg_hdr, log_hdr, &mtr); do { @@ -518,7 +523,6 @@ trx_purge_truncate_rseg_history( page_t* undo_page; trx_ulogf_t* log_hdr; trx_usegf_t* seg_hdr; - ulint n_removed_logs = 0; mtr_t mtr; trx_id_t undo_trx_no; const bool noredo = trx_sys_is_noredo_rseg_slot( @@ -565,13 +569,6 @@ loop: hdr_addr.boffset, limit->undo_no); } - my_atomic_addlint( - &trx_sys->rseg_history_len, -n_removed_logs); - - flst_truncate_end(rseg_hdr + TRX_RSEG_HISTORY, - log_hdr + TRX_UNDO_HISTORY_NODE, - n_removed_logs, &mtr); - mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); @@ -580,7 +577,6 @@ loop: prev_hdr_addr = trx_purge_get_log_from_hist( flst_get_prev_addr(log_hdr + TRX_UNDO_HISTORY_NODE, &mtr)); - n_removed_logs++; seg_hdr = undo_page + TRX_UNDO_SEG_HDR; @@ -592,10 +588,13 @@ loop: mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); - trx_purge_free_segment(rseg, hdr_addr, n_removed_logs, noredo); - - n_removed_logs = 0; + /* calls the trx_purge_remove_log_hdr() + inside trx_purge_free_segment(). */ + trx_purge_free_segment(rseg, hdr_addr, noredo); } else { + /* Remove the log hdr from the rseg history. */ + trx_purge_remove_log_hdr(rseg_hdr, log_hdr, &mtr); + mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); } @@ -698,8 +697,6 @@ namespace undo { IORequest request(IORequest::WRITE); - request.disable_compression(); - err = os_file_write( request, log_file_name, handle, log_buf, 0, sz); @@ -763,8 +760,6 @@ namespace undo { IORequest request(IORequest::WRITE); - request.disable_compression(); - err = os_file_write( request, log_file_name, handle, log_buf, 0, sz); @@ -832,8 +827,6 @@ namespace undo { IORequest request(IORequest::READ); - request.disable_compression(); - dberr_t err; err = os_file_read(request, handle, log_buf, 0, sz); @@ -928,10 +921,9 @@ trx_purge_mark_undo_for_truncate( return; } -#ifdef UNIV_DEBUG - ib::info() << "UNDO tablespace with space identifier " - << undo_trunc->get_marked_space_id() << " marked for truncate"; -#endif /* UNIV_DEBUG */ + DBUG_LOG("undo", + "marking for truncate UNDO tablespace " + << undo_trunc->get_marked_space_id()); /* Step-3: Iterate over all the rsegs of selected UNDO tablespace and mark them temporarily unavailable for allocation.*/ @@ -1912,20 +1904,16 @@ void trx_purge_stop(void) /*================*/ { - purge_state_t state; - int64_t sig_count = os_event_reset(purge_sys->event); - ut_a(srv_n_purge_threads > 0); rw_lock_x_lock(&purge_sys->latch); - ut_a(purge_sys->state != PURGE_STATE_INIT); - ut_a(purge_sys->state != PURGE_STATE_EXIT); - ut_a(purge_sys->state != PURGE_STATE_DISABLED); + const int64_t sig_count = os_event_reset(purge_sys->event); + const purge_state_t state = purge_sys->state; - ++purge_sys->n_stop; + ut_a(state == PURGE_STATE_RUN || state == PURGE_STATE_STOP); - state = purge_sys->state; + ++purge_sys->n_stop; if (state == PURGE_STATE_RUN) { ib::info() << "Stopping purge"; @@ -1938,18 +1926,14 @@ trx_purge_stop(void) purge_sys->state = PURGE_STATE_STOP; - rw_lock_x_unlock(&purge_sys->latch); - if (state != PURGE_STATE_STOP) { - + rw_lock_x_unlock(&purge_sys->latch); /* Wait for purge coordinator to signal that it is suspended. */ os_event_wait_low(purge_sys->event, sig_count); } else { bool once = true; - rw_lock_x_lock(&purge_sys->latch); - /* Wait for purge to signal that it has actually stopped. */ while (purge_sys->running) { @@ -1986,17 +1970,11 @@ trx_purge_run(void) ut_error; case PURGE_STATE_RUN: - case PURGE_STATE_STOP: + ut_a(!purge_sys->n_stop); break; - } - - if (purge_sys->n_stop > 0) { - - ut_a(purge_sys->state == PURGE_STATE_STOP); - - --purge_sys->n_stop; - - if (purge_sys->n_stop == 0) { + case PURGE_STATE_STOP: + ut_a(purge_sys->n_stop); + if (--purge_sys->n_stop == 0) { ib::info() << "Resuming purge"; @@ -2004,8 +1982,6 @@ trx_purge_run(void) } MONITOR_INC_VALUE(MONITOR_PURGE_RESUME_COUNT, 1); - } else { - ut_a(purge_sys->state == PURGE_STATE_RUN); } rw_lock_x_unlock(&purge_sys->latch); diff --git a/storage/innobase/trx/trx0rec.cc b/storage/innobase/trx/trx0rec.cc index b99ce22823b..3cea123cc1b 100644 --- a/storage/innobase/trx/trx0rec.cc +++ b/storage/innobase/trx/trx0rec.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -33,7 +34,6 @@ Created 3/26/1996 Heikki Tuuri #include "mach0data.h" #include "trx0undo.h" #include "mtr0log.h" -#ifndef UNIV_HOTBACKUP #include "dict0dict.h" #include "ut0mem.h" #include "read0read.h" @@ -87,7 +87,6 @@ trx_undof_page_add_undo_rec_log( mlog_catenate_string(mtr, undo_page + old_free + 2, len); } } -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses a redo log record of adding an undo log record. @@ -135,7 +134,6 @@ trx_undo_parse_add_undo_rec( return(ptr + len); } -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Calculates the free space left for extending an undo log record. @return bytes left */ @@ -1245,7 +1243,6 @@ trx_undo_page_report_modify( ut_a(prefix_len < sizeof ext_buf); - spatial_status = dict_col_get_spatial_status( col); @@ -1791,7 +1788,6 @@ trx_undo_rec_get_partial_row( return(const_cast<byte*>(ptr)); } -#endif /* !UNIV_HOTBACKUP */ /***********************************************************************//** Erases the unused undo log page end. @@ -1838,7 +1834,6 @@ trx_undo_parse_erase_page_end( return(ptr); } -#ifndef UNIV_HOTBACKUP /***********************************************************************//** Writes information to an undo log about an insert, update, or a delete marking of a clustered index record. This information is used in a rollback of the @@ -2223,8 +2218,6 @@ trx_undo_prev_version_build( /*!< in: status determine if it is going into this function by purge thread or not. And if we read "after image" of undo log */ - - { trx_undo_rec_t* undo_rec = NULL; dtuple_t* entry; @@ -2242,9 +2235,9 @@ trx_undo_prev_version_build( byte* buf; ut_ad(!rw_lock_own(&purge_sys->latch, RW_LOCK_S)); - ut_ad(mtr_memo_contains_page(index_mtr, index_rec, MTR_MEMO_PAGE_S_FIX) - || mtr_memo_contains_page(index_mtr, index_rec, - MTR_MEMO_PAGE_X_FIX)); + ut_ad(mtr_memo_contains_page_flagged(index_mtr, index_rec, + MTR_MEMO_PAGE_S_FIX + | MTR_MEMO_PAGE_X_FIX)); ut_ad(rec_offs_validate(rec, index, offsets)); ut_a(dict_index_is_clust(index)); @@ -2411,7 +2404,6 @@ trx_undo_prev_version_build( v_status & TRX_UNDO_PREV_IN_PURGE, NULL); } - return(true); } @@ -2495,4 +2487,3 @@ trx_undo_read_v_cols( ut_ad(ptr == end_ptr); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/trx/trx0roll.cc b/storage/innobase/trx/trx0roll.cc index a22b903df7a..dc779fefc58 100644 --- a/storage/innobase/trx/trx0roll.cc +++ b/storage/innobase/trx/trx0roll.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -581,7 +582,8 @@ trx_release_savepoint_for_mysql( { trx_named_savept_t* savep; - ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || trx_state_eq(trx, TRX_STATE_PREPARED)); + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE, true) + || trx_state_eq(trx, TRX_STATE_PREPARED, true)); ut_ad(trx->in_mysql_trx_list); savep = trx_savepoint_find(trx, savepoint_name); diff --git a/storage/innobase/trx/trx0sys.cc b/storage/innobase/trx/trx0sys.cc index a253ea72e95..27321d73bd2 100644 --- a/storage/innobase/trx/trx0sys.cc +++ b/storage/innobase/trx/trx0sys.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -32,10 +33,6 @@ Created 3/26/1996 Heikki Tuuri #include "trx0sys.ic" #endif -#ifdef UNIV_HOTBACKUP -#include "fsp0types.h" - -#else /* !UNIV_HOTBACKUP */ #include "fsp0fsp.h" #include "mtr0log.h" #include "mtr0log.h" @@ -64,7 +61,6 @@ struct file_format_t { /** The transaction system */ trx_sys_t* trx_sys = NULL; -#endif /* !UNIV_HOTBACKUP */ /** List of animal names representing file format. */ static const char* file_format_name_map[] = { @@ -134,7 +130,6 @@ ReadView::check_trx_id_sanity( } } -#ifndef UNIV_HOTBACKUP #ifdef UNIV_DEBUG /* Flag to control TRX_RSEG_N_SLOTS behavior debugging. */ uint trx_rseg_n_slots_debug = 0; @@ -983,8 +978,7 @@ trx_sys_create_noredo_rsegs( Slot-1....Slot-N: reserved for temp-tablespace. Slot-N+1....Slot-127: reserved for system/undo-tablespace. */ for (ulint i = 0; i < n_nonredo_rseg; i++) { - ulint space = srv_tmp_space.space_id(); - if (trx_rseg_create(space, i) == NULL) { + if (trx_rseg_create(SRV_TMP_SPACE_ID, i) == NULL) { break; } ++n_created; @@ -1077,208 +1071,6 @@ trx_sys_create_rsegs( return(n_used); } -#else /* !UNIV_HOTBACKUP */ -/*****************************************************************//** -Prints to stderr the MySQL binlog info in the system header if the -magic number shows it valid. */ -void -trx_sys_print_mysql_binlog_offset_from_page( -/*========================================*/ - const byte* page) /*!< in: buffer containing the trx - system header page, i.e., page number - TRX_SYS_PAGE_NO in the tablespace */ -{ - const trx_sysf_t* sys_header; - - sys_header = page + TRX_SYS; - - if (mach_read_from_4(sys_header + TRX_SYS_MYSQL_LOG_INFO - + TRX_SYS_MYSQL_LOG_MAGIC_N_FLD) - == TRX_SYS_MYSQL_LOG_MAGIC_N) { - - ib::info() << "mysqlbackup: Last MySQL binlog file position " - << mach_read_from_4( - sys_header + TRX_SYS_MYSQL_LOG_INFO - + TRX_SYS_MYSQL_LOG_OFFSET_HIGH) << " " - << mach_read_from_4( - sys_header + TRX_SYS_MYSQL_LOG_INFO - + TRX_SYS_MYSQL_LOG_OFFSET_LOW) - << ", file name " << sys_header - + TRX_SYS_MYSQL_LOG_INFO + TRX_SYS_MYSQL_LOG_NAME; - } -} - -/*****************************************************************//** -Reads the file format id from the first system table space file. -Even if the call succeeds and returns TRUE, the returned format id -may be ULINT_UNDEFINED signalling that the format id was not present -in the data file. -@return TRUE if call succeeds */ -ibool -trx_sys_read_file_format_id( -/*========================*/ - const char *pathname, /*!< in: pathname of the first system - table space file */ - ulint *format_id) /*!< out: file format of the system table - space */ -{ - os_file_t file; - bool success; - byte buf[UNIV_PAGE_SIZE * 2]; - page_t* page = ut_align(buf, UNIV_PAGE_SIZE); - const byte* ptr; - ib_id_t file_format_id; - - *format_id = ULINT_UNDEFINED; - - file = os_file_create_simple_no_error_handling( - innodb_data_file_key, - pathname, - OS_FILE_OPEN, - OS_FILE_READ_ONLY, - srv_read_only_mode, - &success - ); - if (!success) { - /* The following call prints an error message */ - os_file_get_last_error(true); - - ib::error() << "mysqlbackup: Error: trying to read system" - " tablespace file format, but could not open the" - " tablespace file " << pathname << "!"; - return(FALSE); - } - - /* Read the page on which file format is stored */ - - IORequest read_req(IORequest::READ) - - dberr_t err = os_file_read_no_error_handling( - read_req, file, page, TRX_SYS_PAGE_NO * UNIV_PAGE_SIZE, - UNIV_PAGE_SIZE, NULL); - - if (err != DB_SUCCESS) { - /* The following call prints an error message */ - os_file_get_last_error(true); - - ib::error() << "mysqlbackup: Error: trying to read system" - " tablespace file format, but failed to read the" - " tablespace file " << pathname << "!"; - - os_file_close(file); - return(FALSE); - } - os_file_close(file); - - /* get the file format from the page */ - ptr = page + TRX_SYS_FILE_FORMAT_TAG; - file_format_id = mach_read_from_8(ptr); - file_format_id -= TRX_SYS_FILE_FORMAT_TAG_MAGIC_N; - - if (file_format_id >= FILE_FORMAT_NAME_N) { - - /* Either it has never been tagged, or garbage in it. */ - return(TRUE); - } - - *format_id = (ulint) file_format_id; - - return(TRUE); -} - -/*****************************************************************//** -Reads the file format id from the given per-table data file. -@return TRUE if call succeeds */ -ibool -trx_sys_read_pertable_file_format_id( -/*=================================*/ - const char *pathname, /*!< in: pathname of a per-table - datafile */ - ulint *format_id) /*!< out: file format of the per-table - data file */ -{ - os_file_t file; - bool success; - byte buf[UNIV_PAGE_SIZE * 2]; - page_t* page = ut_align(buf, UNIV_PAGE_SIZE); - const byte* ptr; - ib_uint32_t flags; - - *format_id = ULINT_UNDEFINED; - - file = os_file_create_simple_no_error_handling( - innodb_data_file_key, - pathname, - OS_FILE_OPEN, - OS_FILE_READ_ONLY, - srv_read_only_mode, - &success - ); - if (!success) { - /* The following call prints an error message */ - os_file_get_last_error(true); - - ib::error() << "mysqlbackup: Error: trying to read per-table" - " tablespace format, but could not open the tablespace" - " file " << pathname << "!"; - - return(FALSE); - } - - IORequest read_req(IORequest::READ); - - /* Read the first page of the per-table datafile */ - - dberr_t err = os_file_read_no_error_handling( - read_req, file, page, 0, UNIV_PAGE_SIZE, NULL); - - if (err != DB_SUCCESS) { - /* The following call prints an error message */ - os_file_get_last_error(true); - - ib::error() << "mysqlbackup: Error: trying to per-table data" - " file format, but failed to read the tablespace file " - << pathname << "!"; - - os_file_close(file); - return(FALSE); - } - os_file_close(file); - - /* get the file format from the page */ - ptr = page + 54; - flags = mach_read_from_4(ptr); - - if (!fsp_flags_is_valid(flags) { - /* bad tablespace flags */ - return(FALSE); - } - - *format_id = FSP_FLAGS_GET_POST_ANTELOPE(flags); - - return(TRUE); -} - - -/*****************************************************************//** -Get the name representation of the file format from its id. -@return pointer to the name */ -const char* -trx_sys_file_format_id_to_name( -/*===========================*/ - const ulint id) /*!< in: id of the file format */ -{ - if (!(id < FILE_FORMAT_NAME_N)) { - /* unknown id */ - return("Unknown"); - } - - return(file_format_name_map[id]); -} - -#endif /* !UNIV_HOTBACKUP */ - -#ifndef UNIV_HOTBACKUP /********************************************************************* Shutdown/Close the transaction system. */ void @@ -1295,16 +1087,16 @@ trx_sys_close(void) " shutdown: " << size << " read views open"; } - sess_close(trx_dummy_sess); - trx_dummy_sess = NULL; - - trx_purge_sys_close(); - - /* Free the double write data structures. */ - buf_dblwr_free(); + if (trx_dummy_sess) { + sess_close(trx_dummy_sess); + trx_dummy_sess = NULL; + } /* Only prepared transactions may be left in the system. Free them. */ - ut_a(UT_LIST_GET_LEN(trx_sys->rw_trx_list) == trx_sys->n_prepared_trx); + ut_a(UT_LIST_GET_LEN(trx_sys->rw_trx_list) == trx_sys->n_prepared_trx + || !srv_was_started + || srv_read_only_mode + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); for (trx_t* trx = UT_LIST_GET_FIRST(trx_sys->rw_trx_list); trx != NULL; @@ -1359,33 +1151,6 @@ trx_sys_close(void) trx_sys = NULL; } -/** @brief Convert an undo log to TRX_UNDO_PREPARED state on shutdown. - -If any prepared ACTIVE transactions exist, and their rollback was -prevented by innodb_force_recovery, we convert these transactions to -XA PREPARE state in the main-memory data structures, so that shutdown -will proceed normally. These transactions will again recover as ACTIVE -on the next restart, and they will be rolled back unless -innodb_force_recovery prevents it again. - -@param[in] trx transaction -@param[in,out] undo undo log to convert to TRX_UNDO_PREPARED */ -static -void -trx_undo_fake_prepared( - const trx_t* trx, - trx_undo_t* undo) -{ - ut_ad(srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); - ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE)); - ut_ad(trx->is_recovered); - - if (undo != NULL) { - ut_ad(undo->state == TRX_UNDO_ACTIVE); - undo->state = TRX_UNDO_PREPARED; - } -} - /********************************************************************* Check if there are any active (non-prepared) transactions. @return total number of active transactions or 0 if none */ @@ -1393,46 +1158,15 @@ ulint trx_sys_any_active_transactions(void) /*=================================*/ { + ulint total_trx = 0; + trx_sys_mutex_enter(); - ulint total_trx = UT_LIST_GET_LEN(trx_sys->mysql_trx_list); - - if (total_trx == 0) { - total_trx = UT_LIST_GET_LEN(trx_sys->rw_trx_list); - ut_a(total_trx >= trx_sys->n_prepared_trx); - - if (total_trx > trx_sys->n_prepared_trx - && srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO) { - for (trx_t* trx = UT_LIST_GET_FIRST( - trx_sys->rw_trx_list); - trx != NULL; - trx = UT_LIST_GET_NEXT(trx_list, trx)) { - if (!trx_state_eq(trx, TRX_STATE_ACTIVE) - || !trx->is_recovered) { - continue; - } - /* This was a recovered transaction - whose rollback was disabled by - the innodb_force_recovery setting. - Pretend that it is in XA PREPARE - state so that shutdown will work. */ - trx_undo_fake_prepared( - trx, trx->rsegs.m_redo.insert_undo); - trx_undo_fake_prepared( - trx, trx->rsegs.m_redo.update_undo); - trx_undo_fake_prepared( - trx, trx->rsegs.m_noredo.insert_undo); - trx_undo_fake_prepared( - trx, trx->rsegs.m_noredo.update_undo); - trx->state = TRX_STATE_PREPARED; - trx_sys->n_prepared_trx++; - trx_sys->n_prepared_recovered_trx++; - } - } + total_trx = UT_LIST_GET_LEN(trx_sys->rw_trx_list) + + UT_LIST_GET_LEN(trx_sys->mysql_trx_list); - ut_a(total_trx >= trx_sys->n_prepared_trx); - total_trx -= trx_sys->n_prepared_trx; - } + ut_a(total_trx >= trx_sys->n_prepared_trx); + total_trx -= trx_sys->n_prepared_trx; trx_sys_mutex_exit(); @@ -1481,4 +1215,3 @@ trx_sys_validate_trx_list() return(true); } #endif /* UNIV_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 7bead6e16c7..29478edc9e2 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -462,12 +462,6 @@ trx_create_low() /* We just got trx from pool, it should be non locking */ ut_ad(trx->will_lock == 0); - trx->api_trx = false; - - trx->api_auto_commit = false; - - trx->read_write = true; - /* Background trx should not be forced to rollback, we will unset the flag for user trx. */ trx->in_innodb |= TRX_FORCE_ROLLBACK_DISABLE; @@ -632,7 +626,12 @@ trx_free_prepared( /*==============*/ trx_t* trx) /*!< in, own: trx object */ { - ut_a(trx_state_eq(trx, TRX_STATE_PREPARED)); + ut_a(trx_state_eq(trx, TRX_STATE_PREPARED) + || (trx_state_eq(trx, TRX_STATE_ACTIVE) + && trx->is_recovered + && (!srv_was_started + || srv_read_only_mode + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO))); ut_a(trx->magic_n == TRX_MAGIC_N); lock_trx_release_locks(trx); @@ -1354,14 +1353,11 @@ trx_start_low( ++trx->version; /* Check whether it is an AUTOCOMMIT SELECT */ - trx->auto_commit = (trx->api_trx && trx->api_auto_commit) - || thd_trx_is_auto_commit(trx->mysql_thd); + trx->auto_commit = thd_trx_is_auto_commit(trx->mysql_thd); - trx->read_only = - (trx->api_trx && !trx->read_write) + trx->read_only = srv_read_only_mode || (!trx->ddl && !trx->internal - && thd_trx_is_read_only(trx->mysql_thd)) - || srv_read_only_mode; + && thd_trx_is_read_only(trx->mysql_thd)); if (!trx->auto_commit) { ++trx->will_lock; @@ -1488,6 +1484,8 @@ trx_start_low( ut_a(trx->error_state == DB_SUCCESS); + trx->start_time_micro = clock(); + MONITOR_INC(MONITOR_TRX_ACTIVE); } @@ -2097,7 +2095,7 @@ trx_commit_in_memory( trx->dict_operation = TRX_DICT_OP_NONE; #ifdef WITH_WSREP - if (wsrep_on(trx->mysql_thd)) { + if (trx->mysql_thd && wsrep_on(trx->mysql_thd)) { trx->lock.was_chosen_as_deadlock_victim = FALSE; } #endif @@ -3377,24 +3375,20 @@ trx_kill_blocking(trx_t* trx) trx_mutex_exit(victim_trx); -#ifdef UNIV_DEBUG +#ifndef DBUG_OFF char buffer[1024]; - char* thr_text; - trx_id_t id; +#endif /* !DBUG_OFF */ - thr_text = thd_get_error_context_description(victim_trx->mysql_thd, - buffer, sizeof(buffer), - 512); - id = victim_trx->id; -#endif /* UNIV_DEBUG */ - trx_rollback_for_mysql(victim_trx); + DBUG_LOG("trx", + "High Priority Transaction " + << trx->id << " killed transaction " + << victim_trx->id << " in hit list" + << " - " + << thd_get_error_context_description( + victim_trx->mysql_thd, + buffer, sizeof(buffer), 512)); -#ifdef UNIV_DEBUG - ib::info() << "High Priority Transaction (ID): " - << trx->id << " killed transaction (ID): " - << id << " in hit list" - << " - " << thr_text; -#endif /* UNIV_DEBUG */ + trx_rollback_for_mysql(victim_trx); trx_mutex_enter(victim_trx); version++; diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc index 932151b948d..4bc40680c4b 100644 --- a/storage/innobase/trx/trx0undo.cc +++ b/storage/innobase/trx/trx0undo.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -32,7 +33,6 @@ Created 3/26/1996 Heikki Tuuri #endif #include "fsp0fsp.h" -#ifndef UNIV_HOTBACKUP #include "mach0data.h" #include "mtr0log.h" #include "srv0mon.h" @@ -97,7 +97,6 @@ it until a truncate operation occurs, which can remove undo logs from the end of the list and release undo log segments. In stepping through the list, s-latches on the undo log pages are enough, but in a truncate, x-latches must be obtained on the rollback segment and individual pages. */ -#endif /* !UNIV_HOTBACKUP */ /********************************************************************//** Initializes the fields in an undo log segment page. */ @@ -109,7 +108,6 @@ trx_undo_page_init( ulint type, /*!< in: undo log segment type */ mtr_t* mtr); /*!< in: mtr */ -#ifndef UNIV_HOTBACKUP /********************************************************************//** Creates and initializes an undo log memory object. @return own: the undo log memory object */ @@ -126,7 +124,6 @@ trx_undo_mem_create( const XID* xid, /*!< in: X/Open XA transaction identification*/ ulint page_no,/*!< in: undo log header page number */ ulint offset);/*!< in: undo log header byte offset on page */ -#endif /* !UNIV_HOTBACKUP */ /***************************************************************//** Initializes a cached insert undo log header page for new use. NOTE that this function has its own log record type MLOG_UNDO_HDR_REUSE. You must NOT change @@ -150,7 +147,6 @@ trx_undo_discard_latest_update_undo( page_t* undo_page, /*!< in: header page of an undo log of size 1 */ mtr_t* mtr); /*!< in: mtr */ -#ifndef UNIV_HOTBACKUP /***********************************************************************//** Gets the previous record in an undo log from the previous page. @return undo log record, the page s-latched, NULL if none */ @@ -374,9 +370,6 @@ trx_undo_page_init_log( mlog_catenate_ulint_compressed(mtr, type); } -#else /* !UNIV_HOTBACKUP */ -# define trx_undo_page_init_log(undo_page,type,mtr) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses the redo log entry of an undo log page initialization. @@ -431,7 +424,6 @@ trx_undo_page_init( trx_undo_page_init_log(undo_page, type, mtr); } -#ifndef UNIV_HOTBACKUP /***************************************************************//** Creates a new undo log segment in file. @return DB_SUCCESS if page creation OK possible error codes are: @@ -543,9 +535,6 @@ trx_undo_header_create_log( mlog_catenate_ull_compressed(mtr, trx_id); } -#else /* !UNIV_HOTBACKUP */ -# define trx_undo_header_create_log(undo_page,trx_id,mtr) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ /***************************************************************//** Creates a new undo log header in file. NOTE that this function has its own @@ -621,7 +610,6 @@ trx_undo_header_create( return(free); } -#ifndef UNIV_HOTBACKUP /********************************************************************//** Write X/Open XA Transaction Identification (XID) to undo log header */ static @@ -722,9 +710,6 @@ trx_undo_insert_header_reuse_log( mlog_catenate_ull_compressed(mtr, trx_id); } -#else /* !UNIV_HOTBACKUP */ -# define trx_undo_insert_header_reuse_log(undo_page,trx_id,mtr) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ /** Parse the redo log entry of an undo log page header create or reuse. @param[in] type MLOG_UNDO_HDR_CREATE or MLOG_UNDO_HDR_REUSE @@ -820,7 +805,6 @@ trx_undo_insert_header_reuse( return(free); } -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Writes the redo log entry of an update undo log header discard. */ UNIV_INLINE @@ -832,9 +816,6 @@ trx_undo_discard_latest_log( { mlog_write_initial_log_record(undo_page, MLOG_UNDO_HDR_DISCARD, mtr); } -#else /* !UNIV_HOTBACKUP */ -# define trx_undo_discard_latest_log(undo_page, mtr) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses the redo log entry of an undo log page header discard. @@ -898,7 +879,6 @@ trx_undo_discard_latest_update_undo( trx_undo_discard_latest_log(undo_page, mtr); } -#ifndef UNIV_HOTBACKUP /********************************************************************//** Tries to add a page to the undo log segment where the undo log is placed. @return X-latched block if success, else NULL */ @@ -2036,7 +2016,20 @@ trx_undo_free_prepared( ut_ad(srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS); if (trx->rsegs.m_redo.update_undo) { - ut_a(trx->rsegs.m_redo.update_undo->state == TRX_UNDO_PREPARED); + switch (trx->rsegs.m_redo.update_undo->state) { + case TRX_UNDO_PREPARED: + break; + case TRX_UNDO_ACTIVE: + /* lock_trx_release_locks() assigns + trx->is_recovered=false */ + ut_a(!srv_was_started + || srv_read_only_mode + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); + break; + default: + ut_error; + } + UT_LIST_REMOVE(trx->rsegs.m_redo.rseg->update_undo_list, trx->rsegs.m_redo.update_undo); trx_undo_mem_free(trx->rsegs.m_redo.update_undo); @@ -2045,7 +2038,20 @@ trx_undo_free_prepared( } if (trx->rsegs.m_redo.insert_undo) { - ut_a(trx->rsegs.m_redo.insert_undo->state == TRX_UNDO_PREPARED); + switch (trx->rsegs.m_redo.insert_undo->state) { + case TRX_UNDO_PREPARED: + break; + case TRX_UNDO_ACTIVE: + /* lock_trx_release_locks() assigns + trx->is_recovered=false */ + ut_a(!srv_was_started + || srv_read_only_mode + || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); + break; + default: + ut_error; + } + UT_LIST_REMOVE(trx->rsegs.m_redo.rseg->insert_undo_list, trx->rsegs.m_redo.insert_undo); trx_undo_mem_free(trx->rsegs.m_redo.insert_undo); @@ -2176,5 +2182,3 @@ trx_undo_truncate_tablespace( return(success); } - -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/ut/ut0crc32.cc b/storage/innobase/ut/ut0crc32.cc index 2d892be869b..386d32f6a60 100644 --- a/storage/innobase/ut/ut0crc32.cc +++ b/storage/innobase/ut/ut0crc32.cc @@ -2,6 +2,7 @@ Copyright (c) 2009, 2010 Facebook, Inc. All Rights Reserved. Copyright (c) 2011, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2016, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -96,6 +97,9 @@ ut_crc32_func_t ut_crc32_legacy_big_endian; but very slow). */ ut_crc32_func_t ut_crc32_byte_by_byte; +/** Text description of CRC32 implementation */ +const char* ut_crc32_implementation; + /** Swap the byte order of an 8 byte integer. @param[in] i 8-byte integer @return 8-byte integer */ @@ -116,10 +120,6 @@ ut_crc32_swap_byteorder( /* CRC32 hardware implementation. */ -/* Flag that tells whether the CPU supports CRC32 or not */ -bool ut_crc32_sse2_enabled = false; -UNIV_INTERN bool ut_crc32_power8_enabled = false; - #ifdef HAVE_CRC32_VPMSUM extern "C" { unsigned int crc32c_vpmsum(unsigned int crc, const unsigned char *p, unsigned long len); @@ -284,8 +284,6 @@ ut_crc32_hw( { uint32_t crc = 0xFFFFFFFFU; - ut_a(ut_crc32_sse2_enabled); - /* Calculate byte-by-byte up to an 8-byte aligned address. After this consume the input 8-bytes at a time. */ while (len > 0 && (reinterpret_cast<uintptr_t>(buf) & 7) != 0) { @@ -375,8 +373,6 @@ ut_crc32_legacy_big_endian_hw( { uint32_t crc = 0xFFFFFFFFU; - ut_a(ut_crc32_sse2_enabled); - /* Calculate byte-by-byte up to an 8-byte aligned address. After this consume the input 8-bytes at a time. */ while (len > 0 && (reinterpret_cast<uintptr_t>(buf) & 7) != 0) { @@ -427,8 +423,6 @@ ut_crc32_byte_by_byte_hw( { uint32_t crc = 0xFFFFFFFFU; - ut_a(ut_crc32_sse2_enabled); - while (len > 0) { ut_crc32_8_hw(&crc, &buf, &len); } @@ -452,7 +446,6 @@ void ut_crc32_slice8_table_init() /*========================*/ { -#ifndef HAVE_CRC32_VPMSUM /* bit-reversed poly 0x1EDC6F41 (from SSE42 crc32 instruction) */ static const uint32_t poly = 0x82f63b78; uint32_t n; @@ -476,7 +469,6 @@ ut_crc32_slice8_table_init() } ut_crc32_slice8_table_initialized = true; -#endif } /** Calculate CRC32 over 8-bit data using a software implementation. @@ -706,6 +698,12 @@ void ut_crc32_init() /*===========*/ { + ut_crc32_slice8_table_init(); + ut_crc32 = ut_crc32_sw; + ut_crc32_legacy_big_endian = ut_crc32_legacy_big_endian_sw; + ut_crc32_byte_by_byte = ut_crc32_byte_by_byte_sw; + ut_crc32_implementation = "Using generic crc32 instructions"; + #if defined(__GNUC__) && defined(__x86_64__) uint32_t vend[3]; uint32_t model; @@ -733,27 +731,17 @@ ut_crc32_init() probably kill your program. */ -#ifndef UNIV_DEBUG_VALGRIND - ut_crc32_sse2_enabled = (features_ecx >> 20) & 1; -#endif /* UNIV_DEBUG_VALGRIND */ - if (ut_crc32_sse2_enabled) { + if (features_ecx & 1 << 20) { ut_crc32 = ut_crc32_hw; ut_crc32_legacy_big_endian = ut_crc32_legacy_big_endian_hw; ut_crc32_byte_by_byte = ut_crc32_byte_by_byte_hw; + ut_crc32_implementation = "Using SSE2 crc32 instructions"; } -#endif /* defined(__GNUC__) && defined(__x86_64__) */ - -#ifdef HAVE_CRC32_VPMSUM - ut_crc32_power8_enabled = true; +#elif defined(HAVE_CRC32_VPMSUM) ut_crc32 = ut_crc32_power8; + ut_crc32_implementation = "Using POWER8 crc32 instructions"; #endif - if (!ut_crc32_sse2_enabled && !ut_crc32_power8_enabled) { - ut_crc32_slice8_table_init(); - ut_crc32 = ut_crc32_sw; - ut_crc32_legacy_big_endian = ut_crc32_legacy_big_endian_sw; - ut_crc32_byte_by_byte = ut_crc32_byte_by_byte_sw; - } } diff --git a/storage/innobase/ut/ut0dbg.cc b/storage/innobase/ut/ut0dbg.cc index bcf9d5fd1b5..ed4a7778afa 100644 --- a/storage/innobase/ut/ut0dbg.cc +++ b/storage/innobase/ut/ut0dbg.cc @@ -37,16 +37,8 @@ ut_dbg_assertion_failed( ulint line) /*!< in: line number of the assertion */ { ut_print_timestamp(stderr); -#ifdef UNIV_HOTBACKUP fprintf(stderr, " InnoDB: Assertion failure in file %s line %lu\n", file, line); -#else /* UNIV_HOTBACKUP */ - fprintf(stderr, - " InnoDB: Assertion failure in thread " ULINTPF - " in file %s line " ULINTPF "\n", - os_thread_pf(os_thread_get_curr_id()), - innobase_basename(file), line); -#endif /* UNIV_HOTBACKUP */ if (expr) { fprintf(stderr, "InnoDB: Failing assertion: %s\n", expr); diff --git a/storage/innobase/ut/ut0mem.cc b/storage/innobase/ut/ut0mem.cc index 0ad251be157..39b86568aa3 100644 --- a/storage/innobase/ut/ut0mem.cc +++ b/storage/innobase/ut/ut0mem.cc @@ -29,11 +29,9 @@ Created 5/11/1994 Heikki Tuuri #include "ut0mem.ic" #endif -#ifndef UNIV_HOTBACKUP -# include "os0thread.h" -# include "srv0srv.h" -# include <stdlib.h> -#endif /* !UNIV_HOTBACKUP */ +#include "os0thread.h" +#include "srv0srv.h" +#include <stdlib.h> /**********************************************************************//** Copies up to size - 1 characters from the NUL-terminated string src to @@ -81,7 +79,6 @@ ut_strlcpy_rev( return(src_size); } -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Return the number of times s2 occurs in s1. Overlapping instances of s2 are only counted once. @@ -199,5 +196,3 @@ ut_strreplace( return(new_str); } - -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/ut/ut0new.cc b/storage/innobase/ut/ut0new.cc index c2e3eb813af..89d2d6c89ea 100644 --- a/storage/innobase/ut/ut0new.cc +++ b/storage/innobase/ut/ut0new.cc @@ -97,7 +97,6 @@ ut_new_boot() #ifdef UNIV_PFS_MEMORY static const char* auto_event_names[] = { /* Keep this list alphabetically sorted. */ - "api0api", "btr0btr", "btr0bulk", "btr0cur", diff --git a/storage/innobase/ut/ut0ut.cc b/storage/innobase/ut/ut0ut.cc index cc315148a90..2b9e3115a03 100644 --- a/storage/innobase/ut/ut0ut.cc +++ b/storage/innobase/ut/ut0ut.cc @@ -30,23 +30,14 @@ Created 5/11/1994 Heikki Tuuri #endif #ifndef UNIV_INNOCHECKSUM - -#ifndef UNIV_HOTBACKUP -# include <mysql_com.h> -#endif /* !UNIV_HOTBACKUP */ - +#include <mysql_com.h> #include "os0thread.h" #include "ut0ut.h" - #ifdef UNIV_NONINL #include "ut0ut.ic" #endif - -#ifndef UNIV_HOTBACKUP -# include "trx0trx.h" -#endif /* !UNIV_HOTBACKUP */ - -# include <string> +#include "trx0trx.h" +#include <string> #include "log.h" /** A constant to prevent the compiler from optimizing ut_delay() away. */ @@ -113,7 +104,7 @@ ut_time(void) return(time(NULL)); } -#ifndef UNIV_HOTBACKUP + /**********************************************************//** Returns system time. Upon successful completion, the value 0 is returned; otherwise the @@ -193,7 +184,6 @@ ut_time_ms(void) return((ulint) tv.tv_sec * 1000 + tv.tv_usec / 1000); } -#endif /* !UNIV_HOTBACKUP */ /**********************************************************//** Returns the difference of two times in seconds. @@ -293,78 +283,6 @@ ut_sprintf_timestamp( #endif } -#ifdef UNIV_HOTBACKUP -/**********************************************************//** -Sprintfs a timestamp to a buffer with no spaces and with ':' characters -replaced by '_'. */ -void -ut_sprintf_timestamp_without_extra_chars( -/*=====================================*/ - char* buf) /*!< in: buffer where to sprintf */ -{ -#ifdef _WIN32 - SYSTEMTIME cal_tm; - - GetLocalTime(&cal_tm); - - sprintf(buf, "%02d%02d%02d_%2d_%02d_%02d", - (int) cal_tm.wYear % 100, - (int) cal_tm.wMonth, - (int) cal_tm.wDay, - (int) cal_tm.wHour, - (int) cal_tm.wMinute, - (int) cal_tm.wSecond); -#else - struct tm* cal_tm_ptr; - time_t tm; - - struct tm cal_tm; - time(&tm); - localtime_r(&tm, &cal_tm); - cal_tm_ptr = &cal_tm; - sprintf(buf, "%02d%02d%02d_%2d_%02d_%02d", - cal_tm_ptr->tm_year % 100, - cal_tm_ptr->tm_mon + 1, - cal_tm_ptr->tm_mday, - cal_tm_ptr->tm_hour, - cal_tm_ptr->tm_min, - cal_tm_ptr->tm_sec); -#endif -} - -/**********************************************************//** -Returns current year, month, day. */ -void -ut_get_year_month_day( -/*==================*/ - ulint* year, /*!< out: current year */ - ulint* month, /*!< out: month */ - ulint* day) /*!< out: day */ -{ -#ifdef _WIN32 - SYSTEMTIME cal_tm; - - GetLocalTime(&cal_tm); - - *year = (ulint) cal_tm.wYear; - *month = (ulint) cal_tm.wMonth; - *day = (ulint) cal_tm.wDay; -#else - struct tm* cal_tm_ptr; - time_t tm; - - struct tm cal_tm; - time(&tm); - localtime_r(&tm, &cal_tm); - cal_tm_ptr = &cal_tm; - *year = (ulint) cal_tm_ptr->tm_year + 1900; - *month = (ulint) cal_tm_ptr->tm_mon + 1; - *day = (ulint) cal_tm_ptr->tm_mday; -#endif -} - -#else /* UNIV_HOTBACKUP */ - /*************************************************************//** Runs an idle loop on CPU. The argument gives the desired delay in microseconds on 100 MHz Pentium + Visual C++. @@ -390,7 +308,6 @@ ut_delay( return(j); } -#endif /* UNIV_HOTBACKUP */ /*************************************************************//** Prints the contents of a memory buffer in hex and ascii. */ @@ -495,7 +412,6 @@ ut_2_power_up( return(res); } -#ifndef UNIV_HOTBACKUP /** Get a fixed-length string, quoted as an SQL identifier. If the string contains a slash '/', the string will be output as two identifiers separated by a period (.), @@ -614,7 +530,6 @@ ut_copy_file( } } while (len > 0); } -#endif /* !UNIV_HOTBACKUP */ #ifdef _WIN32 # include <stdarg.h> @@ -812,10 +727,6 @@ ut_strerr( return("I/O error"); case DB_TABLE_IN_FK_CHECK: return("Table is being used in foreign key check"); - case DB_DATA_MISMATCH: - return("data mismatch"); - case DB_SCHEMA_NOT_LOCKED: - return("schema not locked"); case DB_NOT_FOUND: return("not found"); case DB_ONLINE_LOG_TOO_BIG: @@ -834,33 +745,20 @@ ut_strerr( return("Table is corrupted"); case DB_FTS_TOO_MANY_WORDS_IN_PHRASE: return("Too many words in a FTS phrase or proximity search"); - case DB_IO_DECOMPRESS_FAIL: - return("Page decompress failed after reading from disk"); case DB_DECRYPTION_FAILED: return("Table is encrypted but decrypt failed."); - case DB_IO_NO_PUNCH_HOLE: - return("No punch hole support"); - case DB_IO_NO_PUNCH_HOLE_FS: - return("Punch hole not supported by the file system"); - case DB_IO_NO_PUNCH_HOLE_TABLESPACE: - return("Punch hole not supported by the tablespace"); - case DB_IO_NO_ENCRYPT_TABLESPACE: - return("Page encryption not supported by the tablespace"); - case DB_IO_DECRYPT_FAIL: - return("Page decryption failed after reading from disk"); case DB_IO_PARTIAL_FAILED: return("Partial IO failed"); case DB_FORCED_ABORT: return("Transaction aborted by another higher priority " "transaction"); - case DB_WRONG_FILE_NAME: - return("Invalid Filename"); - case DB_COMPUTE_VALUE_FAILED: return("Compute generated column failed"); case DB_NO_FK_ON_S_BASE_COL: return("Cannot add foreign key on the base column " "of stored column"); + case DB_IO_NO_PUNCH_HOLE: + return ("File system does not support punch hole (trim) operation."); /* do not add default: in order to produce a warning if new code is added to the enum but not added here */ diff --git a/storage/innobase/ut/ut0wqueue.cc b/storage/innobase/ut/ut0wqueue.cc index cee30925375..ccd5c10555c 100644 --- a/storage/innobase/ut/ut0wqueue.cc +++ b/storage/innobase/ut/ut0wqueue.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2006, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -31,7 +32,9 @@ Created 4/26/2006 Osku Salerma struct ib_wqueue_t { ib_mutex_t mutex; /*!< mutex protecting everything */ ib_list_t* items; /*!< work item list */ - os_event_t event; /*!< event we use to signal additions to list */ + os_event_t event; /*!< event we use to signal additions to list; + os_event_set() and os_event_reset() are + protected by ib_wqueue_t::mutex */ }; /****************************************************************//** |